diff --git a/cmd/baton-github/main.go b/cmd/baton-github/main.go index 56742ec8..86577879 100644 --- a/cmd/baton-github/main.go +++ b/cmd/baton-github/main.go @@ -14,5 +14,8 @@ var version = "dev" func main() { ctx := context.Background() - config.RunConnector(ctx, "baton-github", version, cfg.Config, connector.NewLambdaConnector, connectorrunner.WithSessionStoreEnabled()) + config.RunConnector(ctx, "baton-github", version, cfg.Config, connector.NewLambdaConnector, + connectorrunner.WithSessionStoreEnabled(), + connectorrunner.WithKeepPreviousSyncC1Z(), + ) } diff --git a/go.mod b/go.mod index 1f7fa391..6f496511 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/conductorone/baton-github go 1.25.2 +replace github.com/conductorone/baton-sdk => /Users/matt.kaniaris/work/baton-sdk-2 + require ( github.com/conductorone/baton-sdk v0.17.0 github.com/deckarep/golang-set/v2 v2.9.0 diff --git a/pkg/config/config.go b/pkg/config/config.go index 2b9c5e63..aa0e0aa0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -94,6 +94,13 @@ var Config = field.NewConfiguration( syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, + // Default to a small worker pool: the source-cache warm path fans + // member pages out via SpawnCursors, and cold syncs parallelize + // per-resource work. 4 keeps sustained request rate well under + // GitHub's secondary (points-per-minute) limits; the SDK limiter + // backs off on 429/secondary responses regardless. --workers/ + // BATON_WORKERS still override. + field.WithConnectorDefault(field.WorkerCountField, 4), }, field.WithConnectorDisplayName("GitHub v2"), field.WithHelpUrl("/docs/baton/github-v2"), diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 2cdb5a62..5008e67b 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -3,7 +3,9 @@ package connector import ( "context" "crypto/rsa" + "crypto/sha256" "crypto/x509" + "encoding/hex" "encoding/pem" "errors" "fmt" @@ -127,11 +129,18 @@ type GitHub struct { omitArchivedRepositories bool directCollaboratorsOnly bool enterprises []string + // authScope is a stable, non-secret identity for the configured + // credential, mixed into source-cache scope signatures so two + // different credentials (with different member visibility) never + // share cached scopes. PATs use a token digest; GitHub Apps use the + // stable installation identity, which survives hourly installation + // token rotation. + authScope string } func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncerV2 { resourceSyncers := []connectorbuilder.ResourceSyncerV2{ - OrgBuilder(gh.client, gh.appClient, gh.orgCache, gh.orgs, gh.syncSecrets), + OrgBuilder(gh.client, gh.appClient, gh.orgCache, gh.orgs, gh.syncSecrets, gh.authScope), TeamBuilder(gh.client, gh.orgCache, gh.directCollaboratorsOnly), UserBuilder(gh.client, gh.graphqlClient, gh.orgCache, gh.orgs, gh.customClient, gh.enterprises), RepositoryBuilder(gh.client, gh.orgCache, gh.omitArchivedRepositories, gh.directCollaboratorsOnly), @@ -254,7 +263,7 @@ func (gh *GitHub) Validate(ctx context.Context) (annotations.Annotations, error) zap.Error(err)) } } - return nil, nil + return sourceCacheCapabilityAnnotations(), nil } func (gh *GitHub) validateAppCredentials(ctx context.Context) (annotations.Annotations, error) { @@ -279,7 +288,13 @@ func (gh *GitHub) validateAppCredentials(ctx context.Context) (annotations.Annot } } - return nil, nil + return sourceCacheCapabilityAnnotations(), nil +} + +func sourceCacheCapabilityAnnotations() annotations.Annotations { + return annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()) } // newGitHubClient returns a new GitHub API client authenticated with an access token via oauth2. @@ -346,9 +361,20 @@ func newWithGithubPAT(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { syncSecrets: ghc.SyncSecrets, omitArchivedRepositories: ghc.OmitArchivedRepositories, directCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + authScope: patAuthScope(ghc.Token), }, nil } +// patAuthScope derives a stable, non-secret identity from a personal +// access token for source-cache scope isolation. The digest is truncated +// (it only needs to distinguish credentials, not resist collision +// attacks) and never logged alongside anything that would identify it as +// token-derived material. +func patAuthScope(token string) string { + sum := sha256.Sum256([]byte("baton-github-auth-scope|" + token)) + return "pat:" + hex.EncodeToString(sum[:8]) +} + func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { jwttoken, err := getJWTToken(ghc.AppId, string(ghc.AppPrivatekeyPath)) if err != nil { @@ -433,6 +459,10 @@ func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { syncSecrets: ghc.SyncSecrets, omitArchivedRepositories: ghc.OmitArchivedRepositories, directCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + // The installation identity is stable across the hourly + // installation-token rotation, so cached scopes survive rotation + // (a per-token digest would go cold every hour). + authScope: fmt.Sprintf("app:%s:install:%d", ghc.AppId, installation.GetID()), } return gh, nil } diff --git a/pkg/connector/org.go b/pkg/connector/org.go index 4d17748d..ef573df4 100644 --- a/pkg/connector/org.go +++ b/pkg/connector/org.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "net/url" "strconv" "strings" @@ -39,6 +40,9 @@ type orgResourceType struct { orgs map[string]struct{} orgCache *orgNameCache syncSecrets bool + // authScope isolates source-cache scopes per credential; GitHub ETag + // visibility is credential-specific (see source_cache.go). + authScope string } func organizationResource( @@ -183,15 +187,120 @@ func (o *orgResourceType) orgRoleGrant(roleName string, org *v2.Resource, princi })) } +// orgMembersPageURL builds one page's URL with a fixed query-param order +// so the same logical page hashes to the same scope every sync. +func orgMembersPageURL(orgName, role string, page int) string { + q := url.Values{} + q.Set("page", strconv.Itoa(page)) + q.Set("per_page", strconv.Itoa(maxPageSize)) + q.Set("role", role) + return fmt.Sprintf("orgs/%v/members?%s", orgName, q.Encode()) +} + +// listMembersConditional fetches one page of the org member listing for +// one role, with source-cache revalidation (see source_cache.go for the +// scope model, the horizon clamp, and token-carried validators). +func (o *orgResourceType) listMembersConditional( + ctx context.Context, + orgName string, + role string, + cursor pagedListCursor, + opts resourceSdk.SyncOpAttrs, +) (*conditionalPage, error) { + pageURL := orgMembersPageURL(orgName, role, cursor.Page) + return listUsersPageConditional(ctx, o.client, opts.SourceCache, o.authScope, "org-members", + pageURL, cursor, maxPageSize) +} + +// cursorForResolvedPage builds a page cursor carrying the page's +// validator resolution from the origin's batched walk, so the page does +// no lookup of its own (zero ask/answer bounces on the Lambda +// continuation). Unqueried pages get an unresolved cursor (single-lookup +// fallback). +func cursorForResolvedPage(page int, horizon int, resolved map[int]resolvedPage) pagedListCursor { + c := pagedListCursor{Page: page, Horizon: horizon} + r, ok := resolved[page] + if !ok { + return c + } + if r.Found { + c.Resolution = cursorValidatorHit + c.Etag = r.Validator + } else { + c.Resolution = cursorValidatorMiss + } + return c +} + +// spawnMemberPageCursors resolves one role collection's stored page chain +// (a single batched lookup — one ask bounce on the Lambda continuation) +// and computes the spawn set: on a warm sync the connector emits one +// sibling cursor per stored page 2..horizon and the SDK worker pool +// revalidates them concurrently instead of one round trip per page +// serially. Each token is a self-contained single-state pagination bag +// (the resource identity rides the spawned action) carrying the page's +// validator, so siblings do no lookups of their own. +// +// Returns the page-1 cursor (with its own resolution embedded), the +// marshaled sibling tokens (nil when there is no stored chain — cold +// syncs paginate serially exactly as before), or an error, which may +// wrap sourcecache.ErrLookupDeferred and must propagate. +func (o *orgResourceType) spawnMemberPageCursors( + ctx context.Context, + orgName string, + role string, + opts resourceSdk.SyncOpAttrs, +) (pagedListCursor, []string, error) { + resolved, horizon, err := resolveStoredPageChain(ctx, o.client, opts.SourceCache, o.authScope, "org-members", + func(page int) string { return orgMembersPageURL(orgName, role, page) }) + if err != nil { + return pagedListCursor{}, nil, err + } + if horizon < 2 { + // One stored page or none: nothing to fan out. Page 1 still + // carries its own resolution (and the real horizon, for the + // past-horizon miss propagation on chains) so no further lookups + // happen. + return cursorForResolvedPage(1, horizon, resolved), nil, nil + } + tokens := make([]string, 0, horizon-1) + for p := 2; p <= horizon; p++ { + cursorTok, err := cursorForResolvedPage(p, horizon, resolved).marshal() + if err != nil { + return pagedListCursor{}, nil, err + } + sibling := &pagination.Bag{} + sibling.Push(pagination.PageState{ + ResourceTypeID: role, + Token: cursorTok, + }) + tok, err := sibling.Marshal() + if err != nil { + return pagedListCursor{}, nil, err + } + tokens = append(tokens, tok) + } + return cursorForResolvedPage(1, horizon, resolved), tokens, nil +} + func (o *orgResourceType) Grants( ctx context.Context, resource *v2.Resource, opts resourceSdk.SyncOpAttrs, ) ([]*v2.Grant, *resourceSdk.SyncOpResults, error) { - bag, page, err := parsePageToken(opts.PageToken.Token, resource.Id) - if err != nil { + // The role states carry a JSON pagedListCursor token, so the bag is + // unmarshaled directly instead of through parsePageToken (which + // assumes integer tokens). + bag := &pagination.Bag{} + if err := bag.Unmarshal(opts.PageToken.Token); err != nil { return nil, nil, err } + if bag.Current() == nil { + bag.Push(pagination.PageState{ + ResourceTypeID: resource.Id.ResourceType, + ResourceID: resource.Id.Resource, + }) + } var ( reqAnnos annotations.Annotations @@ -214,33 +323,66 @@ func (o *orgResourceType) Grants( if err != nil { return nil, nil, err } - listOpts := github.ListMembersOptions{ - Role: rId, - ListOptions: github.ListOptions{ - Page: page, - PerPage: maxPageSize, - }, + cursor, err := parsePagedListCursor(bag.PageToken()) + if err != nil { + return nil, nil, err + } + + // The collection's first call resolves the stored chain in one + // batched lookup and fans out: spawn one sibling cursor per stored + // page (each carrying its validator, so siblings do no lookups), + // and adopt the horizon so this page's own chain never continues + // into sibling-owned pages. Spawned and chained cursors carry a + // resolution/horizon already and never re-spawn. + var spawnTokens []string + if cursor.Page == 1 && cursor.Horizon == 0 && cursor.Resolution == "" { + cursor, spawnTokens, err = o.spawnMemberPageCursors(ctx, orgName, rId, opts) + if err != nil { + return nil, nil, err + } } - users, resp, err := o.client.Organizations.ListMembers(ctx, orgName, &listOpts) + + pageRes, err := o.listMembersConditional(ctx, orgName, rId, cursor, opts) if err != nil { + var resp *github.Response + if pageRes != nil { + resp = pageRes.Resp + } if isNotFoundError(resp) { return nil, nil, uhttp.WrapErrors(codes.NotFound, fmt.Sprintf("org: %s not found", orgName)) } - return nil, nil, wrapGitHubError(err, resp, "github-connector: failed to list org members") + return nil, nil, err } - - var nextPage string - nextPage, reqAnnos, err = parseResp(resp) - if err != nil { - return nil, nil, fmt.Errorf("github-connectorv2: failed to parse response: %w", err) + reqAnnos = pageRes.Annos + if len(spawnTokens) > 0 { + reqAnnos.Append(v2.SpawnCursors_builder{ + PageTokens: spawnTokens, + }.Build()) } - err = bag.Next(nextPage) + nextToken := "" + if pageRes.NextPage != 0 { + nextCursor := pagedListCursor{Page: pageRes.NextPage, Horizon: cursor.Horizon} + // Chained pages beyond the stored chain are known misses (the + // chain is contiguous): a cold page's continuation, and the + // probe past the horizon. Carrying the miss skips their + // lookups; if ever wrong it merely forgoes a conditional + // request and fetches cold — never stale. + if cursor.Resolution == cursorValidatorMiss || + (cursor.Horizon > 0 && pageRes.NextPage > cursor.Horizon) { + nextCursor.Resolution = cursorValidatorMiss + } + nextToken, err = nextCursor.marshal() + if err != nil { + return nil, nil, err + } + } + err = bag.Next(nextToken) if err != nil { return nil, nil, err } - for _, user := range users { + for _, user := range pageRes.Users { ur, err := userResource(ctx, user, user.GetEmail(), nil) if err != nil { return nil, nil, err @@ -257,7 +399,7 @@ func (o *orgResourceType) Grants( ) } - pageToken, err = bag.Marshal() + pageToken, err := bag.Marshal() if err != nil { return nil, nil, err } @@ -414,7 +556,7 @@ func (o *orgResourceType) Revoke(ctx context.Context, grant *v2.Grant) (annotati return nil, nil } -func OrgBuilder(client, appClient *github.Client, orgCache *orgNameCache, orgs []string, syncSecrets bool) *orgResourceType { +func OrgBuilder(client, appClient *github.Client, orgCache *orgNameCache, orgs []string, syncSecrets bool, authScope string) *orgResourceType { orgMap := make(map[string]struct{}) for _, o := range orgs { @@ -428,6 +570,7 @@ func OrgBuilder(client, appClient *github.Client, orgCache *orgNameCache, orgs [ appClient: appClient, orgCache: orgCache, syncSecrets: syncSecrets, + authScope: authScope, } } diff --git a/pkg/connector/org_test.go b/pkg/connector/org_test.go index de32fe23..95285aa1 100644 --- a/pkg/connector/org_test.go +++ b/pkg/connector/org_test.go @@ -24,7 +24,7 @@ func TestOrganization(t *testing.T) { githubClient := github.NewClient(mgh.Server()) cache := newOrgNameCache(githubClient) - client := OrgBuilder(githubClient, nil, cache, nil, false) + client := OrgBuilder(githubClient, nil, cache, nil, false, "test-auth") organization, _ := organizationResource(ctx, githubOrganization, nil, false) user, _ := userResource(ctx, githubUser, *githubUser.Email, nil) diff --git a/pkg/connector/source_cache.go b/pkg/connector/source_cache.go new file mode 100644 index 00000000..1f3e38bb --- /dev/null +++ b/pkg/connector/source_cache.go @@ -0,0 +1,376 @@ +package connector + +// Source-cache replay for GitHub conditional requests. +// +// GitHub is the conditional-request model: a warm sync re-presents the +// previous sync's ETag via If-None-Match; a 304 means "this page is +// byte-identical to last sync" and the SDK replays the page's previous +// rows from the prior c1z. Authorized 304s do not count against the +// primary rate limit, so warm syncs of unchanged data are nearly free. +// +// Scope model: ONE SCOPE PER PAGE, not one scope per collection. +// +// The handoff design preferred a whole-collection scope validated by +// page 1's ETag, contingent on an empirical property: "a change anywhere +// in the collection changes page 1's ETag". For GitHub member listings +// that property should be presumed FALSE — members are returned in +// ascending user-id order, so a newly added member lands on the LAST +// page and page 1's body-hash ETag does not move. A collection scope +// keyed on page 1 would silently drop new members on warm syncs. Until +// that property is verified live per endpoint, we use the documented +// fallback: store every page's ETag and revalidate each page. +// +// Per-page soundness: a 304 for page k attests that page k's bytes are +// identical to the stored rows for that page's scope, so replaying them +// is exact. Pages whose contents shifted (insert/delete before them) +// answer 200 and are re-fetched fresh. The union of live pages is +// duplicate-free because all pages are read from the live listing in one +// sync. +// +// Chain continuation on 304: GitHub 304 responses carry no usable Link +// header, so "is there a page k+1" cannot come from the response. The +// validator therefore encodes the page's row count. A full page (row +// count == per_page) means a next page MAY exist and must be visited — +// including the append-past-a-full-tail case, where the probe of page +// k+1 either finds fresh rows (200) or an empty listing (200 []) that +// ends the chain. A partial page ends the chain, and any append onto a +// partial page changes its bytes, forcing a 200. +// +// Scope signatures are versioned (bump scopeSigVersion when the row +// shape changes so every old scope misses at once) and include the auth +// identity, because GitHub ETag visibility is credential-specific: two +// different tokens must never share scopes. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/google/go-github/v69/github" +) + +// scopeSigVersion invalidates ALL previously stored scopes when bumped. +// Bump it whenever the rows emitted for a page change shape (new grant +// fields, different grant ids, different page size, ...). +const scopeSigVersion = "v1" + +// validatorVersion namespaces the encoded validator format. +const validatorVersion = "v1" + +// pageScopeSig is the canonical pre-image for one page's scope hash. The +// request URL carries host (GHE instance), path (org login), and the +// full query shape (role, per_page, page); authScope isolates +// credentials with different visibility. +func pageScopeSig(kind string, authScope string, requestURL string) string { + return scopeSigVersion + "|gh|" + kind + "|auth=" + authScope + "|GET " + requestURL +} + +// encodeValidator packs the page's verbatim ETag together with the row +// count the page produced. The row count drives chain continuation on a +// 304 (full page => a following page may exist). The whole string is +// opaque to the SDK. +func encodeValidator(etag string, rows int) string { + return validatorVersion + "|n=" + strconv.Itoa(rows) + "|" + etag +} + +// decodeValidator reverses encodeValidator. ok=false (unknown version, +// malformed) must degrade to a plain fetch, never an error. +func decodeValidator(v string) (etag string, rows int, ok bool) { + rest, found := strings.CutPrefix(v, validatorVersion+"|n=") + if !found { + return "", 0, false + } + nStr, etag, found := strings.Cut(rest, "|") + if !found { + return "", 0, false + } + n, err := strconv.Atoi(nStr) + if err != nil || n < 0 { + return "", 0, false + } + return etag, n, true +} + +// Validator resolution states carried by pagedListCursor. Unresolved +// (empty) means "do a lookup for this page"; the other two mean the +// origin's batched lookup already resolved it, so the page does NO lookup +// of its own — on the Lambda ask/answer continuation that is the +// difference between zero bounces and one bounce per page. +const ( + // cursorValidatorHit: Etag holds the stored validator verbatim. + cursorValidatorHit = "hit" + // cursorValidatorMiss: the previous sync has no entry for this page + // (known from the origin's contiguous-chain walk); fetch cold. + cursorValidatorMiss = "miss" +) + +// pagedListCursor is the bag token for a source-cached paged listing. +// Page is 1-based; page 0/empty token means page 1. +// +// Horizon is the spawn horizon: the last page of this collection covered +// by a spawned sibling cursor this sync (see resolveStoredPageChain). +// Pages ≤ Horizon are sibling-owned, so a page's own chain never +// continues into them — without this clamp the origin's chain and a +// sibling would both serve the same page. It rides the token (not a +// lookup re-derivation) so it is exact even if the stored manifest has +// gaps, and survives suspend/resume on any worker. +// +// Resolution/Etag carry the page's validator resolution from the +// origin's batched lookup (see the constants above). Carrying validators +// through connector-authored tokens within one sync is blessed by the +// source-cache contract: the replay invariant is "validator came from +// THIS SYNC's lookup", not "from this action's lookup". Old tokens +// without these fields decode as unresolved and behave exactly as +// before. +type pagedListCursor struct { + Page int `json:"page"` + Horizon int `json:"horizon,omitempty"` + Resolution string `json:"res,omitempty"` + Etag string `json:"etag,omitempty"` +} + +func parsePagedListCursor(token string) (pagedListCursor, error) { + if token == "" { + return pagedListCursor{Page: 1}, nil + } + var c pagedListCursor + if err := json.Unmarshal([]byte(token), &c); err != nil { + return pagedListCursor{}, fmt.Errorf("github-connector: bad page cursor %q: %w", token, err) + } + if c.Page < 1 { + c.Page = 1 + } + return c, nil +} + +func (c pagedListCursor) marshal() (string, error) { + b, err := json.Marshal(c) + if err != nil { + return "", err + } + return string(b), nil +} + +// conditionalPage is the outcome of one source-cached page fetch. +type conditionalPage struct { + // Users holds the page's fresh rows. Empty on a 304 replay. + Users []*github.User + // Replayed is true when the page answered 304 and the SDK will copy + // the previous sync's rows for this scope. + Replayed bool + // NextPage is the 1-based page to fetch next; 0 ends the chain. + NextPage int + // Annos carries rate-limit plus the page's SourceCacheScope or + // SourceCacheReplay annotation. + Annos annotations.Annotations + // Resp is the raw response for error classification (may be nil). + Resp *github.Response +} + +// listUsersPageConditional fetches one page of a user-list endpoint with +// source-cache revalidation. pageURL must be byte-stable across syncs +// for the same logical page (fixed query-param order, explicit page and +// per_page). kind names the endpoint family within the scope signature. +// perPage must equal the per_page baked into pageURL; it drives the +// full-page chain-continuation check on a 304. cursor supplies the page +// number, the sibling horizon clamp, and — when the origin's batched +// walk already resolved it — the page's validator, in which case no +// lookup happens here at all (zero ask/answer bounces on the Lambda +// continuation). An unresolved cursor (legacy checkpointed token) falls +// back to a single lookup. +func listUsersPageConditional( + ctx context.Context, + client *github.Client, + lookup sourcecache.Lookup, + authScope string, + kind string, + pageURL string, + cursor pagedListCursor, + perPage int, +) (*conditionalPage, error) { + req, err := client.NewRequest(http.MethodGet, pageURL, nil) + if err != nil { + return nil, err + } + page, horizon := cursor.Page, cursor.Horizon + + scope := sourcecache.HashScope(pageScopeSig(kind, authScope, req.URL.String())) + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + storedValidator, found := "", false + switch cursor.Resolution { + case cursorValidatorHit: + storedValidator, found = cursor.Etag, true + case cursorValidatorMiss: + // Known cold; skip the lookup. + default: + entry, entryFound, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + // May wrap sourcecache.ErrLookupDeferred (ask/answer + // continuation); must propagate unswallowed. + return nil, err + } + if entryFound { + storedValidator, found = entry.ETag, true + } + } + prevEtag, prevRows := "", 0 + validatorOK := false + if found { + prevEtag, prevRows, validatorOK = decodeValidator(storedValidator) + } + // A hit with a decodable validator makes the request conditional; any + // miss or malformed validator degrades to a plain (cold) fetch. + conditional := validatorOK && prevEtag != "" + if conditional { + req.Header.Set("If-None-Match", prevEtag) + } + + resp, err := client.BareDo(ctx, req) + _, annos, parseErr := parseResp(resp) + if parseErr != nil { + return nil, fmt.Errorf("github-connector: failed to parse response: %w", parseErr) + } + + if resp != nil && resp.StatusCode == http.StatusNotModified { + if !conditional { + // Invariant guard: a replay is only legal for a scope whose + // validator came from this sync's lookup. + return nil, fmt.Errorf("github-connector: 304 for %s without a source-cache lookup hit", pageURL) + } + next := 0 + if prevRows >= perPage { + // The stored page was full, so a following page may exist (it + // did last sync, or an append created one since). Visit it: a + // stale hint costs one request against the live listing, which + // answers with fresh rows or an empty 200 that ends the chain. + next = page + 1 + } + if next != 0 && next <= horizon { + // A spawned sibling owns that page; chaining into it would + // serve it twice. + next = 0 + } + annos.Append(v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: storedValidator, + }.Build()) + return &conditionalPage{Replayed: true, NextPage: next, Annos: annos, Resp: resp}, nil + } + if err != nil { + return &conditionalPage{Resp: resp}, wrapGitHubError(err, resp, "github-connector: failed to list "+kind) + } + defer resp.Body.Close() + + var users []*github.User + if err := json.NewDecoder(resp.Body).Decode(&users); err != nil { + return &conditionalPage{Resp: resp}, fmt.Errorf("github-connector: failed to decode %s page: %w", kind, err) + } + + // A 200 with zero rows still carries the scope annotation: the etag + // must survive to the next sync (an empty page is replayable too). + etag := resp.Header.Get("ETag") + scopeEtag := "" + if etag != "" { + scopeEtag = encodeValidator(etag, len(users)) + } + annos.Append(v2.SourceCacheScope_builder{ + ScopeHash: scope, + Etag: scopeEtag, + }.Build()) + next := resp.NextPage + if next != 0 && next <= horizon { + // Sibling-owned page (see the 304 branch): the sibling + // revalidates it independently, fresh or replayed. + next = 0 + } + return &conditionalPage{Users: users, NextPage: next, Annos: annos, Resp: resp}, nil +} + +// storedChainBatchSize is how many pages one LookupMany batch covers. +// On the Lambda ask/answer continuation each batch costs one bounce +// (bounce cap: 4), so the batch size bounds the resolvable chain: +// 3 batches × 256 pages ≈ 76k members per role before the origin action +// would fail the cap — far beyond any real GitHub collection. Validators +// are ~150 bytes encoded, so a full batch is well under the transport's +// answer budget (no truncation re-asks). +const storedChainBatchSize = 256 + +// storedChainBatchCap bounds the total walk; guards a pathological +// manifest (and stays a batch short of the continuation's bounce cap). +const storedChainBatchCap = 3 + +// resolvedPage is one page's validator resolution from the origin's +// batched walk, embeddable into a pagedListCursor. +type resolvedPage struct { + Page int + // Found + Validator mirror the manifest answer for the page's scope. + Found bool + Validator string +} + +// resolveStoredPageChain resolves the collection's stored page chain in +// LookupMany batches, starting at page 1. It returns the resolution for +// every page 1..horizon+1 that was queried, and the horizon: the last +// page of the CONTIGUOUS stored chain (0 = nothing stored, cold sync). +// +// One call resolves everything the sync needs for this collection — +// page 1's own validator, every sibling's validator (carried to the +// spawned cursors so siblings do no lookups of their own), and the +// horizon. On the Lambda continuation the whole walk costs one ask +// bounce per batch; in-process it is a loop of local point-reads. +func resolveStoredPageChain( + ctx context.Context, + client *github.Client, + lookup sourcecache.Lookup, + authScope string, + kind string, + pageURLFor func(page int) string, +) (map[int]resolvedPage, int, error) { + resolved := map[int]resolvedPage{} + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + + horizon := 0 + chainBroken := false + for batch := 0; batch < storedChainBatchCap && !chainBroken; batch++ { + first := batch*storedChainBatchSize + 1 + queries := make([]sourcecache.Query, 0, storedChainBatchSize) + pages := make([]int, 0, storedChainBatchSize) + for p := first; p < first+storedChainBatchSize; p++ { + req, err := client.NewRequest(http.MethodGet, pageURLFor(p), nil) + if err != nil { + return nil, 0, err + } + queries = append(queries, sourcecache.Query{ + RowKind: sourcecache.RowKindGrants, + ScopeHash: sourcecache.HashScope(pageScopeSig(kind, authScope, req.URL.String())), + }) + pages = append(pages, p) + } + answers, err := sourcecache.LookupMany(ctx, lookup, queries) + if err != nil { + // May wrap sourcecache.ErrLookupDeferred; propagate. + return nil, 0, err + } + for i, a := range answers { + p := pages[i] + resolved[p] = resolvedPage{Page: p, Found: a.Found, Validator: a.ETag} + if !a.Found && !chainBroken { + chainBroken = true + } + if a.Found && !chainBroken { + horizon = p + } + } + } + return resolved, horizon, nil +} diff --git a/pkg/connector/source_cache_lookup_test.go b/pkg/connector/source_cache_lookup_test.go new file mode 100644 index 00000000..f235cdb9 --- /dev/null +++ b/pkg/connector/source_cache_lookup_test.go @@ -0,0 +1,155 @@ +package connector + +// Pins the connector side of the ask/answer lookup continuation (the +// Lambda topology) and the token-carried validator contract: +// +// - phase 1: a deferring lookup makes Grants fail with +// ErrLookupDeferred BEFORE any upstream HTTP call (lookup-before- +// fetch), with the whole stored-chain walk collected into ONE batch +// ask; +// - phase 2: the same call served from answers proceeds normally; +// - a cursor carrying a resolved validator (hit or miss) performs NO +// lookup at all — zero bounces for spawned siblings and chained +// pages. + +import ( + "context" + "net/http/httptest" + "net/url" + "testing" + + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + resourceSdk "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/google/go-github/v69/github" + "github.com/stretchr/testify/require" +) + +// panicLookup fails the test if any lookup reaches it: pages with +// token-carried validators must not look anything up. +type panicLookup struct{ t *testing.T } + +func (p panicLookup) LookupPreviousSourceCache(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + p.t.Fatal("lookup called for a page with a token-carried validator") + return sourcecache.Entry{}, false, nil +} + +func newLookupTestClient(t *testing.T, mock *mockGitHubOrg) *github.Client { + t.Helper() + server := httptest.NewServer(mock.handler()) + t.Cleanup(server.Close) + ghClient := github.NewClient(server.Client()) + baseURL, err := url.Parse(server.URL + "/") + require.NoError(t, err) + ghClient.BaseURL = baseURL + return ghClient +} + +func memberRoleToken(t *testing.T, cursor pagedListCursor) string { + t.Helper() + cursorTok, err := cursor.marshal() + require.NoError(t, err) + bag := &pagination.Bag{} + bag.Push(pagination.PageState{ResourceTypeID: orgRoleMember, Token: cursorTok}) + tok, err := bag.Marshal() + require.NoError(t, err) + return tok +} + +func TestOrgGrantsLookupContinuation(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + mock := newMockGitHubOrg(t) + for i := int64(1); i <= 150; i++ { + mock.addMember(i, false) // 2 member pages + } + ghClient := newLookupTestClient(t, mock) + + builder := OrgBuilder(ghClient, nil, newOrgNameCache(ghClient), []string{mockOrgLogin}, false, "test-auth") + orgRes, err := organizationResource(ctx, &github.Organization{ + ID: github.Ptr(mockOrgID), + Login: github.Ptr(mockOrgLogin), + }, nil, false) + require.NoError(t, err) + + // Phase 1: continuation lookup with no answers. The role page-1 call + // must die at the batched walk with ErrLookupDeferred, before ANY + // upstream request, and the whole walk must be one recorded batch. + cont := sourcecache.NewContinuationLookup(nil) + _, _, err = builder.Grants(ctx, orgRes, resourceSdk.SyncOpAttrs{ + PageToken: pagination.Token{Token: memberRoleToken(t, pagedListCursor{Page: 1})}, + Session: &noOpSessionStore{}, + SourceCache: cont, + }) + require.Error(t, err) + require.ErrorIs(t, err, sourcecache.ErrLookupDeferred, + "deferred lookups must propagate unswallowed (Lambda phase 1)") + require.Zero(t, mock.snapshotCounts()["members-200"], + "phase 1 must defer before any upstream member fetch") + + asked := cont.Asked() + require.Len(t, asked, storedChainBatchSize, "the stored-chain walk collects into one batch ask") + + // Phase 2: same call, answers attached (all not-found → cold path). + answers := make([]sourcecache.Answer, 0, len(asked)) + for _, q := range asked { + answers = append(answers, sourcecache.Answer{Query: q, Found: false}) + } + served := sourcecache.NewContinuationLookup(answers) + grants, res, err := builder.Grants(ctx, orgRes, resourceSdk.SyncOpAttrs{ + PageToken: pagination.Token{Token: memberRoleToken(t, pagedListCursor{Page: 1})}, + Session: &noOpSessionStore{}, + SourceCache: served, + }) + require.NoError(t, err, "phase 2 must serve the same call from answers") + require.Len(t, grants, maxPageSize, "cold page 1 fetches fresh") + require.NotEmpty(t, res.NextPageToken, "cold chain continues serially") + c := mock.snapshotCounts() + require.Equal(t, 1, c["members-200"]) + require.Empty(t, served.Asked(), "a fully answered batch must not re-ask") +} + +func TestTokenCarriedValidatorsSkipLookups(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + mock := newMockGitHubOrg(t) + for i := int64(1); i <= 250; i++ { + mock.addMember(i, false) // 3 member pages + } + ghClient := newLookupTestClient(t, mock) + pageURL := orgMembersPageURL(mockOrgLogin, orgRoleMember, 2) + + // Learn page 2's live ETag and row count with a plain fetch (this is + // what a previous sync's manifest would hold). + warmup, err := listUsersPageConditional(ctx, ghClient, sourcecache.NoopLookup{}, "test-auth", "org-members", + pageURL, pagedListCursor{Page: 2}, maxPageSize) + require.NoError(t, err) + require.Len(t, warmup.Users, maxPageSize) + validator := encodeValidator(warmup.Resp.Header.Get("ETag"), len(warmup.Users)) + _ = mock.snapshotCounts() + + // A hit-resolved cursor revalidates conditionally (304 → replay) with + // NO lookup: panicLookup proves the token is the only source. + page, err := listUsersPageConditional(ctx, ghClient, panicLookup{t: t}, "test-auth", "org-members", + pageURL, pagedListCursor{Page: 2, Horizon: 3, Resolution: cursorValidatorHit, Etag: validator}, maxPageSize) + require.NoError(t, err) + require.True(t, page.Replayed, "matching token-carried validator must 304-replay") + require.Zero(t, page.NextPage, "full page's probe candidate (3) is sibling-owned and clamped") + c := mock.snapshotCounts() + require.Equal(t, 1, c["members-304"]) + require.Zero(t, c["members-200"]) + + // A miss-resolved cursor fetches cold with NO lookup and no + // conditional header (the strict mock rejects unissued If-None-Match). + page, err = listUsersPageConditional(ctx, ghClient, panicLookup{t: t}, "test-auth", "org-members", + orgMembersPageURL(mockOrgLogin, orgRoleMember, 3), pagedListCursor{Page: 3, Resolution: cursorValidatorMiss}, maxPageSize) + require.NoError(t, err) + require.False(t, page.Replayed) + require.Len(t, page.Users, 50) + c = mock.snapshotCounts() + require.Equal(t, 1, c["members-200"]) + require.Zero(t, c["members-304"]) +} diff --git a/pkg/connector/sourcecache_fuzz_test.go b/pkg/connector/sourcecache_fuzz_test.go new file mode 100644 index 00000000..e1589f2b --- /dev/null +++ b/pkg/connector/sourcecache_fuzz_test.go @@ -0,0 +1,280 @@ +package connector + +// Randomized churn equivalence fuzzer for the GitHub source-cache warm +// path, ported from the baton-microsoft-entra POC. +// +// The scripted scenarios in sourcecache_sync_test.go each pin one known +// hazard. This test covers the space BETWEEN them: every round applies a +// random batch of org mutations (member add/remove anywhere in the id +// order, promote/demote, login renames, occasional total ETag eviction), +// runs a warm sync chained off the previous round's output, runs a fresh +// uncached control sync of the same org state, and requires the two to be +// byte-identical at the reader surface. Any divergence — a stale replay, +// a dropped tail page, a missed boundary shift — fails with the round's +// seed and mutation log, which replays deterministically. +// +// Runs are deterministic by default (fixed seed) so CI is stable; set +// BATON_FUZZ_SEED to explore a different trajectory, and BATON_FUZZ_ROUNDS +// to run longer soaks. + +import ( + "fmt" + "math/rand" + "os" + "sort" + "strconv" + "testing" + + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/stretchr/testify/require" +) + +// ghFuzzView is a locked copy of the mock org's mutable state, used to +// pick valid mutation targets without racing the handler. +type ghFuzzView struct { + memberIDs []int64 // non-admin + adminIDs []int64 +} + +func (m *mockGitHubOrg) fuzzView() ghFuzzView { + m.mu.Lock() + defer m.mu.Unlock() + var v ghFuzzView + for id, mem := range m.members { + if mem.Admin { + v.adminIDs = append(v.adminIDs, id) + } else { + v.memberIDs = append(v.memberIDs, id) + } + } + sort.Slice(v.memberIDs, func(i, j int) bool { return v.memberIDs[i] < v.memberIDs[j] }) + sort.Slice(v.adminIDs, func(i, j int) bool { return v.adminIDs[i] < v.adminIDs[j] }) + return v +} + +func (v ghFuzzView) allIDs() []int64 { + return append(append([]int64{}, v.memberIDs...), v.adminIDs...) +} + +type ghFuzzOp struct { + name string + ready func(v ghFuzzView) bool + apply func(f *ghFuzzRun, v ghFuzzView) +} + +type ghFuzzRun struct { + t *testing.T + m *mockGitHubOrg + rng *rand.Rand + log []string +} + +func (f *ghFuzzRun) note(format string, args ...any) { + f.log = append(f.log, fmt.Sprintf(format, args...)) +} + +func (f *ghFuzzRun) pickID(ids []int64) int64 { + return ids[f.rng.Intn(len(ids))] +} + +// freshID picks an unused id anywhere in [1, 5000]: low ids fuzz +// mid-collection insertion (page-boundary shifts), high ids fuzz the +// append-at-end / full-tail-probe path. +func (f *ghFuzzRun) freshID(v ghFuzzView) int64 { + used := map[int64]bool{} + for _, id := range v.allIDs() { + used[id] = true + } + for { + id := int64(1 + f.rng.Intn(5000)) + if !used[id] { + return id + } + } +} + +func ghFuzzOps() []ghFuzzOp { + return []ghFuzzOp{ + { + name: "add-member", + ready: func(v ghFuzzView) bool { return true }, + apply: func(f *ghFuzzRun, v ghFuzzView) { + id := f.freshID(v) + f.m.addMember(id, false) + f.note("add-member %d", id) + }, + }, + { + name: "add-admin", + ready: func(v ghFuzzView) bool { return true }, + apply: func(f *ghFuzzRun, v ghFuzzView) { + id := f.freshID(v) + f.m.addMember(id, true) + f.note("add-admin %d", id) + }, + }, + { + name: "remove-member", + ready: func(v ghFuzzView) bool { return len(v.memberIDs) > 1 }, + apply: func(f *ghFuzzRun, v ghFuzzView) { + id := f.pickID(v.memberIDs) + f.m.removeMember(id) + f.note("remove-member %d", id) + }, + }, + { + name: "remove-admin", + ready: func(v ghFuzzView) bool { return len(v.adminIDs) > 1 }, + apply: func(f *ghFuzzRun, v ghFuzzView) { + id := f.pickID(v.adminIDs) + f.m.removeMember(id) + f.note("remove-admin %d", id) + }, + }, + { + name: "promote", + ready: func(v ghFuzzView) bool { return len(v.memberIDs) > 1 }, + apply: func(f *ghFuzzRun, v ghFuzzView) { + id := f.pickID(v.memberIDs) + f.m.setAdmin(id, true) + f.note("promote %d", id) + }, + }, + { + name: "demote", + ready: func(v ghFuzzView) bool { return len(v.adminIDs) > 1 }, + apply: func(f *ghFuzzRun, v ghFuzzView) { + id := f.pickID(v.adminIDs) + f.m.setAdmin(id, false) + f.note("demote %d", id) + }, + }, + { + // A login change perturbs the page's bytes (forcing a 200 and + // a fresh fetch) without changing any grant row — the warm and + // control outputs must still match exactly. + name: "rename", + ready: func(v ghFuzzView) bool { return len(v.memberIDs)+len(v.adminIDs) > 0 }, + apply: func(f *ghFuzzRun, v ghFuzzView) { + ids := v.allIDs() + id := ids[f.rng.Intn(len(ids))] + login := fmt.Sprintf("renamed-%d-%d", id, f.rng.Intn(1000)) + f.m.renameMember(id, login) + f.note("rename %d -> %s", id, login) + }, + }, + } +} + +func ghFuzzEnvInt(name string, def int) int { + if s := os.Getenv(name); s != "" { + if n, err := strconv.Atoi(s); err == nil { + return n + } + } + return def +} + +func TestGitHubSourceCacheChurnFuzz(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runGitHubChurnFuzz(t, workers) + }) + } +} + +func runGitHubChurnFuzz(t *testing.T, workers int) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + seed := int64(ghFuzzEnvInt("BATON_FUZZ_SEED", 20260711)) + rounds := ghFuzzEnvInt("BATON_FUZZ_ROUNDS", 8) + t.Logf("churn fuzz: seed=%d rounds=%d workers=%d (override with BATON_FUZZ_SEED / BATON_FUZZ_ROUNDS)", seed, rounds, workers) + + mock := newMockGitHubOrg(t) + + // Seed org: ~2.5 member pages plus admins, so boundary shifts and the + // tail page are in play from round zero. + for i := int64(1); i <= 230; i++ { + mock.addMember(i*3, false) // gaps leave room for mid-list inserts + } + for i := int64(1); i <= 5; i++ { + mock.addMember(4000+i, true) + } + + h := newGHSyncHarness(ctx, t, mock) + h.workers = workers + f := &ghFuzzRun{t: t, m: mock, rng: rand.New(rand.NewSource(seed))} + ops := ghFuzzOps() + + prev := h.runSync("fuzz-cold", "") + + for round := 1; round <= rounds; round++ { + nOps := 1 + f.rng.Intn(3) + f.log = f.log[:0] + for i := 0; i < nOps; i++ { + // Re-view after each mutation so ops in the same round compose + // against current state, exactly as real churn would. + v := mock.fuzzView() + var ready []ghFuzzOp + for _, op := range ops { + if op.ready(v) { + ready = append(ready, op) + } + } + require.NotEmpty(t, ready) + ready[f.rng.Intn(len(ready))].apply(f, v) + } + + // ~1 round in 6 also evicts every ETag, so degradation to cold is + // fuzzed IN COMBINATION with churn, not only in isolation. + if f.rng.Intn(6) == 0 { + mock.evictEtags() + f.note("evict-etags") + } + + warm := h.runSync(fmt.Sprintf("fuzz-warm-%02d", round), prev) + control := h.runSync(fmt.Sprintf("fuzz-control-%02d", round), "") + + wSnap := h.snapshot(warm) + cSnap := h.snapshot(control) + if !ghAssertSnapshotsEqual(t, cSnap, wSnap) { + t.Fatalf("round %d diverged (seed=%d); mutations this round:\n %s", + round, seed, ghJoinLines(f.log)) + } + prev = warm + } +} + +func ghAssertSnapshotsEqual(t *testing.T, control, warm map[string]string) bool { + t.Helper() + ok := true + for k, cv := range control { + wv, found := warm[k] + if !found { + t.Errorf("warm sync MISSING %s", k) + ok = false + } else if wv != cv { + t.Errorf("warm sync DIFFERS at %s:\n control: %s\n warm: %s", k, cv, wv) + ok = false + } + } + for k := range warm { + if _, found := control[k]; !found { + t.Errorf("warm sync EXTRA %s: %s", k, warm[k]) + ok = false + } + } + return ok +} + +func ghJoinLines(lines []string) string { + out := "" + for i, l := range lines { + if i > 0 { + out += "\n " + } + out += l + } + return out +} diff --git a/pkg/connector/sourcecache_sync_test.go b/pkg/connector/sourcecache_sync_test.go new file mode 100644 index 00000000..ffac6e39 --- /dev/null +++ b/pkg/connector/sourcecache_sync_test.go @@ -0,0 +1,756 @@ +package connector + +// End-to-end source-cache replay harness for GitHub conditional requests, +// ported from the baton-microsoft-entra POC. +// +// Runs the real connector against a strict mock GitHub org with faithful +// conditional-request semantics (per-page body-hash ETags, ascending +// user-id ordering, Link headers on 200 only, NO Link on 304, 304 for a +// matching If-None-Match), through the real SDK sync loop on the Pebble +// engine, chaining each sync's c1z as the next sync's replay source. +// +// The paramount assertion is equivalence: after every warm sync a control +// sync (no previous c1z) runs against the same org state and the two +// files must be identical at the v2 reader surface. Scenarios also assert +// request-count ceilings (200s vs 304s) per round. +// +// The mock is strict: any unexpected request shape, any If-None-Match +// value that was never issued for that URL, or any malformed pagination +// parameter fails the test. + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "sort" + "strconv" + "sync" + "testing" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/connectorclient" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + sdkSync "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/types" + "github.com/conductorone/baton-sdk/pkg/types/sessions" + "github.com/google/go-github/v69/github" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +// --- mock GitHub org --------------------------------------------------------- + +const ( + mockOrgID = int64(999) + mockOrgLogin = "testorg" +) + +type mockMember struct { + ID int64 + Login string + Admin bool +} + +type mockGitHubOrg struct { + mu sync.Mutex + t *testing.T + + members map[int64]*mockMember + + // etagSalt perturbs every computed ETag; bumping it models upstream + // validator eviction (every conditional request answers 200). + etagSalt int + + // issued tracks every ETag handed out per request URI. A presented + // If-None-Match that was never issued for that URI is a connector + // bug (fabricated validator) and fails the test. + issued map[string]map[string]bool + + counts map[string]int +} + +func newMockGitHubOrg(t *testing.T) *mockGitHubOrg { + return &mockGitHubOrg{ + t: t, + members: map[int64]*mockMember{}, + issued: map[string]map[string]bool{}, + counts: map[string]int{}, + } +} + +func (m *mockGitHubOrg) addMember(id int64, admin bool) { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.members[id]; exists { + m.t.Fatalf("addMember: %d already exists", id) + } + m.members[id] = &mockMember{ID: id, Login: fmt.Sprintf("user-%d", id), Admin: admin} +} + +func (m *mockGitHubOrg) removeMember(id int64) { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.members[id]; !exists { + m.t.Fatalf("removeMember: %d does not exist", id) + } + delete(m.members, id) +} + +func (m *mockGitHubOrg) setAdmin(id int64, admin bool) { + m.mu.Lock() + defer m.mu.Unlock() + mem, exists := m.members[id] + if !exists { + m.t.Fatalf("setAdmin: %d does not exist", id) + } + mem.Admin = admin +} + +func (m *mockGitHubOrg) renameMember(id int64, login string) { + m.mu.Lock() + defer m.mu.Unlock() + mem, exists := m.members[id] + if !exists { + m.t.Fatalf("renameMember: %d does not exist", id) + } + mem.Login = login +} + +// evictEtags rotates the salt: every stored validator stops matching and +// all conditional requests answer 200, modeling upstream cache eviction. +func (m *mockGitHubOrg) evictEtags() { + m.mu.Lock() + defer m.mu.Unlock() + m.etagSalt++ +} + +func (m *mockGitHubOrg) snapshotCounts() map[string]int { + m.mu.Lock() + defer m.mu.Unlock() + out := map[string]int{} + for k, v := range m.counts { + out[k] = v + } + m.counts = map[string]int{} + return out +} + +// membersForRole returns the role-filtered listing in ascending user-id +// order — GitHub's member ordering, which is what makes appends land on +// the LAST page. role=member excludes admins, matching GitHub semantics. +func (m *mockGitHubOrg) membersForRole(role string) []*mockMember { + var out []*mockMember + for _, mem := range m.members { + if (role == "admin") == mem.Admin { + out = append(out, mem) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out +} + +func writeRateLimitHeaders(w http.ResponseWriter) { + w.Header().Set("X-RateLimit-Limit", "5000") + w.Header().Set("X-RateLimit-Remaining", "4999") + w.Header().Set("X-RateLimit-Reset", "4102444800") +} + +func mustWriteJSON(t *testing.T, w http.ResponseWriter, obj any) { + t.Helper() + data, err := json.Marshal(obj) + require.NoError(t, err) + _, _ = w.Write(data) +} + +func (m *mockGitHubOrg) orgJSON() map[string]any { + return map[string]any{ + "id": mockOrgID, + "login": mockOrgLogin, + "html_url": "https://github.example/" + mockOrgLogin, + } +} + +func memberJSON(mem *mockMember) map[string]any { + // Shape mirrors the real member listing: no name/email (GitHub does + // not include them there), so a login change is what perturbs bytes. + return map[string]any{ + "id": mem.ID, + "login": mem.Login, + "avatar_url": fmt.Sprintf("https://avatars.example/%d", mem.ID), + "html_url": "https://github.example/" + mem.Login, + "type": "User", + } +} + +func (m *mockGitHubOrg) handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + defer m.mu.Unlock() + + if r.Method != http.MethodGet { + m.t.Errorf("mock github: unexpected method %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + writeRateLimitHeaders(w) + + switch { + case r.URL.Path == "/user/orgs": + m.counts["user-orgs"]++ + mustWriteJSON(m.t, w, []map[string]any{m.orgJSON()}) + + case r.URL.Path == "/user/memberships/orgs/"+mockOrgLogin: + m.counts["org-membership"]++ + mustWriteJSON(m.t, w, map[string]any{ + "role": "admin", + "state": "active", + "organization": m.orgJSON(), + }) + + case r.URL.Path == "/organizations/"+strconv.FormatInt(mockOrgID, 10): + m.counts["org-by-id"]++ + mustWriteJSON(m.t, w, m.orgJSON()) + + case r.URL.Path == "/orgs/"+mockOrgLogin+"/members": + m.handleMembersLocked(w, r) + + default: + m.t.Errorf("mock github: unexpected request %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusNotFound) + mustWriteJSON(m.t, w, map[string]any{"message": "Not Found"}) + } + } +} + +func (m *mockGitHubOrg) handleMembersLocked(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + role := q.Get("role") + if role != "member" && role != "admin" { + m.t.Errorf("mock github: members listing missing/invalid role %q (%s)", role, r.URL.String()) + w.WriteHeader(http.StatusUnprocessableEntity) + return + } + if pp := q.Get("per_page"); pp != strconv.Itoa(maxPageSize) { + m.t.Errorf("mock github: members listing per_page %q, want %d (%s)", pp, maxPageSize, r.URL.String()) + w.WriteHeader(http.StatusUnprocessableEntity) + return + } + page, err := strconv.Atoi(q.Get("page")) + if err != nil || page < 1 { + m.t.Errorf("mock github: members listing bad page %q (%s)", q.Get("page"), r.URL.String()) + w.WriteHeader(http.StatusUnprocessableEntity) + return + } + + list := m.membersForRole(role) + start := (page - 1) * maxPageSize + end := start + maxPageSize + if start > len(list) { + start = len(list) + } + if end > len(list) { + end = len(list) + } + pageMembers := list[start:end] + + body := make([]map[string]any, 0, len(pageMembers)) + for _, mem := range pageMembers { + body = append(body, memberJSON(mem)) + } + bodyBytes, err := json.Marshal(body) + require.NoError(m.t, err) + + uri := r.URL.RequestURI() + sum := sha256.Sum256([]byte(fmt.Sprintf("salt=%d|%s|%s", m.etagSalt, uri, bodyBytes))) + etag := fmt.Sprintf(`W/"%x"`, sum[:12]) + + if inm := r.Header.Get("If-None-Match"); inm != "" { + if !m.issued[uri][inm] { + m.t.Errorf("mock github: If-None-Match %q for %s was never issued for that URL", inm, uri) + } + if inm == etag { + // Faithful pessimistic 304: no body, no Link header, no ETag + // beyond what the connector already holds. + m.counts["members-304"]++ + m.counts[fmt.Sprintf("members-304:%s:p%d", role, page)]++ + w.WriteHeader(http.StatusNotModified) + return + } + } + + if m.issued[uri] == nil { + m.issued[uri] = map[string]bool{} + } + m.issued[uri][etag] = true + + w.Header().Set("ETag", etag) + if end < len(list) { + next := cloneQueryWithPage(q, page+1) + w.Header().Set("Link", + fmt.Sprintf(`; rel="next"`, r.Host, r.URL.Path, next.Encode())) + } + m.counts["members-200"]++ + m.counts[fmt.Sprintf("members-200:%s:p%d", role, page)]++ + _, _ = w.Write(bodyBytes) +} + +func cloneQueryWithPage(q url.Values, page int) url.Values { + next := url.Values{} + for k, vs := range q { + for _, v := range vs { + next.Add(k, v) + } + } + next.Set("page", strconv.Itoa(page)) + return next +} + +// --- in-memory session store --------------------------------------------------- + +// memSessionStore is a minimal in-memory sessions.SessionStore for the +// harness (the production session store rides gRPC; the builder's default +// NoOp store errors on Get, which would fail orgCache lookups). Options +// (sync-id namespacing) are ignored; Clear wipes everything, matching the +// per-sync cleanup semantics closely enough for the harness. +type memSessionStore struct { + mu sync.Mutex + data map[string][]byte +} + +func newMemSessionStore() *memSessionStore { + return &memSessionStore{data: map[string][]byte{}} +} + +func (s *memSessionStore) Get(ctx context.Context, key string, opt ...sessions.SessionStoreOption) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.data[key] + return v, ok, nil +} + +func (s *memSessionStore) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, []string, error) { + s.mu.Lock() + defer s.mu.Unlock() + out := map[string][]byte{} + var missing []string + for _, k := range keys { + if v, ok := s.data[k]; ok { + out[k] = v + } else { + missing = append(missing, k) + } + } + return out, missing, nil +} + +func (s *memSessionStore) Set(ctx context.Context, key string, value []byte, opt ...sessions.SessionStoreOption) error { + s.mu.Lock() + defer s.mu.Unlock() + s.data[key] = value + return nil +} + +func (s *memSessionStore) SetMany(ctx context.Context, values map[string][]byte, opt ...sessions.SessionStoreOption) error { + s.mu.Lock() + defer s.mu.Unlock() + for k, v := range values { + s.data[k] = v + } + return nil +} + +func (s *memSessionStore) Delete(ctx context.Context, key string, opt ...sessions.SessionStoreOption) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.data, key) + return nil +} + +func (s *memSessionStore) Clear(ctx context.Context, opt ...sessions.SessionStoreOption) error { + s.mu.Lock() + defer s.mu.Unlock() + s.data = map[string][]byte{} + return nil +} + +func (s *memSessionStore) GetAll(ctx context.Context, pageToken string, opt ...sessions.SessionStoreOption) (map[string][]byte, string, error) { + s.mu.Lock() + defer s.mu.Unlock() + out := map[string][]byte{} + for k, v := range s.data { + out[k] = v + } + return out, "", nil +} + +// --- harness ----------------------------------------------------------------- + +type ghSyncHarness struct { + t *testing.T + ctx context.Context + mock *mockGitHubOrg + cc types.ConnectorClient + tmpDir string + syncN int + // workers > 0 runs syncs on a parallel worker pool; spawned sibling + // cursors (SpawnCursors fan-out) then revalidate concurrently. + workers int +} + +func newGHSyncHarness(ctx context.Context, t *testing.T, mock *mockGitHubOrg) *ghSyncHarness { + t.Helper() + + server := httptest.NewServer(mock.handler()) + t.Cleanup(server.Close) + + ghClient := github.NewClient(server.Client()) + baseURL, err := url.Parse(server.URL + "/") + require.NoError(t, err) + ghClient.BaseURL = baseURL + + gh := &GitHub{ + client: ghClient, + orgs: []string{mockOrgLogin}, + orgCache: newOrgNameCache(ghClient), + authScope: "test-auth", + } + + srv, err := connectorbuilder.NewConnector(ctx, gh, + connectorbuilder.WithSessionStore(newMemSessionStore())) + require.NoError(t, err) + + // Serve the connector over local gRPC and talk to it through the real + // connector client, mirroring how the CLI runs syncs. + gs := grpc.NewServer() + v2.RegisterConnectorServiceServer(gs, srv) + v2.RegisterGrantsServiceServer(gs, srv) + v2.RegisterEntitlementsServiceServer(gs, srv) + v2.RegisterResourcesServiceServer(gs, srv) + v2.RegisterResourceTypesServiceServer(gs, srv) + v2.RegisterAssetServiceServer(gs, srv) + v2.RegisterEventServiceServer(gs, srv) + v2.RegisterResourceGetterServiceServer(gs, srv) + v2.RegisterTicketsServiceServer(gs, srv) + v2.RegisterActionServiceServer(gs, srv) + v2.RegisterGrantManagerServiceServer(gs, srv) + v2.RegisterResourceManagerServiceServer(gs, srv) + v2.RegisterResourceDeleterServiceServer(gs, srv) + v2.RegisterAccountManagerServiceServer(gs, srv) + v2.RegisterCredentialManagerServiceServer(gs, srv) + + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = gs.Serve(lis) }() + t.Cleanup(gs.Stop) + + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + cc := connectorclient.NewConnectorClient(ctx, conn) + + // In-process lookup delivery: the syncer installs its per-sync lookup + // on the client, which forwards it to the builder (the CLI wrapper + // does this same wiring in internal/connector). + setter, ok := cc.(interface { + SetSourceCacheSetter(sourcecache.SetLookup) + }) + require.True(t, ok, "connector client must accept a source-cache setter") + lookupSink, ok := srv.(sourcecache.SetLookup) + require.True(t, ok, "connectorbuilder server must implement sourcecache.SetLookup") + setter.SetSourceCacheSetter(lookupSink) + + return &ghSyncHarness{t: t, ctx: ctx, mock: mock, cc: cc, tmpDir: t.TempDir()} +} + +// runSync executes one org-targeted full sync into a fresh Pebble c1z, +// optionally replaying from prevPath. Returns the new file's path. +func (h *ghSyncHarness) runSync(name string, prevPath string) string { + h.t.Helper() + h.syncN++ + path := filepath.Join(h.tmpDir, fmt.Sprintf("%02d-%s.c1z", h.syncN, name)) + + store, err := dotc1z.NewStore(h.ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(h.tmpDir), + ) + require.NoError(h.t, err) + + opts := []sdkSync.SyncOpt{ + sdkSync.WithConnectorStore(store), + sdkSync.WithTmpDir(h.tmpDir), + sdkSync.WithSyncResourceTypes([]string{"org"}), + } + if h.workers > 0 { + opts = append(opts, sdkSync.WithWorkerCount(h.workers)) + } + if prevPath != "" { + opts = append(opts, sdkSync.WithPreviousSyncC1ZPath(prevPath)) + } + + syncer, err := sdkSync.NewSyncer(h.ctx, h.cc, opts...) + require.NoError(h.t, err) + require.NoError(h.t, syncer.Sync(h.ctx)) + require.NoError(h.t, syncer.Close(h.ctx)) + return path +} + +// snapshot reads a finished c1z at the v2 reader surface and returns +// id → canonical JSON for resources, entitlements, and grants. +func (h *ghSyncHarness) snapshot(path string) map[string]string { + h.t.Helper() + store, err := dotc1z.NewStore(h.ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + dotc1z.WithTmpDir(h.tmpDir), + ) + require.NoError(h.t, err) + defer func() { _ = store.Close(h.ctx) }() + + latest, err := store.SyncMeta().LatestFullSync(h.ctx) + require.NoError(h.t, err) + require.NotNil(h.t, latest) + require.NoError(h.t, store.SetCurrentSync(h.ctx, latest.ID)) + + out := map[string]string{} + put := func(prefix, id string, msg proto.Message) { + jb, err := protojson.Marshal(msg) + require.NoError(h.t, err) + // protojson output spacing is deliberately unstable; re-marshal + // through encoding/json for canonical (sorted-key) bytes. + var v any + require.NoError(h.t, json.Unmarshal(jb, &v)) + cb, err := json.Marshal(v) + require.NoError(h.t, err) + key := prefix + ":" + id + require.NotContains(h.t, out, key, "duplicate id at reader surface") + out[key] = string(cb) + } + + pageToken := "" + for { + resp, err := store.ListResources(h.ctx, v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: "org", + PageToken: pageToken, + }.Build()) + require.NoError(h.t, err) + for _, r := range resp.GetList() { + put("resource", r.GetId().GetResourceType()+"/"+r.GetId().GetResource(), r) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + pageToken = "" + for { + resp, err := store.ListEntitlements(h.ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + PageToken: pageToken, + }.Build()) + require.NoError(h.t, err) + for _, e := range resp.GetList() { + put("entitlement", e.GetId(), e) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + pageToken = "" + for { + resp, err := store.ListGrants(h.ctx, v2.GrantsServiceListGrantsRequest_builder{ + PageToken: pageToken, + }.Build()) + require.NoError(h.t, err) + for _, g := range resp.GetList() { + put("grant", g.GetId(), g) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + return out +} + +// requireEquivalent is the release-blocker check: a warm (replayed) sync +// must be byte-identical to an uncached control sync at the reader surface. +func (h *ghSyncHarness) requireEquivalent(warmPath, controlPath string, scenario string) { + h.t.Helper() + warm := h.snapshot(warmPath) + control := h.snapshot(controlPath) + require.Equal(h.t, control, warm, + "%s: warm sync diverged from uncached control sync — replay equivalence violated", scenario) +} + +// grantCount tallies grant rows in a snapshot. +func grantCount(snap map[string]string) int { + n := 0 + for k := range snap { + if len(k) > 6 && k[:6] == "grant:" { + n++ + } + } + return n +} + +// --- the scenarios ----------------------------------------------------------- + +func TestGitHubSourceCacheReplayEndToEnd(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runGitHubReplayScenarios(t, workers) + }) + } +} + +func runGitHubReplayScenarios(t *testing.T, workers int) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + mock := newMockGitHubOrg(t) + + // Org: 240 regular members (3 member pages: 100/100/40) and 10 admins + // (1 admin page). Admin grants double (admins get member+admin rows), + // so the expected grant count is 240 + 2*10 = 260. + for i := int64(1); i <= 240; i++ { + mock.addMember(i, false) + } + for i := int64(1000); i < 1010; i++ { + mock.addMember(i, true) + } + + h := newGHSyncHarness(ctx, t, mock) + h.workers = workers + + // --- Sync 1: cold --------------------------------------------------------- + sync1 := h.runSync("cold", "") + c1 := mock.snapshotCounts() + require.Equal(t, 4, c1["members-200"], "cold sync fetches 3 member pages + 1 admin page") + require.Zero(t, c1["members-304"]) + snap1 := h.snapshot(sync1) + require.Equal(t, 260, grantCount(snap1)) + + // --- Sync 2: warm, no upstream change -------------------------------------- + // Every page revalidates with a 304; zero member-listing rate-limit + // units consumed; rows replayed from sync 1's file. + sync2 := h.runSync("warm-noop", sync1) + c2 := mock.snapshotCounts() + require.Zero(t, c2["members-200"], "no-op warm sync must not re-fetch any member page") + require.Equal(t, 4, c2["members-304"], "every page revalidates via 304") + control2 := h.runSync("control-noop", "") + h.requireEquivalent(sync2, control2, "no-op replay") + + // --- Sync 3: append at the end (GitHub id ordering) ------------------------- + // A new member with a high id lands on the LAST member page; the + // prefix pages still 304. This is the case that breaks a + // page-1-validates-the-collection design. + mock.addMember(500, false) + _ = mock.snapshotCounts() // discard the control run's counts + sync3 := h.runSync("warm-append", sync2) + c3 := mock.snapshotCounts() + require.Equal(t, 1, c3["members-200"], "only the tail page re-fetches") + require.Equal(t, 1, c3["members-200:member:p3"]) + require.Equal(t, 3, c3["members-304"], "prefix member pages + admin page still replay") + control3 := h.runSync("control-append", "") + h.requireEquivalent(sync3, control3, "append at end") + require.Equal(t, 261, grantCount(h.snapshot(sync3))) + + // --- Sync 4: append past a FULL tail page (the probe case) ------------------ + // Grow the member list to exactly 300 (3 full pages), sync, then add + // one more member. Pages 1-3 are byte-identical (304), but page 3 was + // FULL, so the connector must probe page 4 and find the new member. + // A design that trusted "no Link header on 304 => chain ends" would + // silently drop this member. + for i := int64(501); i <= 559; i++ { + mock.addMember(i, false) // 241 existing members + 59 = 300 + } + sync4a := h.runSync("warm-fill", sync3) + control4a := h.runSync("control-fill", "") + h.requireEquivalent(sync4a, control4a, "fill to page boundary") + _ = mock.snapshotCounts() + + mock.addMember(600, false) // members: 301 → page 4 exists with 1 row + sync4b := h.runSync("warm-probe", sync4a) + c4 := mock.snapshotCounts() + require.Equal(t, 4, c4["members-304"], "member pages 1-3 and the admin page all replay") + require.Equal(t, 1, c4["members-200"]) + require.Equal(t, 1, c4["members-200:member:p4"], "full tail page forces a probe of page 4") + control4b := h.runSync("control-probe", "") + h.requireEquivalent(sync4b, control4b, "append past full tail") + require.Equal(t, 321, grantCount(h.snapshot(sync4b)), "301 member grants + 2*10 admin grants") + + // --- Sync 5: removal shifts pages left -------------------------------------- + // Removing the FIRST member shifts every member page's bytes left; the + // member count drops to exactly 300. With spawn fan-out the stored + // page-4 scope is still revalidated by its sibling cursor — it answers + // 200 with an empty body (its row moved to page 3, fetched fresh) and + // persists an empty-page validator. + mock.removeMember(1) + _ = mock.snapshotCounts() + sync5 := h.runSync("warm-remove", sync4b) + c5 := mock.snapshotCounts() + require.Equal(t, 4, c5["members-200"], "three shifted member pages + the now-empty stored page 4 re-fetch") + require.Equal(t, 1, c5["members-304"], "admin page still replays") + control5 := h.runSync("control-remove", "") + h.requireEquivalent(sync5, control5, "removal") + + // --- Sync 6: promotion member -> admin --------------------------------------- + mock.setAdmin(2, true) + sync6 := h.runSync("warm-promote", sync5) + control6 := h.runSync("control-promote", "") + h.requireEquivalent(sync6, control6, "promotion") + + // --- Sync 7: upstream evicts every validator --------------------------------- + // Every conditional request answers 200: the warm sync degrades to + // exactly a cold sync (fail toward cold), and the NEXT round is warm + // again off the new validators. + // State: 299 regular members (3 pages: 100/100/99) + 11 admins (1 page), + // plus the sticky empty page-4 scope from sync 5 (an empty stored page + // keeps revalidating — one free 304 per sync — until a page boundary + // shift changes its bytes). + mock.evictEtags() + _ = mock.snapshotCounts() + sync7 := h.runSync("warm-evicted", sync6) + c7 := mock.snapshotCounts() + require.Zero(t, c7["members-304"], "evicted validators can never 304") + require.Equal(t, 5, c7["members-200"], "eviction degrades to a full cold fetch (incl. the stored empty page)") + control7 := h.runSync("control-evicted", "") + h.requireEquivalent(sync7, control7, "total validator eviction") + + // --- Sync 8: recovery after eviction ----------------------------------------- + _ = mock.snapshotCounts() + sync8 := h.runSync("warm-recovered", sync7) + c8 := mock.snapshotCounts() + require.Zero(t, c8["members-200"], "post-eviction round is fully warm again") + require.Equal(t, 5, c8["members-304"]) + control8 := h.runSync("control-recovered", "") + h.requireEquivalent(sync8, control8, "recovery after eviction") + + // --- Sync 9: warm-from-warm chain sanity -------------------------------------- + // sync8 was itself almost entirely replayed; it must still be a valid + // replay source (manifest entries and scope stamps re-persisted). + _ = mock.snapshotCounts() + sync9 := h.runSync("warm-chain", sync8) + c9 := mock.snapshotCounts() + require.Zero(t, c9["members-200"]) + require.Equal(t, 5, c9["members-304"]) + control9 := h.runSync("control-chain", "") + h.requireEquivalent(sync9, control9, "replay-only sync as replay source") +} diff --git a/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go b/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go index 04170efd..066fcff7 100644 --- a/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go +++ b/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go @@ -23,11 +23,13 @@ import ( connectorV2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" connectorwrapperV1 "github.com/conductorone/baton-sdk/pb/c1/connector_wrapper/v1" + batonV1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" ratelimitV1 "github.com/conductorone/baton-sdk/pb/c1/ratelimit/v1" tlsV1 "github.com/conductorone/baton-sdk/pb/c1/utls/v1" "github.com/conductorone/baton-sdk/pkg/bid" ratelimit2 "github.com/conductorone/baton-sdk/pkg/ratelimit" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/ugrpc" @@ -55,19 +57,42 @@ type connectorClient struct { connectorV2.ActionServiceClient sessionStoreSetter sessions.SetSessionStore // this is the session store server + sourceCacheSetter sourcecache.SetLookup // this is the source-cache lookup server } var _ sessions.SetSessionStore = (*connectorClient)(nil) +var _ sourcecache.SetLookup = (*connectorClient)(nil) var _ SetSessionStoreSetter = (*connectorClient)(nil) +var _ SetSourceCacheSetter = (*connectorClient)(nil) type SetSessionStoreSetter interface { SetSessionStoreSetter(setsessionStoreSetter sessions.SetSessionStore) } +type SetSourceCacheSetter interface { + SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) +} + func (c *connectorClient) SetSessionStoreSetter(sessionStoreSetter sessions.SetSessionStore) { c.sessionStoreSetter = sessionStoreSetter } +func (c *connectorClient) SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) { + c.sourceCacheSetter = sourceCacheSetter +} + +// SetSourceCache forwards the syncer's per-sync lookup to whatever +// receives it: the subprocess-mode BatonSourceCacheService server, or the +// in-process builder. A nil setter means the connector never opted in; +// the syncer calls this unconditionally, so stay quiet at debug level. +func (c *connectorClient) SetSourceCache(ctx context.Context, lookup sourcecache.Lookup) { + if c.sourceCacheSetter == nil { + ctxzap.Extract(ctx).Debug("connectorClient's source cache setter is nil — connector did not opt into source caching") + return + } + c.sourceCacheSetter.SetSourceCache(ctx, lookup) +} + func (c *connectorClient) SetSessionStore(ctx context.Context, store sessions.SessionStore) { if c.sessionStoreSetter == nil { // Demoted from Warn to Debug: this path is the normal case for @@ -109,7 +134,8 @@ type wrapper struct { now func() time.Time - SessionServer sessions.SetSessionStore + SessionServer sessions.SetSessionStore + SourceCacheServer sourcecache.SetLookup } type Option func(ctx context.Context, w *wrapper) error @@ -197,6 +223,12 @@ func NewWrapper(ctx context.Context, server interface{}, opts ...Option) (*wrapp server: connectorServer, now: time.Now, } + // In-process delivery: a connectorbuilder-based server implements + // sourcecache.SetLookup itself, so the syncer's per-sync lookup can be + // installed without the subprocess gRPC hop. + if sourceCacheServer, ok := connectorServer.(sourcecache.SetLookup); ok { + w.SourceCacheServer = sourceCacheServer + } for _, o := range opts { err := o(ctx, w) @@ -276,18 +308,29 @@ func (cw *wrapper) runServer(ctx context.Context, serverCred *tlsV1.Credential) return 0, fmt.Errorf("failed to create session listener config: %w", err) } - // TODO(kans): block until we send a request or something/error handling in general. + // One listener serves BatonSessionService (connector session data) + // and BatonSourceCacheService (source-cache scope lookups). Keeping + // them as separate RPCs keeps validator lookups out of the + // connector's local MemorySessionCache and its TTL/eviction rules. l.Info("starting session store server") - server := session.NewGRPCSessionServer() - cw.SessionServer = server + sessionServer := session.NewGRPCSessionServer() + sourceCacheServer := sourcecache.NewGRPCServer() + cw.SessionServer = sessionServer + cw.SourceCacheServer = sourceCacheServer go func() { defer sessionListener.Close() - serverErr := session.StartGRPCSessionServerWithOptions(ctx, sessionListener, server, + grpcServer := grpc.NewServer( grpc.Creds(credentials.NewTLS(tlsConfig)), grpc.ChainUnaryInterceptor(ugrpc.UnaryServerInterceptor(ctx)...), ) - if serverErr != nil { - l.Error("failed to create session store server", zap.Error(serverErr)) + batonV1.RegisterBatonSessionServiceServer(grpcServer, sessionServer) + batonV1.RegisterBatonSourceCacheServiceServer(grpcServer, sourceCacheServer) + go func() { + <-ctx.Done() + grpcServer.GracefulStop() + }() + if serveErr := grpcServer.Serve(sessionListener); serveErr != nil { + l.Error("session/source-cache server stopped", zap.Error(serveErr)) return } }() @@ -440,6 +483,7 @@ func (cw *wrapper) C(ctx context.Context) (types.ConnectorClient, error) { cw.conn = conn client := NewConnectorClient(ctx, cw.conn) client.SetSessionStoreSetter(cw.SessionServer) + client.SetSourceCacheSetter(cw.SourceCacheServer) cw.client = client return client, nil diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.go index 47662d17..487a1b67 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.go @@ -37,6 +37,50 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type PebbleIdIndexFormat int32 + +const ( + PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED PebbleIdIndexFormat = 0 + PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_LEGACY_EXTERNAL_ID PebbleIdIndexFormat = 1 + PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_STRUCTURED_V1 PebbleIdIndexFormat = 2 +) + +// Enum value maps for PebbleIdIndexFormat. +var ( + PebbleIdIndexFormat_name = map[int32]string{ + 0: "PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED", + 1: "PEBBLE_ID_INDEX_FORMAT_LEGACY_EXTERNAL_ID", + 2: "PEBBLE_ID_INDEX_FORMAT_STRUCTURED_V1", + } + PebbleIdIndexFormat_value = map[string]int32{ + "PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED": 0, + "PEBBLE_ID_INDEX_FORMAT_LEGACY_EXTERNAL_ID": 1, + "PEBBLE_ID_INDEX_FORMAT_STRUCTURED_V1": 2, + } +) + +func (x PebbleIdIndexFormat) Enum() *PebbleIdIndexFormat { + p := new(PebbleIdIndexFormat) + *p = x + return p +} + +func (x PebbleIdIndexFormat) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PebbleIdIndexFormat) Descriptor() protoreflect.EnumDescriptor { + return file_c1_c1z_v3_manifest_proto_enumTypes[0].Descriptor() +} + +func (PebbleIdIndexFormat) Type() protoreflect.EnumType { + return &file_c1_c1z_v3_manifest_proto_enumTypes[0] +} + +func (x PebbleIdIndexFormat) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + type PayloadEncoding int32 const ( @@ -92,11 +136,11 @@ func (x PayloadEncoding) String() string { } func (PayloadEncoding) Descriptor() protoreflect.EnumDescriptor { - return file_c1_c1z_v3_manifest_proto_enumTypes[0].Descriptor() + return file_c1_c1z_v3_manifest_proto_enumTypes[1].Descriptor() } func (PayloadEncoding) Type() protoreflect.EnumType { - return &file_c1_c1z_v3_manifest_proto_enumTypes[0] + return &file_c1_c1z_v3_manifest_proto_enumTypes[1] } func (x PayloadEncoding) Number() protoreflect.EnumNumber { @@ -146,8 +190,13 @@ type C1ZManifestV3 struct { // zero. The compactor's auto mode reads this from the envelope // header to force a rebuild once waste crosses its threshold. FoldDeadBytes int64 `protobuf:"varint,41,opt,name=fold_dead_bytes,json=foldDeadBytes,proto3" json:"fold_dead_bytes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Advisory mirror of the Pebble in-payload grant/entitlement ID/index format. + // The Pebble engine-meta stamp remains authoritative; this header field lets + // tooling inspect legacy-vs-structured index state without extracting the + // Pebble payload. + PebbleIdIndexFormat PebbleIdIndexFormat `protobuf:"varint,42,opt,name=pebble_id_index_format,json=pebbleIdIndexFormat,proto3,enum=c1.c1z.v3.PebbleIdIndexFormat" json:"pebble_id_index_format,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *C1ZManifestV3) Reset() { @@ -231,6 +280,13 @@ func (x *C1ZManifestV3) GetFoldDeadBytes() int64 { return 0 } +func (x *C1ZManifestV3) GetPebbleIdIndexFormat() PebbleIdIndexFormat { + if x != nil { + return x.PebbleIdIndexFormat + } + return PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED +} + func (x *C1ZManifestV3) SetEngine(v string) { x.Engine = v } @@ -263,6 +319,10 @@ func (x *C1ZManifestV3) SetFoldDeadBytes(v int64) { x.FoldDeadBytes = v } +func (x *C1ZManifestV3) SetPebbleIdIndexFormat(v PebbleIdIndexFormat) { + x.PebbleIdIndexFormat = v +} + func (x *C1ZManifestV3) HasEngineConfig() bool { if x == nil { return false @@ -329,6 +389,11 @@ type C1ZManifestV3_builder struct { // zero. The compactor's auto mode reads this from the envelope // header to force a rebuild once waste crosses its threshold. FoldDeadBytes int64 + // Advisory mirror of the Pebble in-payload grant/entitlement ID/index format. + // The Pebble engine-meta stamp remains authoritative; this header field lets + // tooling inspect legacy-vs-structured index state without extracting the + // Pebble payload. + PebbleIdIndexFormat PebbleIdIndexFormat } func (b0 C1ZManifestV3_builder) Build() *C1ZManifestV3 { @@ -343,6 +408,7 @@ func (b0 C1ZManifestV3_builder) Build() *C1ZManifestV3 { x.RecordTypes = b.RecordTypes x.SyncRuns = b.SyncRuns x.FoldDeadBytes = b.FoldDeadBytes + x.PebbleIdIndexFormat = b.PebbleIdIndexFormat return m0 } @@ -1003,7 +1069,7 @@ var File_c1_c1z_v3_manifest_proto protoreflect.FileDescriptor const file_c1_c1z_v3_manifest_proto_rawDesc = "" + "\n" + - "\x18c1/c1z/v3/manifest.proto\x12\tc1.c1z.v3\x1a\x1bc1/storage/v3/records.proto\x1a\x19google/protobuf/any.proto\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xc1\x03\n" + + "\x18c1/c1z/v3/manifest.proto\x12\tc1.c1z.v3\x1a\x1bc1/storage/v3/records.proto\x1a\x19google/protobuf/any.proto\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x96\x04\n" + "\rC1ZManifestV3\x12\x16\n" + "\x06engine\x18\x01 \x01(\tR\x06engine\x122\n" + "\x15engine_schema_version\x18\x02 \x01(\rR\x13engineSchemaVersion\x129\n" + @@ -1013,7 +1079,8 @@ const file_c1_c1z_v3_manifest_proto_rawDesc = "" + " \x01(\v2\".google.protobuf.FileDescriptorSetR\vdescriptors\x12<\n" + "\frecord_types\x18\v \x03(\v2\x19.c1.c1z.v3.RecordTypeInfoR\vrecordTypes\x126\n" + "\tsync_runs\x18( \x03(\v2\x19.c1.c1z.v3.SyncRunSummaryR\bsyncRuns\x12&\n" + - "\x0ffold_dead_bytes\x18) \x01(\x03R\rfoldDeadBytes\"\xcc\x01\n" + + "\x0ffold_dead_bytes\x18) \x01(\x03R\rfoldDeadBytes\x12S\n" + + "\x16pebble_id_index_format\x18* \x01(\x0e2\x1e.c1.c1z.v3.PebbleIdIndexFormatR\x13pebbleIdIndexFormat\"\xcc\x01\n" + "\x11IndexedFrameIndex\x126\n" + "\aentries\x18\x01 \x03(\v2\x1c.c1.c1z.v3.IndexedFrameEntryR\aentries\x12$\n" + "\x0etotal_raw_size\x18\x02 \x01(\x03R\ftotalRawSize\x122\n" + @@ -1041,45 +1108,51 @@ const file_c1_c1z_v3_manifest_proto_rawDesc = "" + "\x05stats\x18\x06 \x01(\v2\x1e.c1.storage.v3.SyncStatsRecordR\x05stats\"p\n" + "\x12PebbleEngineConfig\x120\n" + "\x14format_major_version\x18\x01 \x01(\rR\x12formatMajorVersion\x12(\n" + - "\x10cache_size_bytes\x18\x02 \x01(\x04R\x0ecacheSizeBytes*\x9b\x01\n" + + "\x10cache_size_bytes\x18\x02 \x01(\x04R\x0ecacheSizeBytes*\x96\x01\n" + + "\x13PebbleIdIndexFormat\x12&\n" + + "\"PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED\x10\x00\x12-\n" + + ")PEBBLE_ID_INDEX_FORMAT_LEGACY_EXTERNAL_ID\x10\x01\x12(\n" + + "$PEBBLE_ID_INDEX_FORMAT_STRUCTURED_V1\x10\x02*\x9b\x01\n" + "\x0fPayloadEncoding\x12 \n" + "\x1cPAYLOAD_ENCODING_UNSPECIFIED\x10\x00\x12\x1d\n" + "\x19PAYLOAD_ENCODING_TAR_ZSTD\x10\x01\x12\x18\n" + "\x14PAYLOAD_ENCODING_TAR\x10\x02\x12!\n" + "\x1dPAYLOAD_ENCODING_INDEXED_ZSTD\x10\x05\"\x04\b\x03\x10\x03\"\x04\b\x04\x10\x04B0Z.github.com/conductorone/baton-sdk/pb/c1/c1z/v3b\x06proto3" -var file_c1_c1z_v3_manifest_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_c1z_v3_manifest_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_c1_c1z_v3_manifest_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_c1_c1z_v3_manifest_proto_goTypes = []any{ - (PayloadEncoding)(0), // 0: c1.c1z.v3.PayloadEncoding - (*C1ZManifestV3)(nil), // 1: c1.c1z.v3.C1ZManifestV3 - (*IndexedFrameIndex)(nil), // 2: c1.c1z.v3.IndexedFrameIndex - (*IndexedFrameEntry)(nil), // 3: c1.c1z.v3.IndexedFrameEntry - (*RecordTypeInfo)(nil), // 4: c1.c1z.v3.RecordTypeInfo - (*SyncRunSummary)(nil), // 5: c1.c1z.v3.SyncRunSummary - (*PebbleEngineConfig)(nil), // 6: c1.c1z.v3.PebbleEngineConfig - (*anypb.Any)(nil), // 7: google.protobuf.Any - (*descriptorpb.FileDescriptorSet)(nil), // 8: google.protobuf.FileDescriptorSet - (v3.SyncType)(0), // 9: c1.storage.v3.SyncType - (*timestamppb.Timestamp)(nil), // 10: google.protobuf.Timestamp - (*v3.SyncStatsRecord)(nil), // 11: c1.storage.v3.SyncStatsRecord + (PebbleIdIndexFormat)(0), // 0: c1.c1z.v3.PebbleIdIndexFormat + (PayloadEncoding)(0), // 1: c1.c1z.v3.PayloadEncoding + (*C1ZManifestV3)(nil), // 2: c1.c1z.v3.C1ZManifestV3 + (*IndexedFrameIndex)(nil), // 3: c1.c1z.v3.IndexedFrameIndex + (*IndexedFrameEntry)(nil), // 4: c1.c1z.v3.IndexedFrameEntry + (*RecordTypeInfo)(nil), // 5: c1.c1z.v3.RecordTypeInfo + (*SyncRunSummary)(nil), // 6: c1.c1z.v3.SyncRunSummary + (*PebbleEngineConfig)(nil), // 7: c1.c1z.v3.PebbleEngineConfig + (*anypb.Any)(nil), // 8: google.protobuf.Any + (*descriptorpb.FileDescriptorSet)(nil), // 9: google.protobuf.FileDescriptorSet + (v3.SyncType)(0), // 10: c1.storage.v3.SyncType + (*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp + (*v3.SyncStatsRecord)(nil), // 12: c1.storage.v3.SyncStatsRecord } var file_c1_c1z_v3_manifest_proto_depIdxs = []int32{ - 7, // 0: c1.c1z.v3.C1ZManifestV3.engine_config:type_name -> google.protobuf.Any - 0, // 1: c1.c1z.v3.C1ZManifestV3.payload_encoding:type_name -> c1.c1z.v3.PayloadEncoding - 8, // 2: c1.c1z.v3.C1ZManifestV3.descriptors:type_name -> google.protobuf.FileDescriptorSet - 4, // 3: c1.c1z.v3.C1ZManifestV3.record_types:type_name -> c1.c1z.v3.RecordTypeInfo - 5, // 4: c1.c1z.v3.C1ZManifestV3.sync_runs:type_name -> c1.c1z.v3.SyncRunSummary - 3, // 5: c1.c1z.v3.IndexedFrameIndex.entries:type_name -> c1.c1z.v3.IndexedFrameEntry - 9, // 6: c1.c1z.v3.SyncRunSummary.type:type_name -> c1.storage.v3.SyncType - 10, // 7: c1.c1z.v3.SyncRunSummary.started_at:type_name -> google.protobuf.Timestamp - 10, // 8: c1.c1z.v3.SyncRunSummary.ended_at:type_name -> google.protobuf.Timestamp - 11, // 9: c1.c1z.v3.SyncRunSummary.stats:type_name -> c1.storage.v3.SyncStatsRecord - 10, // [10:10] is the sub-list for method output_type - 10, // [10:10] is the sub-list for method input_type - 10, // [10:10] is the sub-list for extension type_name - 10, // [10:10] is the sub-list for extension extendee - 0, // [0:10] is the sub-list for field type_name + 8, // 0: c1.c1z.v3.C1ZManifestV3.engine_config:type_name -> google.protobuf.Any + 1, // 1: c1.c1z.v3.C1ZManifestV3.payload_encoding:type_name -> c1.c1z.v3.PayloadEncoding + 9, // 2: c1.c1z.v3.C1ZManifestV3.descriptors:type_name -> google.protobuf.FileDescriptorSet + 5, // 3: c1.c1z.v3.C1ZManifestV3.record_types:type_name -> c1.c1z.v3.RecordTypeInfo + 6, // 4: c1.c1z.v3.C1ZManifestV3.sync_runs:type_name -> c1.c1z.v3.SyncRunSummary + 0, // 5: c1.c1z.v3.C1ZManifestV3.pebble_id_index_format:type_name -> c1.c1z.v3.PebbleIdIndexFormat + 4, // 6: c1.c1z.v3.IndexedFrameIndex.entries:type_name -> c1.c1z.v3.IndexedFrameEntry + 10, // 7: c1.c1z.v3.SyncRunSummary.type:type_name -> c1.storage.v3.SyncType + 11, // 8: c1.c1z.v3.SyncRunSummary.started_at:type_name -> google.protobuf.Timestamp + 11, // 9: c1.c1z.v3.SyncRunSummary.ended_at:type_name -> google.protobuf.Timestamp + 12, // 10: c1.c1z.v3.SyncRunSummary.stats:type_name -> c1.storage.v3.SyncStatsRecord + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_c1_c1z_v3_manifest_proto_init() } @@ -1092,7 +1165,7 @@ func file_c1_c1z_v3_manifest_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_c1z_v3_manifest_proto_rawDesc), len(file_c1_c1z_v3_manifest_proto_rawDesc)), - NumEnums: 1, + NumEnums: 2, NumMessages: 6, NumExtensions: 0, NumServices: 0, diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.validate.go index 9fa7a041..a69d20d1 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.validate.go @@ -195,6 +195,8 @@ func (m *C1ZManifestV3) validate(all bool) error { // no validation rules for FoldDeadBytes + // no validation rules for PebbleIdIndexFormat + if len(errors) > 0 { return C1ZManifestV3MultiError(errors) } diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest_protoopaque.pb.go index ca37d647..a5ab7cbf 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest_protoopaque.pb.go @@ -37,6 +37,50 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type PebbleIdIndexFormat int32 + +const ( + PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED PebbleIdIndexFormat = 0 + PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_LEGACY_EXTERNAL_ID PebbleIdIndexFormat = 1 + PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_STRUCTURED_V1 PebbleIdIndexFormat = 2 +) + +// Enum value maps for PebbleIdIndexFormat. +var ( + PebbleIdIndexFormat_name = map[int32]string{ + 0: "PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED", + 1: "PEBBLE_ID_INDEX_FORMAT_LEGACY_EXTERNAL_ID", + 2: "PEBBLE_ID_INDEX_FORMAT_STRUCTURED_V1", + } + PebbleIdIndexFormat_value = map[string]int32{ + "PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED": 0, + "PEBBLE_ID_INDEX_FORMAT_LEGACY_EXTERNAL_ID": 1, + "PEBBLE_ID_INDEX_FORMAT_STRUCTURED_V1": 2, + } +) + +func (x PebbleIdIndexFormat) Enum() *PebbleIdIndexFormat { + p := new(PebbleIdIndexFormat) + *p = x + return p +} + +func (x PebbleIdIndexFormat) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PebbleIdIndexFormat) Descriptor() protoreflect.EnumDescriptor { + return file_c1_c1z_v3_manifest_proto_enumTypes[0].Descriptor() +} + +func (PebbleIdIndexFormat) Type() protoreflect.EnumType { + return &file_c1_c1z_v3_manifest_proto_enumTypes[0] +} + +func (x PebbleIdIndexFormat) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + type PayloadEncoding int32 const ( @@ -92,11 +136,11 @@ func (x PayloadEncoding) String() string { } func (PayloadEncoding) Descriptor() protoreflect.EnumDescriptor { - return file_c1_c1z_v3_manifest_proto_enumTypes[0].Descriptor() + return file_c1_c1z_v3_manifest_proto_enumTypes[1].Descriptor() } func (PayloadEncoding) Type() protoreflect.EnumType { - return &file_c1_c1z_v3_manifest_proto_enumTypes[0] + return &file_c1_c1z_v3_manifest_proto_enumTypes[1] } func (x PayloadEncoding) Number() protoreflect.EnumNumber { @@ -113,6 +157,7 @@ type C1ZManifestV3 struct { xxx_hidden_RecordTypes *[]*RecordTypeInfo `protobuf:"bytes,11,rep,name=record_types,json=recordTypes,proto3"` xxx_hidden_SyncRuns *[]*SyncRunSummary `protobuf:"bytes,40,rep,name=sync_runs,json=syncRuns,proto3"` xxx_hidden_FoldDeadBytes int64 `protobuf:"varint,41,opt,name=fold_dead_bytes,json=foldDeadBytes,proto3"` + xxx_hidden_PebbleIdIndexFormat PebbleIdIndexFormat `protobuf:"varint,42,opt,name=pebble_id_index_format,json=pebbleIdIndexFormat,proto3,enum=c1.c1z.v3.PebbleIdIndexFormat"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -202,6 +247,13 @@ func (x *C1ZManifestV3) GetFoldDeadBytes() int64 { return 0 } +func (x *C1ZManifestV3) GetPebbleIdIndexFormat() PebbleIdIndexFormat { + if x != nil { + return x.xxx_hidden_PebbleIdIndexFormat + } + return PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED +} + func (x *C1ZManifestV3) SetEngine(v string) { x.xxx_hidden_Engine = v } @@ -234,6 +286,10 @@ func (x *C1ZManifestV3) SetFoldDeadBytes(v int64) { x.xxx_hidden_FoldDeadBytes = v } +func (x *C1ZManifestV3) SetPebbleIdIndexFormat(v PebbleIdIndexFormat) { + x.xxx_hidden_PebbleIdIndexFormat = v +} + func (x *C1ZManifestV3) HasEngineConfig() bool { if x == nil { return false @@ -300,6 +356,11 @@ type C1ZManifestV3_builder struct { // zero. The compactor's auto mode reads this from the envelope // header to force a rebuild once waste crosses its threshold. FoldDeadBytes int64 + // Advisory mirror of the Pebble in-payload grant/entitlement ID/index format. + // The Pebble engine-meta stamp remains authoritative; this header field lets + // tooling inspect legacy-vs-structured index state without extracting the + // Pebble payload. + PebbleIdIndexFormat PebbleIdIndexFormat } func (b0 C1ZManifestV3_builder) Build() *C1ZManifestV3 { @@ -314,6 +375,7 @@ func (b0 C1ZManifestV3_builder) Build() *C1ZManifestV3 { x.xxx_hidden_RecordTypes = &b.RecordTypes x.xxx_hidden_SyncRuns = &b.SyncRuns x.xxx_hidden_FoldDeadBytes = b.FoldDeadBytes + x.xxx_hidden_PebbleIdIndexFormat = b.PebbleIdIndexFormat return m0 } @@ -934,7 +996,7 @@ var File_c1_c1z_v3_manifest_proto protoreflect.FileDescriptor const file_c1_c1z_v3_manifest_proto_rawDesc = "" + "\n" + - "\x18c1/c1z/v3/manifest.proto\x12\tc1.c1z.v3\x1a\x1bc1/storage/v3/records.proto\x1a\x19google/protobuf/any.proto\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xc1\x03\n" + + "\x18c1/c1z/v3/manifest.proto\x12\tc1.c1z.v3\x1a\x1bc1/storage/v3/records.proto\x1a\x19google/protobuf/any.proto\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x96\x04\n" + "\rC1ZManifestV3\x12\x16\n" + "\x06engine\x18\x01 \x01(\tR\x06engine\x122\n" + "\x15engine_schema_version\x18\x02 \x01(\rR\x13engineSchemaVersion\x129\n" + @@ -944,7 +1006,8 @@ const file_c1_c1z_v3_manifest_proto_rawDesc = "" + " \x01(\v2\".google.protobuf.FileDescriptorSetR\vdescriptors\x12<\n" + "\frecord_types\x18\v \x03(\v2\x19.c1.c1z.v3.RecordTypeInfoR\vrecordTypes\x126\n" + "\tsync_runs\x18( \x03(\v2\x19.c1.c1z.v3.SyncRunSummaryR\bsyncRuns\x12&\n" + - "\x0ffold_dead_bytes\x18) \x01(\x03R\rfoldDeadBytes\"\xcc\x01\n" + + "\x0ffold_dead_bytes\x18) \x01(\x03R\rfoldDeadBytes\x12S\n" + + "\x16pebble_id_index_format\x18* \x01(\x0e2\x1e.c1.c1z.v3.PebbleIdIndexFormatR\x13pebbleIdIndexFormat\"\xcc\x01\n" + "\x11IndexedFrameIndex\x126\n" + "\aentries\x18\x01 \x03(\v2\x1c.c1.c1z.v3.IndexedFrameEntryR\aentries\x12$\n" + "\x0etotal_raw_size\x18\x02 \x01(\x03R\ftotalRawSize\x122\n" + @@ -972,45 +1035,51 @@ const file_c1_c1z_v3_manifest_proto_rawDesc = "" + "\x05stats\x18\x06 \x01(\v2\x1e.c1.storage.v3.SyncStatsRecordR\x05stats\"p\n" + "\x12PebbleEngineConfig\x120\n" + "\x14format_major_version\x18\x01 \x01(\rR\x12formatMajorVersion\x12(\n" + - "\x10cache_size_bytes\x18\x02 \x01(\x04R\x0ecacheSizeBytes*\x9b\x01\n" + + "\x10cache_size_bytes\x18\x02 \x01(\x04R\x0ecacheSizeBytes*\x96\x01\n" + + "\x13PebbleIdIndexFormat\x12&\n" + + "\"PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED\x10\x00\x12-\n" + + ")PEBBLE_ID_INDEX_FORMAT_LEGACY_EXTERNAL_ID\x10\x01\x12(\n" + + "$PEBBLE_ID_INDEX_FORMAT_STRUCTURED_V1\x10\x02*\x9b\x01\n" + "\x0fPayloadEncoding\x12 \n" + "\x1cPAYLOAD_ENCODING_UNSPECIFIED\x10\x00\x12\x1d\n" + "\x19PAYLOAD_ENCODING_TAR_ZSTD\x10\x01\x12\x18\n" + "\x14PAYLOAD_ENCODING_TAR\x10\x02\x12!\n" + "\x1dPAYLOAD_ENCODING_INDEXED_ZSTD\x10\x05\"\x04\b\x03\x10\x03\"\x04\b\x04\x10\x04B0Z.github.com/conductorone/baton-sdk/pb/c1/c1z/v3b\x06proto3" -var file_c1_c1z_v3_manifest_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_c1z_v3_manifest_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_c1_c1z_v3_manifest_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_c1_c1z_v3_manifest_proto_goTypes = []any{ - (PayloadEncoding)(0), // 0: c1.c1z.v3.PayloadEncoding - (*C1ZManifestV3)(nil), // 1: c1.c1z.v3.C1ZManifestV3 - (*IndexedFrameIndex)(nil), // 2: c1.c1z.v3.IndexedFrameIndex - (*IndexedFrameEntry)(nil), // 3: c1.c1z.v3.IndexedFrameEntry - (*RecordTypeInfo)(nil), // 4: c1.c1z.v3.RecordTypeInfo - (*SyncRunSummary)(nil), // 5: c1.c1z.v3.SyncRunSummary - (*PebbleEngineConfig)(nil), // 6: c1.c1z.v3.PebbleEngineConfig - (*anypb.Any)(nil), // 7: google.protobuf.Any - (*descriptorpb.FileDescriptorSet)(nil), // 8: google.protobuf.FileDescriptorSet - (v3.SyncType)(0), // 9: c1.storage.v3.SyncType - (*timestamppb.Timestamp)(nil), // 10: google.protobuf.Timestamp - (*v3.SyncStatsRecord)(nil), // 11: c1.storage.v3.SyncStatsRecord + (PebbleIdIndexFormat)(0), // 0: c1.c1z.v3.PebbleIdIndexFormat + (PayloadEncoding)(0), // 1: c1.c1z.v3.PayloadEncoding + (*C1ZManifestV3)(nil), // 2: c1.c1z.v3.C1ZManifestV3 + (*IndexedFrameIndex)(nil), // 3: c1.c1z.v3.IndexedFrameIndex + (*IndexedFrameEntry)(nil), // 4: c1.c1z.v3.IndexedFrameEntry + (*RecordTypeInfo)(nil), // 5: c1.c1z.v3.RecordTypeInfo + (*SyncRunSummary)(nil), // 6: c1.c1z.v3.SyncRunSummary + (*PebbleEngineConfig)(nil), // 7: c1.c1z.v3.PebbleEngineConfig + (*anypb.Any)(nil), // 8: google.protobuf.Any + (*descriptorpb.FileDescriptorSet)(nil), // 9: google.protobuf.FileDescriptorSet + (v3.SyncType)(0), // 10: c1.storage.v3.SyncType + (*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp + (*v3.SyncStatsRecord)(nil), // 12: c1.storage.v3.SyncStatsRecord } var file_c1_c1z_v3_manifest_proto_depIdxs = []int32{ - 7, // 0: c1.c1z.v3.C1ZManifestV3.engine_config:type_name -> google.protobuf.Any - 0, // 1: c1.c1z.v3.C1ZManifestV3.payload_encoding:type_name -> c1.c1z.v3.PayloadEncoding - 8, // 2: c1.c1z.v3.C1ZManifestV3.descriptors:type_name -> google.protobuf.FileDescriptorSet - 4, // 3: c1.c1z.v3.C1ZManifestV3.record_types:type_name -> c1.c1z.v3.RecordTypeInfo - 5, // 4: c1.c1z.v3.C1ZManifestV3.sync_runs:type_name -> c1.c1z.v3.SyncRunSummary - 3, // 5: c1.c1z.v3.IndexedFrameIndex.entries:type_name -> c1.c1z.v3.IndexedFrameEntry - 9, // 6: c1.c1z.v3.SyncRunSummary.type:type_name -> c1.storage.v3.SyncType - 10, // 7: c1.c1z.v3.SyncRunSummary.started_at:type_name -> google.protobuf.Timestamp - 10, // 8: c1.c1z.v3.SyncRunSummary.ended_at:type_name -> google.protobuf.Timestamp - 11, // 9: c1.c1z.v3.SyncRunSummary.stats:type_name -> c1.storage.v3.SyncStatsRecord - 10, // [10:10] is the sub-list for method output_type - 10, // [10:10] is the sub-list for method input_type - 10, // [10:10] is the sub-list for extension type_name - 10, // [10:10] is the sub-list for extension extendee - 0, // [0:10] is the sub-list for field type_name + 8, // 0: c1.c1z.v3.C1ZManifestV3.engine_config:type_name -> google.protobuf.Any + 1, // 1: c1.c1z.v3.C1ZManifestV3.payload_encoding:type_name -> c1.c1z.v3.PayloadEncoding + 9, // 2: c1.c1z.v3.C1ZManifestV3.descriptors:type_name -> google.protobuf.FileDescriptorSet + 5, // 3: c1.c1z.v3.C1ZManifestV3.record_types:type_name -> c1.c1z.v3.RecordTypeInfo + 6, // 4: c1.c1z.v3.C1ZManifestV3.sync_runs:type_name -> c1.c1z.v3.SyncRunSummary + 0, // 5: c1.c1z.v3.C1ZManifestV3.pebble_id_index_format:type_name -> c1.c1z.v3.PebbleIdIndexFormat + 4, // 6: c1.c1z.v3.IndexedFrameIndex.entries:type_name -> c1.c1z.v3.IndexedFrameEntry + 10, // 7: c1.c1z.v3.SyncRunSummary.type:type_name -> c1.storage.v3.SyncType + 11, // 8: c1.c1z.v3.SyncRunSummary.started_at:type_name -> google.protobuf.Timestamp + 11, // 9: c1.c1z.v3.SyncRunSummary.ended_at:type_name -> google.protobuf.Timestamp + 12, // 10: c1.c1z.v3.SyncRunSummary.stats:type_name -> c1.storage.v3.SyncStatsRecord + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_c1_c1z_v3_manifest_proto_init() } @@ -1023,7 +1092,7 @@ func file_c1_c1z_v3_manifest_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_c1z_v3_manifest_proto_rawDesc), len(file_c1_c1z_v3_manifest_proto_rawDesc)), - NumEnums: 1, + NumEnums: 2, NumMessages: 6, NumExtensions: 0, NumServices: 0, diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go new file mode 100644 index 00000000..437dfd89 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -0,0 +1,914 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build !protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.Mode = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.Mode = b.Mode + return m0 +} + +// SourceCacheScope is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_hash so a future sync can replay them as a unit. +type SourceCacheScope struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheScope) Reset() { + *x = SourceCacheScope{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheScope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheScope) ProtoMessage() {} + +func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheScope) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheScope) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheScope) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheScope) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheScope) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheScope) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheScope_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { + m0 := &SourceCacheScope{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_hash into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose etag it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3" json:"overlay,omitempty"` + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheReplay) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheReplay) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeHash string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.Overlay = b.Overlay + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per action; a connector +// that keeps asking past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Queries []*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3" json:"queries,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + return x.Queries + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.Queries = v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.Queries = b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// An ABSENT query (asked but not answered — e.g. dropped to the answer +// size budget) is distinct from found=false: not-found means the previous +// sync has no entry (fetch fresh); absent means unresolved (the connector +// may ask again, subject to the bounce cap). Not-found answers are always +// complete for the queried set — only found answers with large etags are +// ever dropped to budget. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Answers []*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3" json:"answers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + return x.Answers + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.Answers = v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.Answers = b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { + x.ScopeHash = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeHash string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + Found bool `protobuf:"varint,3,opt,name=found,proto3" json:"found,omitempty"` + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string `protobuf:"bytes,4,opt,name=etag,proto3" json:"etag,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { + x.Etag = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeHash string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + x.Found = b.Found + x.Etag = b.Etag + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + + "\x10SourceCacheScope\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + + "\x11SourceCacheReplay\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + + "\x18SourceCacheLookupAnswers\x12J\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + + "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go new file mode 100644 index 00000000..a301fbcb --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go @@ -0,0 +1,1003 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_source_cache.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheCapability) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheCapabilityMultiError, or nil if none found. +func (m *SourceCacheCapability) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheCapability) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Mode + + if len(errors) > 0 { + return SourceCacheCapabilityMultiError(errors) + } + + return nil +} + +// SourceCacheCapabilityMultiError is an error wrapping multiple validation +// errors returned by SourceCacheCapability.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheCapabilityMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheCapabilityMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheCapabilityMultiError) AllErrors() []error { return m } + +// SourceCacheCapabilityValidationError is the validation error returned by +// SourceCacheCapability.Validate if the designated constraints aren't met. +type SourceCacheCapabilityValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheCapabilityValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheCapabilityValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheCapabilityValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheCapabilityValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheCapabilityValidationError) ErrorName() string { + return "SourceCacheCapabilityValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheCapabilityValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheCapability.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheCapabilityValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheCapabilityValidationError{} + +// Validate checks the field values on SourceCacheScope with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheScope) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheScope with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheScopeMultiError, or nil if none found. +func (m *SourceCacheScope) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheScope) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeHash + + // no validation rules for Etag + + if len(errors) > 0 { + return SourceCacheScopeMultiError(errors) + } + + return nil +} + +// SourceCacheScopeMultiError is an error wrapping multiple validation errors +// returned by SourceCacheScope.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheScopeMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheScopeMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheScopeMultiError) AllErrors() []error { return m } + +// SourceCacheScopeValidationError is the validation error returned by +// SourceCacheScope.Validate if the designated constraints aren't met. +type SourceCacheScopeValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheScopeValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheScopeValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheScopeValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheScopeValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheScopeValidationError) ErrorName() string { return "SourceCacheScopeValidationError" } + +// Error satisfies the builtin error interface +func (e SourceCacheScopeValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheScope.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheScopeValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheScopeValidationError{} + +// Validate checks the field values on SourceCacheReplay with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheReplay) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheReplay with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheReplayMultiError, or nil if none found. +func (m *SourceCacheReplay) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheReplay) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeHash + + // no validation rules for Etag + + // no validation rules for Overlay + + if len(errors) > 0 { + return SourceCacheReplayMultiError(errors) + } + + return nil +} + +// SourceCacheReplayMultiError is an error wrapping multiple validation errors +// returned by SourceCacheReplay.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheReplayMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheReplayMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheReplayMultiError) AllErrors() []error { return m } + +// SourceCacheReplayValidationError is the validation error returned by +// SourceCacheReplay.Validate if the designated constraints aren't met. +type SourceCacheReplayValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheReplayValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheReplayValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheReplayValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheReplayValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheReplayValidationError) ErrorName() string { + return "SourceCacheReplayValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheReplayValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheReplay.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheReplayValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheReplayValidationError{} + +// Validate checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupOffer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupOfferMultiError, or nil if none found. +func (m *SourceCacheLookupOffer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupOffer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SourceCacheLookupOfferMultiError(errors) + } + + return nil +} + +// SourceCacheLookupOfferMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupOffer.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupOfferMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupOfferMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupOfferMultiError) AllErrors() []error { return m } + +// SourceCacheLookupOfferValidationError is the validation error returned by +// SourceCacheLookupOffer.Validate if the designated constraints aren't met. +type SourceCacheLookupOfferValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupOfferValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupOfferValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupOfferValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupOfferValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupOfferValidationError) ErrorName() string { + return "SourceCacheLookupOfferValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupOfferValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupOffer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupOfferValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupOfferValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAskMultiError, or nil if none found. +func (m *SourceCacheLookupAsk) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetQueries()); l < 1 || l > 4096 { + err := SourceCacheLookupAskValidationError{ + field: "Queries", + reason: "value must contain between 1 and 4096 items, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + for idx, item := range m.GetQueries() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAskMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAskMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAsk.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAskMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAskMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAskMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAskValidationError is the validation error returned by +// SourceCacheLookupAsk.Validate if the designated constraints aren't met. +type SourceCacheLookupAskValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAskValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAskValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAskValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAskValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAskValidationError) ErrorName() string { + return "SourceCacheLookupAskValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAskValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAskValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAskValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswersMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + for idx, item := range m.GetAnswers() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswersMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswersMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAnswers.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAnswersMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswersMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswersMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswersValidationError is the validation error returned by +// SourceCacheLookupAnswers.Validate if the designated constraints aren't met. +type SourceCacheLookupAnswersValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswersValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswersValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswersValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswersValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswersValidationError) ErrorName() string { + return "SourceCacheLookupAnswersValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswersValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswersValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswersValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk_Query) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAsk_QueryMultiError, or nil if none found. +func (m *SourceCacheLookupAsk_Query) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk_Query) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeHash()); l < 1 || l > 256 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return SourceCacheLookupAsk_QueryMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAsk_QueryMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAsk_Query.ValidateAll() if +// the designated constraints aren't met. +type SourceCacheLookupAsk_QueryMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAsk_QueryMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAsk_QueryMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAsk_QueryValidationError is the validation error returned +// by SourceCacheLookupAsk_Query.Validate if the designated constraints aren't met. +type SourceCacheLookupAsk_QueryValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAsk_QueryValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAsk_QueryValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAsk_QueryValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAsk_QueryValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAsk_QueryValidationError) ErrorName() string { + return "SourceCacheLookupAsk_QueryValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAsk_QueryValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk_Query.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAsk_QueryValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAsk_QueryValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers_Answer with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers_Answer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers_Answer with +// the rules defined in the proto definition for this message. If any rules +// are violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswers_AnswerMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers_Answer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers_Answer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeHash()); l < 1 || l > 256 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + // no validation rules for Found + + if m.GetEtag() != "" { + + if len(m.GetEtag()) > 65536 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "Etag", + reason: "value length must be at most 65536 bytes", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswers_AnswerMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswers_AnswerMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAnswers_Answer.ValidateAll() +// if the designated constraints aren't met. +type SourceCacheLookupAnswers_AnswerMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswers_AnswerMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswers_AnswerMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswers_AnswerValidationError is the validation error +// returned by SourceCacheLookupAnswers_Answer.Validate if the designated +// constraints aren't met. +type SourceCacheLookupAnswers_AnswerValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswers_AnswerValidationError) ErrorName() string { + return "SourceCacheLookupAnswers_AnswerValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswers_AnswerValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers_Answer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswers_AnswerValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswers_AnswerValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go new file mode 100644 index 00000000..23ad8c5b --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -0,0 +1,860 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.xxx_hidden_Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.xxx_hidden_Mode = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Mode = b.Mode + return m0 +} + +// SourceCacheScope is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_hash so a future sync can replay them as a unit. +type SourceCacheScope struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheScope) Reset() { + *x = SourceCacheScope{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheScope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheScope) ProtoMessage() {} + +func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheScope) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheScope) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheScope) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheScope) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheScope) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheScope) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheScope_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { + m0 := &SourceCacheScope{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_hash into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose etag it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheReplay) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.xxx_hidden_Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheReplay) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.xxx_hidden_Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeHash string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_Overlay = b.Overlay + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per action; a connector +// that keeps asking past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Queries *[]*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + if x.xxx_hidden_Queries != nil { + return *x.xxx_hidden_Queries + } + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.xxx_hidden_Queries = &v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Queries = &b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// An ABSENT query (asked but not answered — e.g. dropped to the answer +// size budget) is distinct from found=false: not-found means the previous +// sync has no entry (fetch fresh); absent means unresolved (the connector +// may ask again, subject to the bounce cap). Not-found answers are always +// complete for the queried set — only found answers with large etags are +// ever dropped to budget. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Answers *[]*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + if x.xxx_hidden_Answers != nil { + return *x.xxx_hidden_Answers + } + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.xxx_hidden_Answers = &v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Answers = &b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeHash string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Found bool `protobuf:"varint,3,opt,name=found,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,4,opt,name=etag,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeHash string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Found = b.Found + x.xxx_hidden_Etag = b.Etag + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + + "\x10SourceCacheScope\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + + "\x11SourceCacheReplay\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + + "\x18SourceCacheLookupAnswers\x12J\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + + "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go new file mode 100644 index 00000000..52a9383b --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -0,0 +1,250 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build !protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// whose resource carries ONLY the resource type (empty resource id); the +// connector answers with grants for the whole type, across as many +// paginated cursors as it chooses to spawn (see SpawnCursors). +// +// This is the grants-phase analogue of StaticEntitlements: the connector +// takes over enumeration for the type, and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SpawnCursors is attached to a ListGrants response to enqueue additional +// independent sibling cursors. Each token is delivered back to the +// connector as the page token of its own action — scheduled by the +// syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants +// responses; the spawned actions inherit the response's resource identity +// (type only for type-scoped calls, type+resource for per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +type SpawnCursors struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Page tokens for the sibling cursors to enqueue, one action each. + PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3" json:"page_tokens,omitempty"` + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3" json:"estimated_total,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpawnCursors) Reset() { + *x = SpawnCursors{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpawnCursors) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpawnCursors) ProtoMessage() {} + +func (x *SpawnCursors) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SpawnCursors) GetPageTokens() []string { + if x != nil { + return x.PageTokens + } + return nil +} + +func (x *SpawnCursors) GetEstimatedTotal() int64 { + if x != nil { + return x.EstimatedTotal + } + return 0 +} + +func (x *SpawnCursors) SetPageTokens(v []string) { + x.PageTokens = v +} + +func (x *SpawnCursors) SetEstimatedTotal(v int64) { + x.EstimatedTotal = v +} + +type SpawnCursors_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + PageTokens []string + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 SpawnCursors_builder) Build() *SpawnCursors { + m0 := &SpawnCursors{} + b, x := &b0, m0 + _, _ = b, x + x.PageTokens = b.PageTokens + x.EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\"\x12\n" + + "\x10TypeScopedGrants\"X\n" + + "\fSpawnCursors\x12\x1f\n" + + "\vpage_tokens\x18\x01 \x03(\tR\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go new file mode 100644 index 00000000..5679e6e9 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go @@ -0,0 +1,237 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on TypeScopedGrants with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *TypeScopedGrants) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on TypeScopedGrants with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// TypeScopedGrantsMultiError, or nil if none found. +func (m *TypeScopedGrants) ValidateAll() error { + return m.validate(true) +} + +func (m *TypeScopedGrants) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return TypeScopedGrantsMultiError(errors) + } + + return nil +} + +// TypeScopedGrantsMultiError is an error wrapping multiple validation errors +// returned by TypeScopedGrants.ValidateAll() if the designated constraints +// aren't met. +type TypeScopedGrantsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m TypeScopedGrantsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m TypeScopedGrantsMultiError) AllErrors() []error { return m } + +// TypeScopedGrantsValidationError is the validation error returned by +// TypeScopedGrants.Validate if the designated constraints aren't met. +type TypeScopedGrantsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e TypeScopedGrantsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e TypeScopedGrantsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e TypeScopedGrantsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e TypeScopedGrantsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e TypeScopedGrantsValidationError) ErrorName() string { return "TypeScopedGrantsValidationError" } + +// Error satisfies the builtin error interface +func (e TypeScopedGrantsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sTypeScopedGrants.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = TypeScopedGrantsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = TypeScopedGrantsValidationError{} + +// Validate checks the field values on SpawnCursors with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *SpawnCursors) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SpawnCursors with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in SpawnCursorsMultiError, or +// nil if none found. +func (m *SpawnCursors) ValidateAll() error { + return m.validate(true) +} + +func (m *SpawnCursors) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for EstimatedTotal + + if len(errors) > 0 { + return SpawnCursorsMultiError(errors) + } + + return nil +} + +// SpawnCursorsMultiError is an error wrapping multiple validation errors +// returned by SpawnCursors.ValidateAll() if the designated constraints aren't met. +type SpawnCursorsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SpawnCursorsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SpawnCursorsMultiError) AllErrors() []error { return m } + +// SpawnCursorsValidationError is the validation error returned by +// SpawnCursors.Validate if the designated constraints aren't met. +type SpawnCursorsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SpawnCursorsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SpawnCursorsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SpawnCursorsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SpawnCursorsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SpawnCursorsValidationError) ErrorName() string { return "SpawnCursorsValidationError" } + +// Error satisfies the builtin error interface +func (e SpawnCursorsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSpawnCursors.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SpawnCursorsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SpawnCursorsValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go new file mode 100644 index 00000000..e9b3571b --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -0,0 +1,247 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// whose resource carries ONLY the resource type (empty resource id); the +// connector answers with grants for the whole type, across as many +// paginated cursors as it chooses to spawn (see SpawnCursors). +// +// This is the grants-phase analogue of StaticEntitlements: the connector +// takes over enumeration for the type, and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SpawnCursors is attached to a ListGrants response to enqueue additional +// independent sibling cursors. Each token is delivered back to the +// connector as the page token of its own action — scheduled by the +// syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants +// responses; the spawned actions inherit the response's resource identity +// (type only for type-scoped calls, type+resource for per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +type SpawnCursors struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3"` + xxx_hidden_EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpawnCursors) Reset() { + *x = SpawnCursors{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpawnCursors) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpawnCursors) ProtoMessage() {} + +func (x *SpawnCursors) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SpawnCursors) GetPageTokens() []string { + if x != nil { + return x.xxx_hidden_PageTokens + } + return nil +} + +func (x *SpawnCursors) GetEstimatedTotal() int64 { + if x != nil { + return x.xxx_hidden_EstimatedTotal + } + return 0 +} + +func (x *SpawnCursors) SetPageTokens(v []string) { + x.xxx_hidden_PageTokens = v +} + +func (x *SpawnCursors) SetEstimatedTotal(v int64) { + x.xxx_hidden_EstimatedTotal = v +} + +type SpawnCursors_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + PageTokens []string + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 SpawnCursors_builder) Build() *SpawnCursors { + m0 := &SpawnCursors{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_PageTokens = b.PageTokens + x.xxx_hidden_EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\"\x12\n" + + "\x10TypeScopedGrants\"X\n" + + "\fSpawnCursors\x12\x1f\n" + + "\vpage_tokens\x18\x01 \x03(\tR\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/grant.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/grant.pb.go index 5d8799eb..15cef2e8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/grant.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/grant.pb.go @@ -82,12 +82,13 @@ func (b0 GrantSources_builder) Build() *GrantSources { } type Grant struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - Entitlement *Entitlement `protobuf:"bytes,1,opt,name=entitlement,proto3" json:"entitlement,omitempty"` - Principal *Resource `protobuf:"bytes,2,opt,name=principal,proto3" json:"principal,omitempty"` - Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` - Sources *GrantSources `protobuf:"bytes,5,opt,name=sources,proto3" json:"sources,omitempty"` - Annotations []*anypb.Any `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + Entitlement *Entitlement `protobuf:"bytes,1,opt,name=entitlement,proto3" json:"entitlement,omitempty"` + Principal *Resource `protobuf:"bytes,2,opt,name=principal,proto3" json:"principal,omitempty"` + // These ids may not map one to one with the grant itself. + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` + Sources *GrantSources `protobuf:"bytes,5,opt,name=sources,proto3" json:"sources,omitempty"` + Annotations []*anypb.Any `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -210,6 +211,7 @@ type Grant_builder struct { Entitlement *Entitlement Principal *Resource + // These ids may not map one to one with the grant itself. Id string Sources *GrantSources Annotations []*anypb.Any diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/grant_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/grant_protoopaque.pb.go index 632b6c98..7caced7b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/grant_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/grant_protoopaque.pb.go @@ -212,6 +212,7 @@ type Grant_builder struct { Entitlement *Entitlement Principal *Resource + // These ids may not map one to one with the grant itself. Id string Sources *GrantSources Annotations []*anypb.Any diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go new file mode 100644 index 00000000..5d0c2090 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go @@ -0,0 +1,249 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build !protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.RowKind = v +} + +func (x *LookupRequest) SetScopeHash(v string) { + x.ScopeHash = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool `protobuf:"varint,1,opt,name=found,proto3" json:"found,omitempty"` + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *LookupResponse) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.Found = v +} + +func (x *LookupResponse) SetEtag(v string) { + x.Etag = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.Found = b.Found + x.Etag = b.Etag + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + + "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go new file mode 100644 index 00000000..dfc7cca7 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go @@ -0,0 +1,271 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on LookupRequest with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupRequest) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupRequest with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupRequestMultiError, or +// nil if none found. +func (m *LookupRequest) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupRequest) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := utf8.RuneCountInString(m.GetRowKind()); l < 1 || l > 64 { + err := LookupRequestValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := utf8.RuneCountInString(m.GetScopeHash()); l < 1 || l > 256 { + err := LookupRequestValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupRequestMultiError(errors) + } + + return nil +} + +// LookupRequestMultiError is an error wrapping multiple validation errors +// returned by LookupRequest.ValidateAll() if the designated constraints +// aren't met. +type LookupRequestMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupRequestMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupRequestMultiError) AllErrors() []error { return m } + +// LookupRequestValidationError is the validation error returned by +// LookupRequest.Validate if the designated constraints aren't met. +type LookupRequestValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupRequestValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupRequestValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupRequestValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupRequestValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupRequestValidationError) ErrorName() string { return "LookupRequestValidationError" } + +// Error satisfies the builtin error interface +func (e LookupRequestValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupRequest.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupRequestValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupRequestValidationError{} + +// Validate checks the field values on LookupResponse with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupResponse) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupResponse with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupResponseMultiError, +// or nil if none found. +func (m *LookupResponse) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupResponse) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Found + + if utf8.RuneCountInString(m.GetEtag()) > 65536 { + err := LookupResponseValidationError{ + field: "Etag", + reason: "value length must be at most 65536 runes", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupResponseMultiError(errors) + } + + return nil +} + +// LookupResponseMultiError is an error wrapping multiple validation errors +// returned by LookupResponse.ValidateAll() if the designated constraints +// aren't met. +type LookupResponseMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupResponseMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupResponseMultiError) AllErrors() []error { return m } + +// LookupResponseValidationError is the validation error returned by +// LookupResponse.Validate if the designated constraints aren't met. +type LookupResponseValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupResponseValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupResponseValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupResponseValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupResponseValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupResponseValidationError) ErrorName() string { return "LookupResponseValidationError" } + +// Error satisfies the builtin error interface +func (e LookupResponseValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupResponse.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupResponseValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupResponseValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go new file mode 100644 index 00000000..8db25cd3 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go @@ -0,0 +1,145 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + BatonSourceCacheService_Lookup_FullMethodName = "/c1.connectorapi.baton.v1.BatonSourceCacheService/Lookup" +) + +// BatonSourceCacheServiceClient is the client API for BatonSourceCacheService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (etag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceClient interface { + Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) +} + +type batonSourceCacheServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewBatonSourceCacheServiceClient(cc grpc.ClientConnInterface) BatonSourceCacheServiceClient { + return &batonSourceCacheServiceClient{cc} +} + +func (c *batonSourceCacheServiceClient) Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(LookupResponse) + err := c.cc.Invoke(ctx, BatonSourceCacheService_Lookup_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// BatonSourceCacheServiceServer is the server API for BatonSourceCacheService service. +// All implementations should embed UnimplementedBatonSourceCacheServiceServer +// for forward compatibility. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (etag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceServer interface { + Lookup(context.Context, *LookupRequest) (*LookupResponse, error) +} + +// UnimplementedBatonSourceCacheServiceServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedBatonSourceCacheServiceServer struct{} + +func (UnimplementedBatonSourceCacheServiceServer) Lookup(context.Context, *LookupRequest) (*LookupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Lookup not implemented") +} +func (UnimplementedBatonSourceCacheServiceServer) testEmbeddedByValue() {} + +// UnsafeBatonSourceCacheServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to BatonSourceCacheServiceServer will +// result in compilation errors. +type UnsafeBatonSourceCacheServiceServer interface { + mustEmbedUnimplementedBatonSourceCacheServiceServer() +} + +func RegisterBatonSourceCacheServiceServer(s grpc.ServiceRegistrar, srv BatonSourceCacheServiceServer) { + // If the following call pancis, it indicates UnimplementedBatonSourceCacheServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&BatonSourceCacheService_ServiceDesc, srv) +} + +func _BatonSourceCacheService_Lookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BatonSourceCacheService_Lookup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, req.(*LookupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// BatonSourceCacheService_ServiceDesc is the grpc.ServiceDesc for BatonSourceCacheService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var BatonSourceCacheService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "c1.connectorapi.baton.v1.BatonSourceCacheService", + HandlerType: (*BatonSourceCacheServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Lookup", + Handler: _BatonSourceCacheService_Lookup_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "c1/connectorapi/baton/v1/source_cache.proto", +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go new file mode 100644 index 00000000..9bfc8953 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go @@ -0,0 +1,235 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *LookupRequest) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Found bool `protobuf:"varint,1,opt,name=found,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *LookupResponse) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *LookupResponse) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Found = b.Found + x.xxx_hidden_Etag = b.Etag + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + + "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go index 312f32a0..29552e74 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go @@ -455,8 +455,11 @@ type ResourceRecord struct { Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3" json:"parent,omitempty"` Annotations []*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3" json:"annotations,omitempty"` DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -533,6 +536,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.ResourceTypeId = v } @@ -561,6 +571,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -593,6 +607,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -606,6 +623,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.Parent = b.Parent x.Annotations = b.Annotations x.DiscoveredAt = b.DiscoveredAt + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -632,8 +650,11 @@ type EntitlementRecord struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3" json:"grantable_to_resource_type_ids,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EntitlementRecord) Reset() { @@ -724,6 +745,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.ExternalId = v } @@ -760,6 +788,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -806,6 +838,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -821,6 +856,7 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.DiscoveredAt = b.DiscoveredAt x.Slug = b.Slug x.GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -846,9 +882,15 @@ type GrantRecord struct { Annotations []*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3" json:"annotations,omitempty"` // map — same wire shape as // c1.connector.v2.GrantSources.sources. - Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -932,6 +974,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.ExternalId = v } @@ -964,6 +1013,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.Sources = v } +func (x *GrantRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1032,6 +1085,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1046,6 +1105,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.NeedsExpansion = b.NeedsExpansion x.Annotations = b.Annotations x.Sources = b.Sources + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -1653,6 +1713,130 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_hash), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-computed scope hash (lowercase hex). + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string `protobuf:"bytes,3,opt,name=etag,proto3" json:"etag,omitempty"` + DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheEntryRecord) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheEntryRecord) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed scope hash (lowercase hex). + ScopeHash string + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string + DiscoveredAt *timestamppb.Timestamp +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.DiscoveredAt = b.DiscoveredAt + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1677,7 +1861,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1687,8 +1871,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + + "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1701,8 +1887,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + + "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1715,7 +1903,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + + "\x11source_scope_hash\x18\n" + + " \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1765,7 +1956,15 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + + "\x14source_cache_entries\x12\brow_kind\x12\n" + + "scope_hash*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1775,58 +1974,60 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 16: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp + (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1842,7 +2043,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go index 29287724..0c0b299e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go @@ -544,6 +544,8 @@ func (m *ResourceRecord) validate(all bool) error { } } + // no validation rules for SourceScopeHash + if len(errors) > 0 { return ResourceRecordMultiError(errors) } @@ -746,6 +748,8 @@ func (m *EntitlementRecord) validate(all bool) error { // no validation rules for Slug + // no validation rules for SourceScopeHash + if len(errors) > 0 { return EntitlementRecordMultiError(errors) } @@ -1048,6 +1052,8 @@ func (m *GrantRecord) validate(all bool) error { } } + // no validation rules for SourceScopeHash + if len(errors) > 0 { return GrantRecordMultiError(errors) } @@ -1683,3 +1689,140 @@ var _ interface { Cause() error ErrorName() string } = SessionRecordValidationError{} + +// Validate checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheEntryRecord) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheEntryRecordMultiError, or nil if none found. +func (m *SourceCacheEntryRecord) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheEntryRecord) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for RowKind + + // no validation rules for ScopeHash + + // no validation rules for Etag + + if all { + switch v := interface{}(m.GetDiscoveredAt()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetDiscoveredAt()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + } + } + } + + if len(errors) > 0 { + return SourceCacheEntryRecordMultiError(errors) + } + + return nil +} + +// SourceCacheEntryRecordMultiError is an error wrapping multiple validation +// errors returned by SourceCacheEntryRecord.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheEntryRecordMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheEntryRecordMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheEntryRecordMultiError) AllErrors() []error { return m } + +// SourceCacheEntryRecordValidationError is the validation error returned by +// SourceCacheEntryRecord.Validate if the designated constraints aren't met. +type SourceCacheEntryRecordValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheEntryRecordValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheEntryRecordValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheEntryRecordValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheEntryRecordValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheEntryRecordValidationError) ErrorName() string { + return "SourceCacheEntryRecordValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheEntryRecordValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheEntryRecord.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheEntryRecordValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheEntryRecordValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go index bf2acc8c..797ad824 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go @@ -441,16 +441,17 @@ func (b0 ResourceTypeRecord_builder) Build() *ResourceTypeRecord { } type ResourceRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` - xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` - xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` - xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` - xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` + xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` + xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` + xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` + xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -529,6 +530,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.xxx_hidden_ResourceTypeId = v } @@ -557,6 +565,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.xxx_hidden_DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -589,6 +601,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -602,6 +617,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.xxx_hidden_Parent = b.Parent x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } @@ -616,6 +632,7 @@ type EntitlementRecord struct { xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` xxx_hidden_Slug string `protobuf:"bytes,9,opt,name=slug,proto3"` xxx_hidden_GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -710,6 +727,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -746,6 +770,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.xxx_hidden_GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -792,6 +820,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -807,21 +838,23 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.xxx_hidden_DiscoveredAt = b.DiscoveredAt x.xxx_hidden_Slug = b.Slug x.xxx_hidden_GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } type GrantRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` - xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` - xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` - xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` - xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` - xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` + xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` + xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` + xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` + xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -907,6 +940,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -939,6 +979,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.xxx_hidden_Sources = v } +func (x *GrantRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1007,6 +1051,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1021,6 +1071,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.xxx_hidden_NeedsExpansion = b.NeedsExpansion x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_Sources = b.Sources + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } @@ -1622,6 +1673,126 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_hash), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,3,opt,name=etag,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheEntryRecord) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.xxx_hidden_DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheEntryRecord) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.xxx_hidden_DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.xxx_hidden_DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.xxx_hidden_DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed scope hash (lowercase hex). + ScopeHash string + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string + DiscoveredAt *timestamppb.Timestamp +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1646,7 +1817,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1656,8 +1827,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + + "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1670,8 +1843,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + + "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1684,7 +1859,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + + "\x11source_scope_hash\x18\n" + + " \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1734,7 +1912,15 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + + "\x14source_cache_entries\x12\brow_kind\x12\n" + + "scope_hash*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1744,58 +1930,60 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 16: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp + (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1811,7 +1999,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/bid/bid.go b/vendor/github.com/conductorone/baton-sdk/pkg/bid/bid.go index cd24e2cd..b6e541ad 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/bid/bid.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/bid/bid.go @@ -100,11 +100,11 @@ func NewBidParseError(bs *bidScanner, msg string, a ...any) *BIDParseError { func MakeBid(b BID) (string, error) { switch bType := b.(type) { case *v2.Resource: - return makeResourceBid(bType) + return MakeResourceBid(bType) case *v2.Entitlement: - return makeEntitlementBid(bType) + return MakeEntitlementBid(bType) case *v2.Grant: - return makeGrantBid(bType) + return MakeGrantBid(bType) } return "", NewBidStringError(b, "unknown bid type: %T", b) } @@ -164,7 +164,7 @@ func entitlementPartToStr(e *v2.Entitlement) (string, error) { return strings.Join([]string{resourcePart, escapeParts(e.GetSlug())}, ":"), nil } -func makeResourceBid(r *v2.Resource) (string, error) { +func MakeResourceBid(r *v2.Resource) (string, error) { resourcePart, err := resourcePartToStr(r) if err != nil { return "", err @@ -173,7 +173,7 @@ func makeResourceBid(r *v2.Resource) (string, error) { return strings.Join([]string{BidPrefix, ResourceBidPrefix, resourcePart}, ":"), nil } -func makeEntitlementBid(e *v2.Entitlement) (string, error) { +func MakeEntitlementBid(e *v2.Entitlement) (string, error) { entitlementPart, err := entitlementPartToStr(e) if err != nil { return "", err @@ -182,7 +182,7 @@ func makeEntitlementBid(e *v2.Entitlement) (string, error) { return strings.Join([]string{BidPrefix, EntitlementBidPrefix, entitlementPart}, ":"), nil } -func makeGrantBid(g *v2.Grant) (string, error) { +func MakeGrantBid(g *v2.Grant) (string, error) { principalPart, err := resourcePartToStr(g.GetPrincipal()) if err != nil { return "", err diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go index 036e9ee9..59d87248 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go @@ -11,6 +11,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/conductorone/baton-sdk/pkg/field" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/spf13/cobra" @@ -20,7 +21,12 @@ import ( ) type RunTimeOpts struct { - SessionStore sessions.SessionStore + SessionStore sessions.SessionStore + // SourceCacheLookup resolves a scope's previous-sync validator for + // source-cache replay (see pkg/sourcecache). In subprocess mode this + // is a gRPC client to the parent SDK's BatonSourceCacheService; when + // unset the framework falls back to NoopLookup. + SourceCacheLookup sourcecache.Lookup TokenSource oauth2.TokenSource SelectedAuthMethod string SyncResourceTypeIDs []string diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go index 8b2d1a80..7bde72bd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go @@ -9,9 +9,11 @@ import ( "fmt" "io" "os" + "sync" "time" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/types" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "github.com/maypok86/otter/v2" @@ -31,10 +33,10 @@ import ( baton_v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" "github.com/conductorone/baton-sdk/pkg/connectorrunner" "github.com/conductorone/baton-sdk/pkg/crypto" - "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/tempdir" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/uhttp" @@ -53,29 +55,55 @@ type eventLogEnabledKey struct{} type ContrainstSetter func(*cobra.Command, field.Configuration) error -// In one shot & service mode, the child process uses this client to connect to the session store server... +// parentControlPlaneDialer opens (lazily, at most once) a single gRPC +// connection from the connector subprocess back to the parent SDK's +// control-plane listener. The same connection multiplexes +// BatonSessionService (connector session data) and BatonSourceCacheService +// (source-cache scope lookups) — the two services share a listener on the +// parent (internal/connector.runServer), so they share a client conn here. // -// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, -// so there is quite a bit of pass through. -func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - return func(_ context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - l := ctxzap.Extract(ctx) - clientTLSConfig, err := utls2.ClientConfig(ctx, serverCfg.GetCredential()) - if err != nil { - return nil, err +// dial returns (nil, nil) when the parent did not start a control-plane +// listener (session store disabled); callers substitute no-op +// implementations in that case. +type parentControlPlaneDialer struct { + once sync.Once + conn *grpc.ClientConn + dialErr error + ctx context.Context + cfg *v1.ServerConfig +} + +func newParentControlPlaneDialer(ctx context.Context, serverCfg *v1.ServerConfig) *parentControlPlaneDialer { + return &parentControlPlaneDialer{ctx: ctx, cfg: serverCfg} +} + +// Close releases the underlying gRPC connection if dial ever succeeded. +// Safe when dial never ran or failed, and safe to call multiple times. +func (d *parentControlPlaneDialer) Close() { + if d.conn == nil { + return + } + _ = d.conn.Close() +} + +func (d *parentControlPlaneDialer) dial() (*grpc.ClientConn, error) { + d.once.Do(func() { + if d.cfg.GetSessionStoreListenPort() == 0 { + return } - if serverCfg.GetSessionStoreListenPort() == 0 { - return &session.NoOpSessionStore{}, nil + clientTLSConfig, err := utls2.ClientConfig(d.ctx, d.cfg.GetCredential()) + if err != nil { + d.dialErr = err + return } - // connected, grpc will handle retries for us. - dialCtx, canc := context.WithTimeout(ctx, 5*time.Second) + dialCtx, canc := context.WithTimeout(d.ctx, 5*time.Second) defer canc() var dialErr error var conn *grpc.ClientConn for { conn, err = grpc.DialContext( //nolint:staticcheck // grpc.DialContext is deprecated but we are using it still. - ctx, - fmt.Sprintf("127.0.0.1:%d", serverCfg.GetSessionStoreListenPort()), + d.ctx, + fmt.Sprintf("127.0.0.1:%d", d.cfg.GetSessionStoreListenPort()), grpc.WithTransportCredentials(credentials.NewTLS(clientTLSConfig)), grpc.WithBlock(), //nolint:staticcheck // grpc.WithBlock is deprecated but we are using it still. ) @@ -84,26 +112,59 @@ func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) select { case <-time.After(time.Millisecond * 500): case <-dialCtx.Done(): - return nil, dialErr + d.dialErr = dialErr + return } continue } break } + d.conn = conn + }) + return d.conn, d.dialErr +} +// In one shot & service mode, the child process uses this client to connect to the session store server... +// +// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, +// so there is quite a bit of pass through. +func getGRPCSessionStoreClient(ctx context.Context, dialer *parentControlPlaneDialer) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + return func(_ context.Context, _ ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + l := ctxzap.Extract(ctx) + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return &session.NoOpSessionStore{}, nil + } client := baton_v1.NewBatonSessionServiceClient(conn) - ss, err := session.NewGRPCSessionStore(ctx, client, opt...) + ss, err := session.NewGRPCSessionStore(ctx, client) if err != nil { - err2 := conn.Close() - if err2 != nil { - l.Error("error closing connection", zap.Error(err2)) - } + l.Error("error creating session store client", zap.Error(err)) return nil, err } return ss, nil } } +// buildGRPCSourceCacheLookup returns the connector-side source-cache Lookup +// that talks to the parent's BatonSourceCacheService over the same +// loopback-TLS connection the session client uses. Returns NoopLookup when +// no parent control-plane listener is configured (matches the "no previous +// sync" behavior). +func buildGRPCSourceCacheLookup(dialer *parentControlPlaneDialer) (sourcecache.Lookup, error) { + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return sourcecache.NoopLookup{}, nil + } + client := baton_v1.NewBatonSourceCacheServiceClient(conn) + return sourcecache.NewGRPCLookup(client), nil +} + func MakeMainCommand[T field.Configurable]( ctx context.Context, name string, @@ -377,13 +438,20 @@ func MakeMainCommand[T field.Configurable]( } } - if v.GetBool(field.ParallelSyncField.GetName()) { - opts = append(opts, connectorrunner.WithWorkerCount(-1)) - } - + // Worker count resolves through viper with no zero sentinel in the + // default chain: explicit flag > env (BATON_WORKERS) > config file > + // connector default (field.WithConnectorDefault on WorkerCountField) + // > shared default (0). A resolved 0 — explicit or defaulted — means + // sequential, which is the runner's zero-value behavior (normalized + // to one worker downstream), so no option is appended; -1 means + // auto-detect. The deprecated --parallel-sync only applies when + // workers resolved to 0. workers := v.GetInt(field.WorkerCountField.GetName()) - if workers != 0 { + switch { + case workers != 0: opts = append(opts, connectorrunner.WithWorkerCount(workers)) + case v.GetBool(field.ParallelSyncField.GetName()): + opts = append(opts, connectorrunner.WithWorkerCount(-1)) } c1zTmpDir := tempdir.Resolve(v.GetString("c1z-temp-dir")) @@ -403,6 +471,15 @@ func MakeMainCommand[T field.Configurable]( opts = append(opts, connectorrunner.WithExternalResourceC1Z(externalResourceC1ZPath)) } + if v.GetString(field.PreviousSyncC1ZField.GetName()) != "" { + previousSyncC1ZPath := v.GetString(field.PreviousSyncC1ZField.GetName()) + _, err := os.Open(previousSyncC1ZPath) + if err != nil { + return fmt.Errorf("the specified previous sync c1z file does not exist: %s", previousSyncC1ZPath) + } + opts = append(opts, connectorrunner.WithPreviousSyncC1Z(previousSyncC1ZPath)) + } + if v.GetString("external-resource-entitlement-id-filter") != "" { externalResourceEntitlementIdFilter := v.GetString("external-resource-entitlement-id-filter") opts = append(opts, connectorrunner.WithExternalResourceEntitlementFilter(externalResourceEntitlementIdFilter)) @@ -439,7 +516,7 @@ func MakeMainCommand[T field.Configurable]( return err } if storageEngine != "" { - opts = append(opts, connectorrunner.WithStorageEngine(dotc1z.Engine(storageEngine))) + opts = append(opts, connectorrunner.WithStorageEngine(c1zstore.Engine(storageEngine))) } taskConcurrency := v.GetInt(field.TaskConcurrencyField.GetName()) @@ -626,7 +703,17 @@ func MakeGRPCServerCommand[T field.Configurable]( runCtx = context.WithValue(runCtx, uhttp.ContextHTTPTimeoutKey, time.Duration(httpTimeout)*time.Second) sessionStoreMaximumSize := v.GetInt(field.ServerSessionStoreMaximumSizeField.GetName()) - sessionConstructor := getGRPCSessionStoreClient(runCtx, serverCfg) + // One dialer per subprocess; the session store client and the + // source-cache lookup client share the underlying gRPC connection + // (both services are registered on the same parent listener in + // internal/connector.runServer). + controlPlaneDialer := newParentControlPlaneDialer(runCtx, serverCfg) + defer controlPlaneDialer.Close() + sessionConstructor := getGRPCSessionStoreClient(runCtx, controlPlaneDialer) + sourceCacheLookup, err := buildGRPCSourceCacheLookup(controlPlaneDialer) + if err != nil { + return fmt.Errorf("failed to build source cache lookup: %w", err) + } c, err := getconnector(runCtx, t, RunTimeOpts{ SessionStore: NewLazyCachingSessionStore(sessionConstructor, func(otterOptions *otter.Options[string, []byte]) { if sessionStoreMaximumSize <= 0 { @@ -635,6 +722,7 @@ func MakeGRPCServerCommand[T field.Configurable]( otterOptions.MaximumWeight = uint64(sessionStoreMaximumSize) } }), + SourceCacheLookup: sourceCacheLookup, SelectedAuthMethod: v.GetString("auth-method"), SyncResourceTypeIDs: v.GetStringSlice("sync-resource-types"), }) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go index 3586605f..f250a20a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go @@ -40,6 +40,7 @@ func RunConnector[T field.Configurable]( } builderOpts = append(builderOpts, connectorbuilder.WithSessionStore(runTimeOpts.SessionStore)) + builderOpts = append(builderOpts, connectorbuilder.WithSourceCache(runTimeOpts.SourceCacheLookup)) c, err := connectorbuilder.NewConnector(ctx, connector, builderOpts...) if err != nil { @@ -160,6 +161,20 @@ func DefineConfigurationV2[T field.Configurable]( } confschema.Fields = fields + // The replay flags are hidden by default (source-cache replay is + // author-opt-in functionality); surface them in help only for + // connectors whose author baked the capability into their runner + // options. Hidden flags still parse, so this is a visibility decision, + // not a behavioral one. + if connectorrunner.DeclaresPreviousSyncCapability(ctx, options...) { + for i, f := range confschema.Fields { + switch f.FieldName { + case field.PreviousSyncC1ZField.FieldName, field.KeepPreviousSyncC1ZField.FieldName: + confschema.Fields[i].SyncerConfig.Hidden = false + } + } + } + // setup CLI with cobra mainCMD := &cobra.Command{ Use: connectorName, @@ -292,6 +307,12 @@ func verifyStructFields[T field.Configurable](schema field.Configuration) error return fmt.Errorf("T must be a struct type, got %v", configType.Kind()) //nolint:staticcheck // we want to capital letter here } for _, field := range schema.Fields { + if field.WasReExported { + // Re-exported shared SDK fields (field.WithConnectorDefault) + // are parsed by the SDK's own flag handling, not the + // connector's configuration struct. + continue + } fieldFound := false for i := 0; i < configType.NumField(); i++ { structField := configType.Field(i) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go index bfc1a39e..68fb27d8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go @@ -22,6 +22,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/metrics" "github.com/conductorone/baton-sdk/pkg/retry" "github.com/conductorone/baton-sdk/pkg/sdk" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/types/tasks" @@ -76,6 +77,7 @@ type builder struct { nowFunc func() time.Time clientSecret *jose.JSONWebKey sessionStore sessions.SessionStore + sourceCache sourcecache.Lookup metadataProvider MetadataProvider validateProvider ValidateProvider ticketManager TicketManagerLimited @@ -241,6 +243,30 @@ func WithSessionStore(ss sessions.SessionStore) Opt { } } +// WithSourceCache supplies the connector-facing source-cache Lookup exposed +// on SyncOpAttrs.SourceCache (see pkg/sourcecache). A nil lookup is +// normalized to NoopLookup so connectors never nil-check. +func WithSourceCache(lookup sourcecache.Lookup) Opt { + return func(b *builder) error { + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + b.sourceCache = lookup + return nil + } +} + +// SetSourceCache implements sourcecache.SetLookup so an in-process runner +// (the syncer, via its connector client) can install/clear the active +// lookup per sync. Subprocess mode instead routes lookups over +// BatonSourceCacheService and never calls this. +func (b *builder) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + b.sourceCache = lookup +} + func (b *builder) options(opts ...Opt) error { for _, opt := range opts { if err := opt(b); err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go index a7d99394..3193ed84 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go @@ -2,11 +2,14 @@ package connectorbuilder import ( "context" + "errors" "fmt" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/conductorone/baton-sdk/pkg/types/tasks" "github.com/conductorone/baton-sdk/pkg/uotel" @@ -64,6 +67,111 @@ type StaticEntitlementSyncerV2 interface { StaticEntitlements(ctx context.Context, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) } +// TypeScopedGrantsSyncer is the grants-phase analogue of +// StaticEntitlementSyncerV2: the connector enumerates grants for a WHOLE +// resource type instead of being called once per resource. Implement it on +// a resource syncer whose ResourceType carries the v2.TypeScopedGrants +// annotation; the syncer then issues ListGrants calls with an empty +// resource id, which the builder routes here. +// +// The first call of a sync arrives with an empty page token (the planning +// call); the connector may answer with rows directly and/or spawn +// additional independent cursors by attaching a v2.SpawnCursors annotation +// whose page tokens are delivered back through opts.PageToken, one action +// each. Source-cache scope/replay/tombstone annotations work exactly as on +// per-resource grants pages. +type TypeScopedGrantsSyncer interface { + GrantsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) +} + +// syncOpAttrs assembles the per-call SyncOpAttrs handed to V2 resource +// syncers. SourceCache is never nil: when the runner supplied no lookup +// (source cache disabled/degraded) connectors see NoopLookup and every +// lookup misses. Session is likewise never a nil-backed wrapper: without a +// configured store, connectors see the erroring no-op store instead of a +// panic on first use. +func (b *builder) syncOpAttrs(activeSyncID string, token pagination.Token) resource.SyncOpAttrs { + sc := b.sourceCache + if sc == nil { + sc = sourcecache.NoopLookup{} + } + ss := b.sessionStore + if ss == nil { + ss = &session.NoOpSessionStore{} + } + return resource.SyncOpAttrs{ + SyncID: activeSyncID, + PageToken: token, + Session: WithSyncId(ss, activeSyncID), + SourceCache: sc, + } +} + +// continuationOpAttrs builds SyncOpAttrs for a list RPC, selecting the +// source-cache lookup by topology: +// +// - a runner-supplied lookup (in-process interface, subprocess loopback +// gRPC) always wins: those topologies answer lookups directly and +// never bounce; +// - otherwise, when the request carries SourceCacheLookupOffer (and/or +// SourceCacheLookupAnswers from a previous bounce), a per-request +// ContinuationLookup is installed — it serves answered scopes and +// defers the rest via ErrLookupDeferred; +// - otherwise NoopLookup (today's behavior). +// +// The returned ContinuationLookup is nil when the continuation is not in +// play for this request. +func (b *builder) continuationOpAttrs(activeSyncID string, token pagination.Token, reqAnnos annotations.Annotations) (resource.SyncOpAttrs, *sourcecache.ContinuationLookup, error) { + attrs := b.syncOpAttrs(activeSyncID, token) + if b.sourceCache != nil { + return attrs, nil, nil + } + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return attrs, nil, fmt.Errorf("error parsing source-cache lookup answers annotation: %w", err) + } + if !hasAnswers && !reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) { + return attrs, nil, nil + } + var answers []sourcecache.Answer + if hasAnswers { + answers = sourcecache.AnswersFromProto(answersMsg) + } + cl := sourcecache.NewContinuationLookup(answers) + attrs.SourceCache = cl + return attrs, cl, nil +} + +// continuationOutcome inspects a list handler's result when a +// ContinuationLookup was installed. +// +// Returns (askAnnos, deferred, misuseErr): +// - deferred=true: the handler deferred on recorded scopes. Respond with +// askAnnos (the SourceCacheLookupAsk), NO rows, NO next page token, +// and no failure metrics — this is a protocol turn, not a failure. +// - misuseErr != nil: the handler swallowed a deferred lookup (scopes +// were recorded but no ErrLookupDeferred propagated). Loud failure: +// silently continuing past a deferral re-asks forever and dies at the +// bounce cap in production, so it must fail here, visibly. +// - all-zero: the handler resolved normally; use its result as-is. +func continuationOutcome(cl *sourcecache.ContinuationLookup, handlerErr error) (annotations.Annotations, bool, error) { + if cl == nil { + return nil, false, nil + } + asked := cl.Asked() + if errors.Is(handlerErr, sourcecache.ErrLookupDeferred) && len(asked) > 0 { + annos := annotations.Annotations{} + annos.Update(sourcecache.AskProto(asked)) + return annos, true, nil + } + if handlerErr == nil && len(asked) > 0 { + return nil, false, status.Errorf(codes.Internal, + "connector swallowed a deferred source-cache lookup (%d scope(s) asked, no error propagated): ErrLookupDeferred must be returned — wrap with %%w, never swallow", len(asked)) + } + return nil, false, nil +} + // ResourceTargetedSyncer extends ResourceSyncer to add capabilities for directly syncing an individual resource // // Implementing this interface indicates the connector supports calling "get" on a resource @@ -129,12 +237,26 @@ func (b *builder) ListResources(ctx context.Context, request *v2.ResourcesServic Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } out, retOptions, err := rb.List(ctx, request.GetParentResourceId(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.ResourcesServiceListResourcesResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.ResourcesServiceListResourcesResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -217,7 +339,7 @@ func (b *builder) ListStaticEntitlements(ctx context.Context, request *v2.Entitl Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts := b.syncOpAttrs(request.GetActiveSyncId(), token) out, retOptions, err := rbse.StaticEntitlements(ctx, opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} @@ -265,12 +387,26 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } out, retOptions, err := rb.Entitlements(ctx, request.GetResource(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.EntitlementsServiceListEntitlementsResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.EntitlementsServiceListEntitlementsResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -322,12 +458,47 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} - out, retOptions, err := rb.Grants(ctx, request.GetResource(), opts) + reqAnnos := annotations.Annotations(request.GetAnnotations()) + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, reqAnnos) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + + typeScoped := reqAnnos.Contains(&v2.TypeScopedGrants{}) + + var out []*v2.Grant + var retOptions *resource.SyncOpResults + if typeScoped { + // Type-scoped grants call (see v2.TypeScopedGrants): the request + // annotation is the routing marker (the resource is a + // self-referential {type, type} stub to satisfy wire validation). + // Only legal against a syncer that opted in. + tsgs, tsOk := rb.(TypeScopedGrantsSyncer) + if !tsOk { + err = status.Errorf(codes.InvalidArgument, + "error: type-scoped list grants for resource type %s, but its syncer does not implement TypeScopedGrantsSyncer", rid.GetResourceType()) + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + out, retOptions, err = tsgs.GrantsForResourceType(ctx, rid.GetResourceType(), opts) + } else { + out, retOptions, err = rb.Grants(ctx, request.GetResource(), opts) + } if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.GrantsServiceListGrantsResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.GrantsServiceListGrantsResponse_builder{ List: out, Annotations: retOptions.Annotations, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go new file mode 100644 index 00000000..bd085ac0 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go @@ -0,0 +1,15 @@ +package connectorclient + +import ( + "context" + + "github.com/conductorone/baton-sdk/internal/connector" + "github.com/conductorone/baton-sdk/pkg/types" + "google.golang.org/grpc" +) + +// NewConnectorClient takes a grpc.ClientConnInterface and returns an implementation of the ConnectorClient interface. +// Note: lambda functions directly instantiate the connector client, so this function is not used in this package. +func NewConnectorClient(ctx context.Context, cc grpc.ClientConnInterface) types.ConnectorClient { + return connector.NewConnectorClient(ctx, cc) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go index bddfc7b9..4db9cf18 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go @@ -13,7 +13,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/healthcheck" "github.com/conductorone/baton-sdk/pkg/synccompactor" @@ -422,10 +422,11 @@ type runnerConfig struct { syncDifferConfig *syncDifferConfig syncCompactorConfig *syncCompactorConfig skipFullSync bool - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine workerCount int targetedSyncResourceIDs []string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string keepPreviousSyncC1ZCapable bool keepPreviousSyncC1ZEnabled bool @@ -669,7 +670,7 @@ func WithWorkerCount(workerCount int) Option { } } -func WithStorageEngine(engine dotc1z.Engine) Option { +func WithStorageEngine(engine c1zstore.Engine) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.storageEngine = engine return nil @@ -760,6 +761,13 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(ctx context.Context, cfg *runnerConfig) error { + cfg.previousSyncC1Z = previousSyncC1Z + return nil + } +} + func WithExternalResourceEntitlementFilter(entitlementId string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.externalResourceEntitlementIdFilter = entitlementId @@ -802,6 +810,25 @@ func WithKeepPreviousSyncC1ZRuntimeOptIn() Option { } } +// DeclaresPreviousSyncCapability reports whether opts include the connector +// author's replay declaration (WithKeepPreviousSyncC1Z). Used at +// command-definition time to decide whether the replay CLI flags +// (--previous-sync-c1z / --keep-previous-sync-c1z) appear in help: they are +// hidden for the overwhelming majority of connectors that don't support +// replay, and surfaced only for the ones whose author baked the capability +// into their RunConnector options. Options are applied to a scratch config; +// they are plain setters, so this is side-effect free. +func DeclaresPreviousSyncCapability(ctx context.Context, opts ...Option) bool { + cfg := &runnerConfig{} + for _, o := range opts { + if o == nil { + continue + } + _ = o(ctx, cfg) + } + return cfg.keepPreviousSyncC1ZCapable +} + func WithDiffSyncs(c1zPath string, baseSyncID string, newSyncID string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.onDemand = true @@ -973,7 +1000,15 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op wrapperOpts = append(wrapperOpts, connector.WithTargetedSyncResources(cfg.targetedSyncResourceIDs)) } - if cfg.sessionStoreEnabled { + // The parent control-plane listener serves BOTH BatonSessionService and + // BatonSourceCacheService (internal/connector.runServer). Source-cache + // replay therefore needs the listener even when the connector never uses + // sessions: without it the subprocess connector's lookup degrades to + // NoopLookup, every scope misses, and every sync runs a full cold + // enumeration despite a valid --previous-sync-c1z. Start it whenever + // replay could be in play: the author declared the capability + // (WithKeepPreviousSyncC1Z) or a previous-sync c1z was configured. + if cfg.sessionStoreEnabled || cfg.keepPreviousSyncC1ZCapable || cfg.previousSyncC1Z != "" { wrapperOpts = append(wrapperOpts, connector.WithSessionStoreEnabled()) } @@ -1081,6 +1116,7 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op tm, err = local.NewSyncer(ctx, cfg.c1zPath, local.WithTmpDir(cfg.tempDir), local.WithExternalResourceC1Z(cfg.externalResourceC1Z), + local.WithPreviousSyncC1Z(cfg.previousSyncC1Z), local.WithExternalResourceEntitlementIdFilter(cfg.externalResourceEntitlementIdFilter), local.WithTargetedSyncResources(resources), local.WithSkipEntitlementsAndGrants(cfg.skipEntitlementsAndGrants), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go index 17045488..fa1f03fe 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go @@ -39,7 +39,7 @@ var AllSyncTypes = []SyncType{ // unless the value is recognized. type StoreMetadata struct { // Engine identifies the storage backend. Values match - // dotc1z.Engine string values; using string here keeps + // c1zstore.Engine string values; using string here keeps // connectorstore from depending on dotc1z (avoids an import // cycle). // "sqlite" — original .c1z, v1 magic + zstd-compressed SQLite @@ -56,7 +56,7 @@ type StoreMetadata struct { // PayloadEncoding identifies the v3 envelope payload framing. // Empty for v1 / SQLite. Values match - // dotc1z.PayloadEncoding.String(): + // c1zstore.PayloadEncoding.String(): // "tar_zstd" — Pebble checkpoint as zstd-compressed tar // "tar" — Pebble checkpoint as uncompressed tar // "" — N/A or unset diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go index aea6de21..2845ae7f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go @@ -31,6 +31,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -73,7 +74,7 @@ type C1File struct { deferredIndexTables []tableDescriptor // Cached sync run for listConnectorObjects (avoids N+1 queries) - cachedViewSyncRun *SyncRun + cachedViewSyncRun *c1zstore.SyncRun cachedViewSyncMu sync.Mutex cachedViewSyncErr error @@ -94,22 +95,22 @@ type C1File struct { // engine is the storage engine to use for newly created files. // Reads dispatch on magic byte regardless of this value. Default // is EngineSQLite (v1 .c1z format). - engine Engine + engine c1zstore.Engine // payloadEncoding selects the v3 envelope payload framing for // Pebble-written files. Zero value = PayloadEncodingTarZstd // (default). Ignored by the SQLite engine. - payloadEncoding PayloadEncoding + payloadEncoding c1zstore.PayloadEncoding } // *C1File satisfies connectorstore.Writer (the connector-facing contract), // connectorstore.LatestFinishedSyncIDFetcher (narrow optional capability -// added in PR #774), and dotc1z.C1ZStore (the internal sync-pipeline +// added in PR #774), and c1zstore.Store (the internal sync-pipeline // contract asserted in c1file_store.go alongside the sub-store assertions). var ( _ connectorstore.Writer = (*C1File)(nil) _ connectorstore.LatestFinishedSyncIDFetcher = (*C1File)(nil) - _ C1ZStore = (*C1File)(nil) + _ c1zstore.Store = (*C1File)(nil) ) type C1FOption func(*C1File) @@ -210,7 +211,7 @@ func WithC1FSyncCountLimit(limit int) C1FOption { // Engine selection only affects newly created files. Existing files // dispatch on their magic byte; readers handle both v1 and v3 // regardless of this option. -func WithC1FEngine(engine Engine) C1FOption { +func WithC1FEngine(engine c1zstore.Engine) C1FOption { return func(o *C1File) { o.engine = engine } @@ -218,7 +219,7 @@ func WithC1FEngine(engine Engine) C1FOption { // WithC1FPayloadEncoding selects the v3 envelope payload encoding // (TAR_ZSTD default, TAR uncompressed). No-op for SQLite engines. -func WithC1FPayloadEncoding(enc PayloadEncoding) C1FOption { +func WithC1FPayloadEncoding(enc c1zstore.PayloadEncoding) C1FOption { return func(o *C1File) { o.payloadEncoding = enc } @@ -293,7 +294,7 @@ func NewC1File(ctx context.Context, dbFilePath string, opts ...C1FOption) (*C1Fi // engine manages its own storage and does not use those indexes, so bulk // load does not apply there. Make the combination an explicit, logged // no-op rather than leaving it silently unspecified. - if c1File.bulkLoad && c1File.engine == EnginePebble { + if c1File.bulkLoad && c1File.engine == c1zstore.EnginePebble { l.Info("new-c1-file: bulk load ignored for the pebble engine; the deferred-index optimization applies only to the sqlite engine") c1File.bulkLoad = false } @@ -311,7 +312,7 @@ func NewC1File(ctx context.Context, dbFilePath string, opts ...C1FOption) (*C1Fi // Normalize the engine zero value so downstream switch/if-eq // checks treat an unset engine as EngineSQLite. if c1File.engine == "" { - c1File.engine = EngineSQLite + c1File.engine = c1zstore.EngineSQLite } err = c1File.validateDb(ctx) @@ -341,13 +342,13 @@ type c1zOptions struct { // engine is the storage engine to use for newly created files. // Reads dispatch on magic byte regardless. Default EngineSQLite. - engine Engine + engine c1zstore.Engine // payloadEncoding controls the v3 envelope payload framing. Only // honored when engine == EnginePebble (the v3 path). Allowed // values: PayloadEncodingTarZstd (default), PayloadEncodingTar. // Zero value means PayloadEncodingTarZstd. - payloadEncoding PayloadEncoding + payloadEncoding c1zstore.PayloadEncoding // decoderPool optionally scopes v3 payload-decoder reuse to the // caller's operation. See WithDecoderPool. @@ -430,7 +431,7 @@ func WithSyncLimit(limit int) C1ZOption { // // Reading existing files dispatches on the file's magic byte and is // independent of this option. -func WithEngine(engine Engine) C1ZOption { +func WithEngine(engine c1zstore.Engine) C1ZOption { return func(o *c1zOptions) { o.engine = engine } @@ -465,7 +466,7 @@ func WithBulkLoad(enabled bool) C1ZOption { // // No-op for SQLite engines; the encoding selector applies only to // the v3 envelope written by Pebble. -func WithPayloadEncoding(enc PayloadEncoding) C1ZOption { +func WithPayloadEncoding(enc c1zstore.PayloadEncoding) C1ZOption { return func(o *c1zOptions) { o.payloadEncoding = enc } @@ -491,8 +492,11 @@ func NewC1ZFile(ctx context.Context, outputFilePath string, opts ...C1ZOption) ( return nil, err } - if options.engine == EnginePebble && !options.readOnly { - err = fmt.Errorf("new-c1z-file: %s is a v1/sqlite c1z and engine %q was requested; NewC1ZFile cannot return a *C1File for it — open with NewStore to convert", outputFilePath, EnginePebble) + if options.engine == c1zstore.EnginePebble && !options.readOnly { + err = fmt.Errorf( + "new-c1z-file: %s is a v1/sqlite c1z and engine %q was requested; "+ + "NewC1ZFile cannot return a *C1File for it — open with NewStore to convert", + outputFilePath, c1zstore.EnginePebble) return nil, err } @@ -535,7 +539,7 @@ func NewC1ZFile(ctx context.Context, outputFilePath string, opts ...C1ZOption) ( if options.engine != "" { c1fopts = append(c1fopts, WithC1FEngine(options.engine)) } - if options.payloadEncoding != PayloadEncodingUnspecified { + if options.payloadEncoding != c1zstore.PayloadEncodingUnspecified { c1fopts = append(c1fopts, WithC1FPayloadEncoding(options.payloadEncoding)) } @@ -551,7 +555,7 @@ func NewC1ZFile(ctx context.Context, outputFilePath string, opts ...C1ZOption) ( func cleanupDbDir(dbFilePath string, err error) error { // Stat dbFilePath to make sure it's a file, not a directory. - stat, statErr := os.Stat(dbFilePath) + stat, statErr := os.Stat(dbFilePath) // #nosec G703 -- c1z db path is caller-controlled by API design. if statErr != nil { if errors.Is(statErr, os.ErrNotExist) { // If the file doesn't exist, we can't clean up the directory. @@ -564,7 +568,7 @@ func cleanupDbDir(dbFilePath string, err error) error { return errors.Join(err, fmt.Errorf("cleanupDbDir: dbFilePath %s is a directory, not a file: %w", dbFilePath, statErr)) } - cleanupErr := os.RemoveAll(filepath.Dir(dbFilePath)) + cleanupErr := os.RemoveAll(filepath.Dir(dbFilePath)) // #nosec G703 -- removing temp db directory derived from caller-controlled c1z path. if cleanupErr != nil { err = errors.Join(err, cleanupErr) } @@ -1420,7 +1424,7 @@ func (c *C1File) OutputFilepath() (string, error) { func (c *C1File) Metadata() connectorstore.StoreMetadata { engine := c.engine if engine == "" { - engine = EngineSQLite + engine = c1zstore.EngineSQLite } return connectorstore.StoreMetadata{ Engine: string(engine), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go index 226a1783..dbc4cb35 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go @@ -17,21 +17,21 @@ import ( // wrapper structs satisfy each sub-interface. These assertions catch // signature drift at build time rather than at the first runtime call. var ( - _ C1ZStore = (*C1File)(nil) - _ GrantStore = c1FileGrantStore{} - _ SyncMeta = c1FileSyncMeta{} - _ FileOps = c1FileFileOps{} - _ SessionStore = c1FileSessionStore{} + _ c1zstore.Store = (*C1File)(nil) + _ c1zstore.GrantStore = c1FileGrantStore{} + _ c1zstore.SyncMeta = c1FileSyncMeta{} + _ c1zstore.FileOps = c1FileFileOps{} + _ SessionStore = c1FileSessionStore{} ) // Grants returns the grant-store slice of this c1z. -func (c *C1File) Grants() GrantStore { return c1FileGrantStore{c} } +func (c *C1File) Grants() c1zstore.GrantStore { return c1FileGrantStore{c} } // SyncMeta returns the sync-metadata slice of this c1z. -func (c *C1File) SyncMeta() SyncMeta { return c1FileSyncMeta{c} } +func (c *C1File) SyncMeta() c1zstore.SyncMeta { return c1FileSyncMeta{c} } // FileOps returns the file-operations slice of this c1z. -func (c *C1File) FileOps() FileOps { return c1FileFileOps{c} } +func (c *C1File) FileOps() c1zstore.FileOps { return c1FileFileOps{c} } // SessionStore returns the session-store slice of this c1z. func (c *C1File) SessionStore() sessions.SessionStore { return c1FileSessionStore{c} } @@ -97,7 +97,7 @@ func (c *C1File) StoreExpandedGrants(ctx context.Context, grants ...*v2.Grant) e // PendingExpansionPage implements GrantStore. Thin wrapper over // listExpandableGrantsInternal(Mode: ExpansionNeedsOnly) that reshapes // the internal row struct into the exported PendingExpansion shape. -func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken string) ([]PendingExpansion, string, error) { +func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken string) ([]c1zstore.PendingExpansion, string, error) { defs, nextPageToken, err := g.c.listExpandableGrantsInternal(ctx, grantListOptions{ Mode: grantListModeExpansionNeedsOnly, PageToken: pageToken, @@ -105,12 +105,12 @@ func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken st if err != nil { return nil, "", err } - out := make([]PendingExpansion, 0, len(defs)) + out := make([]c1zstore.PendingExpansion, 0, len(defs)) for _, def := range defs { if def == nil { continue } - out = append(out, PendingExpansion{ + out = append(out, c1zstore.PendingExpansion{ GrantExternalID: def.GrantExternalID, TargetEntitlementID: def.TargetEntitlementID, PrincipalResourceTypeID: def.PrincipalResourceTypeID, @@ -136,17 +136,17 @@ func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken st // walks terminate promptly when the caller's deadline/cancel fires; // rows within a single page are still delivered (they are already in // memory), so cancellation responsiveness is page-grained, not row-grained. -func (g c1FileGrantStore) PendingExpansion(ctx context.Context) iter.Seq2[PendingExpansion, error] { - return func(yield func(PendingExpansion, error) bool) { +func (g c1FileGrantStore) PendingExpansion(ctx context.Context) iter.Seq2[c1zstore.PendingExpansion, error] { + return func(yield func(c1zstore.PendingExpansion, error) bool) { pageToken := "" for { if err := ctx.Err(); err != nil { - _ = yield(PendingExpansion{}, err) + _ = yield(c1zstore.PendingExpansion{}, err) return } page, nextPageToken, err := g.PendingExpansionPage(ctx, pageToken) if err != nil { - _ = yield(PendingExpansion{}, err) + _ = yield(c1zstore.PendingExpansion{}, err) return } for _, pe := range page { @@ -172,7 +172,7 @@ func (g c1FileGrantStore) ListWithAnnotationsForResourcePage( syncID string, pageToken string, pageSize uint32, -) ([]GrantAnnotation, string, error) { +) ([]c1zstore.GrantAnnotation, string, error) { resp, err := g.c.listGrantsWithExpansionInternal(ctx, grantListOptions{ Mode: grantListModePayloadWithExpansion, Resource: resource, @@ -189,13 +189,13 @@ func (g c1FileGrantStore) ListWithAnnotationsForResourcePage( // grantAnnotationRowsFromInternal converts the internal row shape into // the exported GrantAnnotation shape, unifying the code path between // ListWithAnnotationsPage and ListWithAnnotationsForResourcePage. -func grantAnnotationRowsFromInternal(rows []*internalGrantRow) []GrantAnnotation { - out := make([]GrantAnnotation, 0, len(rows)) +func grantAnnotationRowsFromInternal(rows []*internalGrantRow) []c1zstore.GrantAnnotation { + out := make([]c1zstore.GrantAnnotation, 0, len(rows)) for _, row := range rows { if row == nil { continue } - ga := GrantAnnotation{ + ga := c1zstore.GrantAnnotation{ Grant: row.Grant, GrantExternalID: row.Grant.GetId(), TargetEntitlementID: row.Grant.GetEntitlement().GetId(), @@ -223,7 +223,7 @@ func grantAnnotationRowsFromInternal(rows []*internalGrantRow) []GrantAnnotation // from the underlying grant proto, regardless of whether the grant has // an expansion annotation, so callers don't need to branch on // Annotation-nil to get identity. -func (g c1FileGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]GrantAnnotation, string, error) { +func (g c1FileGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]c1zstore.GrantAnnotation, string, error) { resp, err := g.c.listGrantsWithExpansionInternal(ctx, grantListOptions{ Mode: grantListModePayloadWithExpansion, PageToken: pageToken, @@ -237,17 +237,17 @@ func (g c1FileGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken // ListWithAnnotations implements GrantStore. Convenience iterator that // walks every page via ListWithAnnotationsPage. Cancellation behavior is // identical to PendingExpansion (page-grained). -func (g c1FileGrantStore) ListWithAnnotations(ctx context.Context) iter.Seq2[GrantAnnotation, error] { - return func(yield func(GrantAnnotation, error) bool) { +func (g c1FileGrantStore) ListWithAnnotations(ctx context.Context) iter.Seq2[c1zstore.GrantAnnotation, error] { + return func(yield func(c1zstore.GrantAnnotation, error) bool) { pageToken := "" for { if err := ctx.Err(); err != nil { - _ = yield(GrantAnnotation{}, err) + _ = yield(c1zstore.GrantAnnotation{}, err) return } page, nextPageToken, err := g.ListWithAnnotationsPage(ctx, pageToken) if err != nil { - _ = yield(GrantAnnotation{}, err) + _ = yield(c1zstore.GrantAnnotation{}, err) return } for _, ga := range page { @@ -276,7 +276,7 @@ func (s c1FileSyncMeta) MarkSyncSupportsDiff(ctx context.Context, syncID string) // LatestFullSync implements SyncMeta. Returns the most-recent finished // SyncTypeFull run, or nil if none. -func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*SyncRun, error) { +func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*c1zstore.SyncRun, error) { run, err := s.c.getFinishedSync(ctx, 0, connectorstore.SyncTypeFull) if err != nil { return nil, err @@ -286,7 +286,7 @@ func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*SyncRun, error) { // LatestFinishedSyncOfAnyType implements SyncMeta. Returns the most-recent // finished sync of any type (including diff types), or nil if none. -func (s c1FileSyncMeta) LatestFinishedSyncOfAnyType(ctx context.Context) (*SyncRun, error) { +func (s c1FileSyncMeta) LatestFinishedSyncOfAnyType(ctx context.Context) (*c1zstore.SyncRun, error) { run, err := s.c.getFinishedSync(ctx, 0, connectorstore.SyncTypeAny) if err != nil { return nil, err @@ -314,7 +314,7 @@ type c1FileFileOps struct{ c *C1File } // CloneSync implements FileOps. Translates the engine-neutral // CloneSyncOptions into the SQLite-specific C1FOptions applied to the // destination file. -func (f c1FileFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { +func (f c1FileFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { cloneOpts := c1zstore.NewCloneSyncOptions(opts...) var c1fOpts []C1FOption if cloneOpts.TmpDir != "" { @@ -326,7 +326,7 @@ func (f c1FileFileOps) CloneSync(ctx context.Context, outPath string, syncID str // CopyIsolateSync implements FileOps. Translates the engine-neutral // CloneSyncOptions into the SQLite-specific C1FOptions applied to the // destination file. -func (f c1FileFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { +func (f c1FileFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { cloneOpts := c1zstore.NewCloneSyncOptions(opts...) var c1fOpts []C1FOption if cloneOpts.TmpDir != "" { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go index 5ab9d50f..8aa50210 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go @@ -7,9 +7,8 @@ // pkg/connectorstore. Storage engines (pkg/dotc1z's SQLite C1File, // pkg/dotc1z/engine/pebble's Adapter) implement these interfaces without // importing pkg/dotc1z, which lets dotc1z import the engines and register -// them statically. pkg/dotc1z re-exports every type here under its -// historical name (dotc1z.C1ZStore = c1zstore.Store, etc.), so callers -// outside the engine packages can keep using the dotc1z names. +// them statically. Callers reference these types directly through this +// package (c1zstore.Store, c1zstore.Engine, etc.). package c1zstore import ( diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/engine.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/engine.go index 8dd1c1b5..833fa7cd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/engine.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/engine.go @@ -27,14 +27,28 @@ const ( EnginePebble Engine = "pebble" // PebbleManifestEngine is the engine name written into the v3 - // envelope manifest for the single-sync (sync_id-less) keyspace. - // It is deliberately NOT "pebble": the manifest engine name is the - // one field readers validate at dispatch, so a name pre-single-sync - // SDKs don't recognize makes them fail loudly ("engine not - // available: pebble2") instead of opening the file and reading its - // keys as empty. Readers that understand the single-sync layout map - // this name back to EnginePebble. - PebbleManifestEngine = "pebble2" + // envelope manifest. It is deliberately NOT "pebble": the manifest + // engine name is the one field readers validate at dispatch, so a + // name older SDKs don't recognize makes them fail loudly ("engine + // not available: pebble3") instead of opening the file and reading + // its keyspaces as empty. It MUST be bumped on every incompatible + // keyspace layout change: + // + // - "pebble2" fenced the single-sync (sync_id-less) keyspace from + // pre-single-sync readers; + // - "pebble3" fences the structural-identity keyspace (identity- + // keyed grant/entitlement primaries, retired by_entitlement + // index family) from pebble2-era readers, which would otherwise + // scan the retired index ranges and silently report zero rows. + // + // Readers that understand the current layout map this name (and the + // legacy names below, whose interiors the id-index migration + // re-keys on a writable open) back to EnginePebble. + PebbleManifestEngine = "pebble3" + + // PebbleManifestEngineV2 is the retired manifest name for the + // single-sync external-id keyspace. Accepted on read; never written. + PebbleManifestEngineV2 = "pebble2" ) // PayloadEncoding selects the v3 envelope payload framing. Only the diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go index e9aa16b6..8f1b07e8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go @@ -9,7 +9,7 @@ import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" // SelectSyncsToDelete applies the SDK retention policy to a snapshot of sync // runs and returns the IDs whose data should be deleted. See // c1zstore.SelectSyncsToDelete for the policy details. -func SelectSyncsToDelete(candidates []SyncRun, currentSyncID string, syncLimit int) []string { +func SelectSyncsToDelete(candidates []c1zstore.SyncRun, currentSyncID string, syncLimit int) []string { return c1zstore.SelectSyncsToDelete(candidates, currentSyncID, syncLimit) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go index 429df6b7..9c34ed97 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" @@ -195,8 +196,8 @@ func (c *C1File) SnapshotTo(ctx context.Context, outPath string, opts ...C1FOpti return err } - if c.engine == EnginePebble { - err = fmt.Errorf("snapshot-to: unsupported for the %q engine; it manages its own storage", EnginePebble) + if c.engine == c1zstore.EnginePebble { + err = fmt.Errorf("snapshot-to: unsupported for the %q engine; it manages its own storage", c1zstore.EnginePebble) return err } if c.readOnly { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go index 7eab5047..c6d5086b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go @@ -9,6 +9,7 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -22,7 +23,7 @@ type pebbleOpenOptions struct { skipCleanup bool skipVacuum bool v2GrantsWriter bool - payloadEncoding PayloadEncoding + payloadEncoding c1zstore.PayloadEncoding } func pebbleOpenOptionsFromC1Z(options *c1zOptions) pebbleOpenOptions { @@ -49,23 +50,23 @@ func convertSQLiteC1ZToPebble(ctx context.Context, src *C1File, outPath string) defer func() { uotel.EndSpanWithError(span, err) }() tmpOut := outPath + ".pebble-convert.tmp" - if removeErr := os.Remove(tmpOut); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + if removeErr := os.Remove(tmpOut); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { // #nosec G703 -- conversion output path is caller-controlled by API design. err = removeErr return err } if _, err = src.ToPebble(ctx, tmpOut, ""); err != nil { - _ = os.Remove(tmpOut) + _ = os.Remove(tmpOut) // #nosec G703 -- cleanup of caller-selected conversion temp output. return err } if err = src.closeWithoutSave(ctx); err != nil { - _ = os.Remove(tmpOut) + _ = os.Remove(tmpOut) // #nosec G703 -- cleanup of caller-selected conversion temp output. return fmt.Errorf("convert-open: close sqlite source: %w", err) } - if err = os.Rename(tmpOut, outPath); err != nil { - _ = os.Remove(tmpOut) + if err = os.Rename(tmpOut, outPath); err != nil { // #nosec G703 -- conversion output path is caller-controlled by API design. + _ = os.Remove(tmpOut) // #nosec G703 -- cleanup of caller-selected conversion temp output. return fmt.Errorf("convert-open: replace output c1z: %w", err) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go index b9e962dd..b2c38511 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go @@ -21,6 +21,7 @@ import ( v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" ) // Adapter wraps an *Engine and implements connectorstore.Writer @@ -297,6 +298,54 @@ func (a *Adapter) EndSync(ctx context.Context) error { if err != nil { return err } + // The sync's writes are done. From here to save/close the store only + // runs the deferred index build, the stats sidecar, and the durability + // flush — automatic compactions in that window are incremental + // level-by-level rewrites that compete with those phases for CPU and IO, + // so stop granting new ones. The deferred index build below consolidates + // the grant keyspace itself (its scan is teed into a flat rebuild that + // replaces the range via IngestAndExcise), so the saved artifact ships + // with near-zero compaction debt without running the compactor. + // Seal BEFORE finalize, not after: the seal must cover the deferred + // index build and the pending-marker clear, or a straggler record + // writer that was blocked on writeMu could commit in the gap between + // them — a row present in the primary keyspace but permanently missing + // from by_principal, in the saved artifact. Finalize's own steps run + // on AllowSealed paths; sync-run metadata stamps remain allowed. + // Sealing also pauses compactions for the EndSync-to-close window. + a.engine.seal() + if err := a.engine.endSyncFinalize(ctx, existing); err != nil { + // On failure the sync stays bound and the caller may keep writing + // (or retry EndSync later): leave the sealed state and resume + // compactions, or L0 would accumulate until pebble stalls writes at + // L0StopWritesThreshold with nothing left to resume the scheduler. + a.engine.unseal() + return err + } + a.current = syncRunState{} + return nil +} + +// endSyncFinalize runs the sealed tail of EndSync: the deferred index +// build, the ended_at stamp, the stats sidecar, and the durability flush. +// Runs with the engine SEALED (see EndSync) — every write below goes +// through an AllowSealed path. Split out so EndSync can unseal on failure. +func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord) error { + // Build the deferred by_principal index BEFORE stamping ended_at (an + // interrupted build must leave the sync visibly unfinished so a resume + // re-runs EndSync and the rebuild — the pending marker is durable, see + // markDeferredIdxPending) and BEFORE the stats sidecar (the build's + // full grant scan also accumulates the grant portion of the stats via + // stashDeferredGrantStats, letting PersistSyncStats skip a second + // O(grants) pass over the keyspace). + if e.deferredIdxPending.Load() { + if err := e.BuildDeferredGrantIndexes(ctx); err != nil { + return fmt.Errorf("EndSync: build deferred grant indexes: %w", err) + } + if err := e.clearDeferredIdxPending(); err != nil { + return fmt.Errorf("EndSync: clear deferred index marker: %w", err) + } + } updated := v3.SyncRunRecord_builder{ SyncId: existing.GetSyncId(), Type: existing.GetType(), @@ -305,7 +354,7 @@ func (a *Adapter) EndSync(ctx context.Context) error { EndedAt: timestamppb.Now(), SyncToken: existing.GetSyncToken(), }.Build() - if err := a.engine.PutSyncRunRecord(ctx, updated); err != nil { + if err := e.PutSyncRunRecord(ctx, updated); err != nil { return err } // Populate the stats sidecar BEFORE the durability flush. Stats @@ -316,7 +365,7 @@ func (a *Adapter) EndSync(ctx context.Context) error { // framework will backfill next time the file opens. We log a // warning so the failure is visible in production telemetry but // don't fail the sync end on stats-sidecar trouble. - if err := a.engine.PersistSyncStats(ctx, existing.GetSyncId()); err != nil { + if err := e.PersistSyncStats(ctx, existing.GetSyncId()); err != nil { ctxzap.Extract(ctx).Warn("pebble: persist sync stats sidecar failed; Stats() will fall back to O(N) iteration until the next Open backfills it", zap.String("sync_id", existing.GetSyncId()), zap.Error(err), @@ -325,11 +374,7 @@ func (a *Adapter) EndSync(ctx context.Context) error { // Single flush + WAL fsync at sync end. This is the durability // boundary — counterpart to MarkFreshSync at StartNewSync. After // this returns, all writes from the sync are on disk. - if err := a.engine.EndFreshSync(ctx); err != nil { - return err - } - a.current = syncRunState{} - return nil + return e.EndFreshSync(ctx) } // === writes === @@ -356,12 +401,27 @@ func (a *Adapter) PutGrants(ctx context.Context, grants ...*v2.Grant) error { return ErrNoCurrentSync } records := translateGrants(syncID, grants) + stampSourceScope(ctx, records, func(r *v3.GrantRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutGrantRecords(ctx, records...); err != nil { return fmt.Errorf("PutGrants: %w", err) } return nil } +// stampSourceScope stamps the source-cache scope hash carried by ctx +// (sourcecache.WithScope) onto freshly translated records. The syncer +// sets the scope around a page's store writes when the page carried a +// SourceCacheScope annotation; everything else writes unstamped rows. +func stampSourceScope[T any](ctx context.Context, records []T, set func(T, string)) { + scope := sourcecache.ScopeFromContext(ctx) + if scope == "" { + return + } + for _, r := range records { + set(r, scope) + } +} + // UnsafePutUniqueGrants writes grants on the trusted-import path: records // are encoded in parallel and written unconditionally, with no read-before-write // and no dedup pass. Do not use it for live connector output. The destination @@ -519,6 +579,7 @@ func (a *Adapter) PutResources(ctx context.Context, resources ...*v2.Resource) e } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.ResourceRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutResourceRecords(ctx, records...); err != nil { return fmt.Errorf("PutResources: %w", err) } @@ -546,13 +607,16 @@ func (a *Adapter) PutEntitlements(ctx context.Context, entitlements ...*v2.Entit } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.EntitlementRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutEntitlementRecords(ctx, records...); err != nil { return fmt.Errorf("PutEntitlements: %w", err) } return nil } -// DeleteGrant removes a grant by id. +// DeleteGrant removes a grant by its raw public id, resolved through the +// bare-id lookup edge. Callers holding the full grant should prefer +// DeleteGrantByRefs, which needs no id-string resolution. func (a *Adapter) DeleteGrant(ctx context.Context, grantID string) error { syncID := a.currentSyncID() if syncID == "" { @@ -561,6 +625,25 @@ func (a *Adapter) DeleteGrant(ctx context.Context, grantID string) error { return a.engine.DeleteGrantRecord(ctx, grantID) } +// DeleteGrantByRefs removes a grant addressed by the structured refs of the +// supplied v2 grant — the exact delete path, no lossy id string involved. +// Incomplete refs are an error, never a fallback to bare-id resolution: +// this is a sync-internal surface, and string resolution is reserved for +// interactive/CLI edges (see lookup.go). A grant whose refs cannot derive +// an identity could not have been stored in the first place, so there is +// nothing a string could correctly address here. +func (a *Adapter) DeleteGrantByRefs(ctx context.Context, grant *v2.Grant) error { + syncID := a.currentSyncID() + if syncID == "" { + return ErrNoCurrentSync + } + rec := V2GrantToV3(syncID, grant) + if _, err := grantIdentityFromRecord(rec); err != nil { + return fmt.Errorf("DeleteGrantByRefs: grant %q: %w", grant.GetId(), err) + } + return a.engine.DeleteGrantByIdentityRefs(ctx, rec) +} + // PutAsset writes a single asset row. assetRef carries the // (resource_type, resource_id) pair we use as the external_id — // joined with a "/" separator since the engine's AssetRecord PK is @@ -637,7 +720,7 @@ func (a *Adapter) GetAsset(ctx context.Context, req *v2.AssetServiceGetAssetRequ // - page_size == 0 || page_size > MaxPageSize → DefaultPageSize (10000) // - page_token is opaque base64; pass nextPageToken back verbatim // - filter by req.Resource — the entitlement-side resource of each -// grant — when set; uses the by_entitlement_resource index. This +// grant — when set; uses primary grant entitlement-resource prefixes. This // matches SQLite's `listGrantsGeneric` which filters on // grants.resource_id / resource_type_id (the entitlement's // resource columns). Callers who want to filter by principal @@ -902,7 +985,7 @@ func (a *Adapter) CurrentDBSizeBytes() (int64, error) { // // Strings are inlined rather than referencing dotc1z constants // because this subpackage is imported by dotc1z, so the reverse -// import would cycle. The values match dotc1z.EnginePebble.String() +// import would cycle. The values match c1zstore.EnginePebble.String() // and dotc1z.C1ZFormatV3.String() — see connectorstore.StoreMetadata // docs for the canonical value list. func (a *Adapter) Metadata() connectorstore.StoreMetadata { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go index 714cb40b..fe19c1f2 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go @@ -3,13 +3,16 @@ package pebble import ( "context" "errors" + "fmt" "iter" "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/types/known/anypb" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" ) // Grants returns the GrantStore implementation backed by the Pebble @@ -29,6 +32,14 @@ type pebbleGrantStore struct { a *Adapter } +var expandedGrantImmutableAnnotationAny = func() *anypb.Any { + a, err := anypb.New(&v2.GrantImmutable{}) + if err != nil { + panic(fmt.Errorf("pebble: marshal GrantImmutable annotation: %w", err)) + } + return a +}() + // StoreExpandedGrants writes a batch of grants that have already // had their expansion annotations consumed by the expander. Matches // the SQLite C1File.Grants().StoreExpandedGrants contract: the @@ -72,6 +83,127 @@ func (g pebbleGrantStore) StoreExpandedGrants(ctx context.Context, grants ...*v2 // equivalent to V2GrantToV3 and (like it) leaves discovered_at unset: // PutExpandedGrantRecords stamps/preserves it. The returned pointers alias // the arena, which outlives this call's use of `merged`. + merged := g.translateExpanded(syncID, grants) + return g.a.engine.PutExpandedGrantRecords(ctx, merged) +} + +// StoreNewExpandedGrants is the fast path for synthesized expanded grants. The +// caller guarantees no existing grant has the same structured identity, so Pebble +// can skip the read-before-write Get used to preserve side-state and clean stale +// indexes for updates. +func (g pebbleGrantStore) StoreNewExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { + syncID := g.a.currentSyncID() + if syncID == "" { + return ErrNoCurrentSync + } + if len(grants) == 0 { + return nil + } + merged := g.translateExpanded(syncID, grants) + return g.a.engine.PutSynthesizedGrantRecords(ctx, merged) +} + +// appendSynthesizedGrantRecords translates one destination's synthesized +// contributions into engine records, appending to records. Shared by the +// per-destination write path and the layer-scoped layer session. +func appendSynthesizedGrantRecords(records []synthesizedGrantRecord, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) ([]synthesizedGrantRecord, error) { + if len(principals) != len(sources) { + return records, fmt.Errorf("store new expanded grant contributions: principals/sources length mismatch") + } + entRef := entitlementToRef(dest) + if entRef == nil { + return records, fmt.Errorf("store new expanded grant contributions: entitlement is nil") + } + entID := entitlementIdentityFromParts(entRef.GetResourceTypeId(), entRef.GetResourceId(), entRef.GetEntitlementId()) + for i, principal := range principals { + if principal == nil || principal.GetResourceTypeId() == "" || principal.GetResourceId() == "" { + continue + } + if len(sources[i]) == 0 { + return records, fmt.Errorf("store new expanded grant contributions: empty sources") + } + id := grantIdentity{ + entitlement: entID, + principalTypeID: principal.GetResourceTypeId(), + principalID: principal.GetResourceId(), + } + // The public external id (the legacy concat) is not materialized + // here: the engine encoders build it directly on the wire from the + // refs (appendSynthGrantExternalIDWire). + records = append(records, synthesizedGrantRecord{ + id: id, + entitlement: entRef, + principal: principal, + sources: sources[i], + }) + } + return records, nil +} + +func (g pebbleGrantStore) StoreNewExpandedGrantContributions(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error { + if g.a.currentSyncID() == "" { + return ErrNoCurrentSync + } + if len(principals) == 0 { + return nil + } + // Pooled: the engine's Put/ingest paths copy everything they need out of + // records before returning, so the backing array is safe to recycle. + recordsPtr := getSynthRecords() + records := *recordsPtr + defer func() { + *recordsPtr = records + putSynthRecords(recordsPtr) + }() + records, err := appendSynthesizedGrantRecords(records, dest, principals, sources) + if err != nil { + return err + } + return g.a.engine.PutSynthesizedGrantContributions(ctx, records) +} + +// BeginExpandedGrantLayer opens a layer-scoped synthesized-grant layer session +// on the engine. Returns false when the engine cannot serve one (e.g. the +// by_principal index is not deferred); callers fall back to +// StoreNewExpandedGrantContributions. +func (g pebbleGrantStore) BeginExpandedGrantLayer(ctx context.Context) (bool, error) { + if g.a.currentSyncID() == "" { + return false, ErrNoCurrentSync + } + return g.a.engine.BeginSynthesizedGrantLayer(ctx) +} + +// AddExpandedGrantLayerContributions streams one destination's synthesized +// contributions into the open layer session. Rows become readable when +// FinishExpandedGrantLayer publishes the layer. +func (g pebbleGrantStore) AddExpandedGrantLayerContributions(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error { + if len(principals) == 0 { + return nil + } + // Pooled: the engine encodes everything it needs out of records before + // returning, so the backing array is safe to recycle. + recordsPtr := getSynthRecords() + records := *recordsPtr + defer func() { + *recordsPtr = records + putSynthRecords(recordsPtr) + }() + records, err := appendSynthesizedGrantRecords(records, dest, principals, sources) + if err != nil { + return err + } + return g.a.engine.AddSynthesizedGrantLayerContributions(ctx, records) +} + +func (g pebbleGrantStore) FinishExpandedGrantLayer(ctx context.Context) error { + return g.a.engine.FinishSynthesizedGrantLayer(ctx) +} + +func (g pebbleGrantStore) AbortExpandedGrantLayer(ctx context.Context) error { + return g.a.engine.AbortSynthesizedGrantLayer(ctx) +} + +func (g pebbleGrantStore) translateExpanded(syncID string, grants []*v2.Grant) []*v3.GrantRecord { merged := make([]*v3.GrantRecord, 0, len(grants)) arena := newGrantTranslateArena(len(grants)) for _, gr := range grants { @@ -88,9 +220,13 @@ func (g pebbleGrantStore) StoreExpandedGrants(ctx context.Context, grants ...*v2 // because the caller left a residual GrantExpandable annotation. newRec.SetExpansion(nil) newRec.SetNeedsExpansion(false) + // Same shape for the source-cache scope stamp: existing records get + // their prior stamp restored in PutExpandedGrantRecords; brand-new + // expander-derived rows are never part of a source scope. + newRec.SetSourceScopeHash("") merged = append(merged, newRec) } - return g.a.engine.PutExpandedGrantRecords(ctx, merged) + return merged } // PendingExpansionPage returns the next page of grants whose @@ -122,11 +258,15 @@ func (g pebbleGrantStore) PendingExpansionPage(ctx context.Context, pageToken st continue } exp := rec.GetExpansion() + ent := rec.GetEntitlement() + princ := rec.GetPrincipal() + // Raw pass-through: refs and expandable ids are the connector's own + // strings; the graph, sources maps, and stored refs all share them. out = append(out, c1zstore.PendingExpansion{ GrantExternalID: rec.GetExternalId(), - TargetEntitlementID: rec.GetEntitlement().GetEntitlementId(), - PrincipalResourceTypeID: rec.GetPrincipal().GetResourceTypeId(), - PrincipalResourceID: rec.GetPrincipal().GetResourceId(), + TargetEntitlementID: ent.GetEntitlementId(), + PrincipalResourceTypeID: princ.GetResourceTypeId(), + PrincipalResourceID: princ.GetResourceId(), Annotation: v2.GrantExpandable_builder{ EntitlementIds: exp.GetEntitlementIds(), Shallow: exp.GetShallow(), @@ -181,13 +321,15 @@ func (g pebbleGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken } rows := make([]c1zstore.GrantAnnotation, 0, len(records)) for _, rec := range records { + ent := rec.GetEntitlement() + princ := rec.GetPrincipal() rows = append(rows, c1zstore.GrantAnnotation{ Grant: V3GrantToV2(rec), Annotation: expansionRecordToV2(rec.GetExpansion()), GrantExternalID: rec.GetExternalId(), - TargetEntitlementID: rec.GetEntitlement().GetEntitlementId(), - PrincipalResourceTypeID: rec.GetPrincipal().GetResourceTypeId(), - PrincipalResourceID: rec.GetPrincipal().GetResourceId(), + TargetEntitlementID: ent.GetEntitlementId(), + PrincipalResourceTypeID: princ.GetResourceTypeId(), + PrincipalResourceID: princ.GetResourceId(), NeedsExpansion: rec.GetNeedsExpansion(), }) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_reader.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_reader.go index 65b3fdee..624a9ecf 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_reader.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_reader.go @@ -166,12 +166,15 @@ func (a *Adapter) ListEntitlementsByIds( }.Build(), nil } -// GrantsForEntitlementPrincipalSorted reports that ListGrantsForEntitlement +// GrantsForEntitlementPrincipalSorted reports whether ListGrantsForEntitlement // yields grants in non-decreasing principal (resource_type, resource) order. -// The by_entitlement index is keyed -// entitlement_id|principal_rt|principal_id|external_id, and pagination walks it -// in key order, so consumers can group by principal without buffering and -// sorting the whole entitlement. +// True under the structured key layout: the primary grant key ends in the +// tuple-encoded (principal_rt, principal_id) components and the tuple codec +// is order-preserving, so a per-entitlement primary prefix scan IS a +// principal-ordered scan. This gates the topological-merge expansion path; +// streamingPrincipalGroupStream additionally fails loudly if the order +// invariant is ever violated, and the differential/parity suites compare +// the sorted and unsorted evaluators against SQLite. func (a *Adapter) GrantsForEntitlementPrincipalSorted() bool { return true } // ListGrantsForEntitlement paginates grants on a specific @@ -193,6 +196,15 @@ func (a *Adapter) ListGrantsForEntitlement( if ent == nil || ent.GetId() == "" { return nil, errors.New("ListGrantsForEntitlement: missing entitlement id") } + entIdentity, err := a.entitlementIdentityForRequest(ctx, ent) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + // Unknown entitlement → no grants, matching the legacy + // empty-prefix-scan semantics. + return reader_v2.GrantsReaderServiceListGrantsForEntitlementResponse_builder{}.Build(), nil + } + return nil, err + } limit := clampPageSize(req.GetPageSize()) cursor := req.GetPageToken() @@ -205,17 +217,16 @@ func (a *Adapter) ListGrantsForEntitlement( rtSet[rt] = struct{}{} } - // cursorFor returns the by_entitlement index key for rec — + // cursorFor returns the primary grant key for rec — // needed because a post-filter break at len(out) == limit may // leave matching records unconsumed in the engine page, and // the engine's end-of-page cursor would skip them. cursorFor := func(rec *v3.GrantRecord) string { - p := rec.GetPrincipal() - return encodeCursor(encodeGrantByEntitlementIndexKey( - ent.GetId(), - p.GetResourceTypeId(), p.GetResourceId(), - rec.GetExternalId(), - )) + id, err := grantIdentityFromRecord(rec) + if err != nil { + return "" + } + return encodeCursor(encodeGrantIdentityKey(id)) } out := make([]*v2.Grant, 0, limit) @@ -233,10 +244,10 @@ func (a *Adapter) ListGrantsForEntitlement( var next string if principalID != nil { records, next, err = a.engine.PaginateGrantsByEntitlementPrincipal(ctx, - ent.GetId(), principalID.GetResourceType(), principalID.GetResource(), cursor, fetchLimit) + entIdentity, principalID.GetResourceType(), principalID.GetResource(), cursor, fetchLimit) } else { records, next, err = a.engine.PaginateGrantsByEntitlement(ctx, - ent.GetId(), cursor, fetchLimit) + entIdentity, cursor, fetchLimit) } if err != nil { return nil, c1zstore.AdaptNotFound(err, pebble.ErrNotFound) @@ -297,13 +308,36 @@ func (a *Adapter) ListGrantPrincipalKeysForEntitlement( if entitlement == nil || entitlement.GetId() == "" { return nil, "", errors.New("ListGrantPrincipalKeysForEntitlement: missing entitlement id") } - keys, next, err := a.engine.PaginateGrantPrincipalKeysByEntitlement(ctx, entitlement.GetId(), pageToken, clampPageSize(pageSize)) + entIdentity, err := a.entitlementIdentityForRequest(ctx, entitlement) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return nil, "", nil + } + return nil, "", err + } + keys, next, err := a.engine.PaginateGrantPrincipalKeysByEntitlement(ctx, entIdentity, pageToken, clampPageSize(pageSize)) if err != nil { return nil, "", c1zstore.AdaptNotFound(err, pebble.ErrNotFound) } return keys, next, nil } +// entitlementIdentityForRequest resolves the structural identity for a +// request-supplied entitlement. When the request carries the resource ref +// (the normal case — the expander and c1 send full stubs), identity derives +// exactly from structured parts; otherwise the raw id string resolves +// through the bare-id lookup (exactly-one rule; pebble.ErrNotFound when the +// id matches nothing). +func (a *Adapter) entitlementIdentityForRequest(ctx context.Context, ent *v2.Entitlement) (entitlementIdentity, error) { + if ent == nil || ent.GetId() == "" { + return entitlementIdentity{}, errors.New("missing entitlement id") + } + if res := ent.GetResource(); res.GetId().GetResourceType() != "" && res.GetId().GetResource() != "" { + return entitlementIdentityFromParts(res.GetId().GetResourceType(), res.GetId().GetResource(), ent.GetId()), nil + } + return a.engine.resolveGrantScanEntitlementIdentity(ctx, ent.GetId()) +} + // ListGrantsForPrincipal is the Go-level convenience method that // matches C1File.ListGrantsForPrincipal. It is NOT a gRPC RPC — // the explorer / cel-search consumers reach C1File directly today. @@ -338,7 +372,8 @@ func (a *Adapter) ListGrantsForPrincipal( out := make([]*v2.Grant, 0, len(records)) for _, rec := range records { // Optional entitlement filter — narrows the principal scan - // to a single entitlement when the caller passes one. + // to a single entitlement when the caller passes one. Raw string + // equality: refs and request ids are the same connector strings. if ent := req.GetEntitlement(); ent != nil && ent.GetId() != "" { if rec.GetEntitlement().GetEntitlementId() != ent.GetId() { continue @@ -353,8 +388,8 @@ func (a *Adapter) ListGrantsForPrincipal( } // ListGrantsForResourceType paginates grants whose principal is of -// the given resource_type_id, via idxGrantByPrincipalResourceType. -// The cursor is the index key. +// the given resource_type_id, via the by_principal index prefix. +// The cursor is the by_principal index key. // // Implements reader_v2.GrantsReaderServiceServer. func (a *Adapter) ListGrantsForResourceType( @@ -429,7 +464,10 @@ func (a *Adapter) ListSyncs(ctx context.Context, req *reader_v2.SyncsReaderServi return nil, err } prefix := encodeSyncRunFullPrefix() - lower, upper := rangeAfter(prefix, cursorBytes) + lower, upper, err := rangeAfter(prefix, cursorBytes) + if err != nil { + return nil, err + } iter, err := a.engine.DB().NewIter(&pebble.IterOptions{ LowerBound: lower, UpperBound: upper, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go index 0938167c..fb27441f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go @@ -12,8 +12,9 @@ import ( "os" "path/filepath" "runtime" - "sort" + "slices" "sync" + "sync/atomic" "time" "github.com/cockroachdb/pebble/v2/objstorage/objstorageprovider" @@ -51,11 +52,9 @@ const bulkSpillBufferSize = 1 << 20 // grantIndexFamilies are the index-discriminator bytes AddGrants can // emit (see grantIndexKeys). var grantIndexFamilies = []byte{ - idxGrantByEntitlement, idxGrantByPrincipal, idxGrantByNeedsExpansion, - idxGrantByPrincipalResourceType, - idxGrantByEntitlementResource, + idxGrantBySourceScope, } // bulkSSTWriter builds one SST file for a single disjoint key bucket. @@ -117,30 +116,32 @@ func (w *bulkSSTWriter) finish() error { // is written once into its final table — no WAL append, no L0 flush, no // background compaction debt. // -// Resource types, resources, and entitlements stream straight into one -// SST per bucket and must arrive in strictly increasing encoded-key -// order (SQLite BINARY collation order on the key's tuple columns — -// the tuple key codec is order-preserving; violations fail with -// ErrBulkImportOutOfOrder). These add methods are single-threaded. +// Resource types and resources stream straight into one SST per bucket +// and must arrive in strictly increasing encoded-key order (SQLite BINARY +// collation order on the key's tuple columns — the tuple key codec is +// order-preserving; violations fail with ErrBulkImportOutOfOrder). These +// add methods are single-threaded. // -// Grants scale across goroutines: each scanning goroutine takes its own -// shard (NewGrantShard) covering a disjoint ascending external-id range, -// so the grant hot path acquires no shared lock at all. Shard primaries -// stream straight into one final SST per shard (ordered by the shard -// contract — no spill, no sort, no merge); the five secondary index -// families (derived internally from the translated records — the same -// nil-guards and key shapes as the engine's canonical writeXxxIndexes -// paths) are key-only spill-sorted. Spill chunks sort and flush to disk -// in the background while the scans keep streaming; Finish k-way merges -// each family's sorted runs (across all shards) into one SST per -// family, in parallel, and ingests everything in a single call. +// Entitlements and grants are keyed by structural identity, whose tuple +// order does NOT match the converter's external-id scan order, so they go +// through spill sorters instead (entitlements single-threaded on the +// parent; grants scale across goroutines, each scanning goroutine taking +// its own shard via NewGrantShard so the grant hot path acquires no +// shared lock at all). The secondary index families are derived +// internally from the translated records — the same nil-guards and key +// shapes as the engine's canonical writeXxxIndexes paths — and are +// key-only spill-sorted. Spill chunks sort and flush to disk in the +// background while the scans keep streaming; Finish k-way merges each +// family's sorted runs (across all shards) into one SST per family, in +// parallel, and ingests everything in a single call. // -// Every record/index tuple must appear at most once (no dedup, no -// read-before-write); SQLite's UNIQUE(external_id, sync_id) gives -// converters that guarantee. The destination sync must be freshly -// started (MarkFreshSync via StartNewSync) and empty; nothing else may -// write to the engine between Start and Finish. Used by C1File.ToPebble -// for sqlite→pebble conversion. +// SQLite's UNIQUE(external_id, sync_id) guarantees each row appears once. +// Grant rows with distinct external ids but identical refs fold to one +// structural identity; Finish field-merges such duplicates and warns. +// Entitlement identities embed the raw external id and cannot fold. The +// destination sync must be freshly started (MarkFreshSync via +// StartNewSync) and empty; nothing else may write to the engine between +// Start and Finish. Used by C1File.ToPebble for sqlite→pebble conversion. type BulkSyncImport struct { e *Engine syncID string @@ -149,16 +150,20 @@ type BulkSyncImport struct { resourceTypes *bulkSSTWriter resources *bulkSSTWriter - entitlements *bulkSSTWriter + + // entitlements go through a spill sorter, not an ordered SST writer: + // the converter scans in external-id order, but the structural identity + // key does not sort the same way (tuple separators sort below printable + // bytes, and the flag component reorders stripped vs opaque ids), so the + // stream must be re-sorted before it can become an SST. + entitlements *spillSorter // sortSem bounds concurrently running background chunk sorts // across all sorters and shards. sortSem chan struct{} - // Parent-level sorters for the (single-threaded) resource and - // entitlement index keys. - idxResourceByParent *spillSorter - idxEntitlementByResource *spillSorter + // Parent-level sorter for the (single-threaded) resource index keys. + idxResourceByParent *spillSorter // mu guards shard registration/aggregation. The grant hot path // never takes it. @@ -168,6 +173,25 @@ type BulkSyncImport struct { resourcesByRT map[string]int64 entitlementsByRT map[string]int64 + + // Duplicate-fold accounting, written by Finish's grants merge goroutine + // and read by ComputedStats afterwards. Legacy files can hold grant rows + // with distinct external ids but identical refs — one structural + // identity; those fold to a single row at merge time, and the streaming + // per-RT counters above must be corrected to match. Entitlements cannot + // fold: their identity embeds the raw external id, which SQLite's + // UNIQUE(external_id, sync_id) makes unique. + grantDupRowsMerged int64 + grantDupRowsByEntRT map[string]int64 + + // Ref-less legacy rows cannot be represented in the structured keyspace + // at all; they are dropped with a warning, matching the in-place + // id-index migration's policy (a single malformed row in a legacy + // SQLite file must not fail the whole conversion). Written by + // AddEntitlements (single-threaded) and by shards under their own + // accounting, aggregated at Finish. + entitlementsSkippedMissingRefs atomic.Int64 + grantsSkippedMissingRefs atomic.Int64 } // StartBulkSyncImport opens a bulk import targeting syncID, which must be @@ -195,12 +219,13 @@ func (e *Engine) StartBulkSyncImport(ctx context.Context, syncID string, tmpDir // pins one chunk arena, so this bounds sort memory too). sorters := min(4, max(2, runtime.GOMAXPROCS(0)/2)) b := &BulkSyncImport{ - e: e, - syncID: syncID, - dir: dir, - sortSem: make(chan struct{}, sorters), - resourcesByRT: map[string]int64{}, - entitlementsByRT: map[string]int64{}, + e: e, + syncID: syncID, + dir: dir, + sortSem: make(chan struct{}, sorters), + resourcesByRT: map[string]int64{}, + entitlementsByRT: map[string]int64{}, + grantDupRowsByEntRT: map[string]int64{}, } for _, w := range []struct { slot **bulkSSTWriter @@ -208,7 +233,6 @@ func (e *Engine) StartBulkSyncImport(ctx context.Context, syncID string, tmpDir }{ {&b.resourceTypes, "resource-types"}, {&b.resources, "resources"}, - {&b.entitlements, "entitlements"}, } { sw, err := newBulkSSTWriter(dir, w.name) if err != nil { @@ -217,27 +241,25 @@ func (e *Engine) StartBulkSyncImport(ctx context.Context, syncID string, tmpDir } *w.slot = sw } + b.entitlements = newSpillSorter(dir, "entitlements", b.sortSem, bulkSpillKeyChunkBytes) b.idxResourceByParent = newSpillSorter(dir, fmt.Sprintf("index-%02x-p", idxResourceByParent), b.sortSem, bulkSpillKeyChunkBytes) - b.idxEntitlementByResource = newSpillSorter(dir, fmt.Sprintf("index-%02x-p", idxEntitlementByResource), b.sortSem, bulkSpillKeyChunkBytes) return b, nil } // BulkGrantShard is one goroutine's private view of the grant import: -// an ordered primary SST writer and per-family index sorters. Create +// a primary-record spill sorter and per-family index sorters. Create // one per scanning goroutine via NewGrantShard; AddGrants on a shard // takes no shared locks. Close the shard when its scan completes. // -// Grants within a shard MUST arrive sorted by external id, and shards -// must cover pairwise-disjoint external-id ranges — exactly what a -// sharded `ORDER BY external_id` scan over partitioned ranges of -// SQLite's UNIQUE(external_id, sync_id) index produces. That lets each -// shard stream its primaries straight into a final SST with no spill, -// no sort, and no merge; pebble's Ingest rejects overlapping tables, so -// a boundary bug cannot corrupt the store. The index keys sort on other -// fields and go through the shard's spill sorters. +// Grant primary keys are structural identities, which do not sort in the +// converter's external-id scan order, so both the primary rows and the +// index keys go through spill sorters; Finish k-way merges the runs of +// all shards per family. Arrival order therefore does not matter for +// correctness. Distinct legacy external ids that fold to one structural +// identity are merged at Finish with a warning. type BulkGrantShard struct { b *BulkSyncImport - grants *bulkSSTWriter + grants *spillSorter idx map[byte]*spillSorter entRT map[string]int64 closed bool @@ -255,13 +277,9 @@ func (b *BulkSyncImport) NewGrantShard() (*BulkGrantShard, error) { defer b.mu.Unlock() id := b.shardSeq b.shardSeq++ - w, err := newBulkSSTWriter(b.dir, fmt.Sprintf("grants-s%03d", id)) - if err != nil { - return nil, err - } s := &BulkGrantShard{ b: b, - grants: w, + grants: newSpillSorter(b.dir, fmt.Sprintf("grants-s%03d", id), b.sortSem, bulkSpillKeyChunkBytes), idx: map[byte]*spillSorter{}, entRT: map[string]int64{}, } @@ -303,7 +321,15 @@ func (s *BulkGrantShard) AddGrantsWithDiscoveredAt(ctx context.Context, grants [ return s.fail(err) } s.valBuf = val - if err := s.grants.add(encodeGrantKey(r.GetExternalId()), val); err != nil { + id, err := grantIdentityFromRecord(r) + if err != nil { + // Missing entitlement/principal refs: unrepresentable in the + // structured keyspace. Drop with a warning (counted, logged at + // Finish) — the in-place migration applies the same policy. + s.b.grantsSkippedMissingRefs.Add(1) + continue + } + if err := s.grants.add(encodeGrantIdentityKey(id), val); err != nil { return s.fail(err) } s.entRT[r.GetEntitlement().GetResourceTypeId()]++ @@ -338,9 +364,7 @@ func (s *BulkGrantShard) Close() { return } s.closed = true - if err := s.grants.finish(); err != nil { - _ = s.fail(err) - } + s.grants.cutAndDispatch() for _, w := range s.idx { w.cutAndDispatch() } @@ -436,9 +460,11 @@ func (b *BulkSyncImport) AddResourcesWithDiscoveredAt(ctx context.Context, resou return nil } -// AddEntitlements translates and appends entitlements, which must arrive -// sorted by external id. by_resource index keys are derived and spilled -// with the same resource guard as writeEntitlementIndexes. +// AddEntitlements translates and appends entitlements. Rows are keyed by +// structural identity and re-sorted through a spill sorter, so arrival +// order does not matter (the converter's external-id scan order does NOT +// match identity-tuple order). Identity embeds the raw external id, so +// duplicates are impossible for well-formed sources and fail the merge. func (b *BulkSyncImport) AddEntitlements(ctx context.Context, ents ...*v2.Entitlement) error { return b.AddEntitlementsWithDiscoveredAt(ctx, ents, nil) } @@ -464,19 +490,18 @@ func (b *BulkSyncImport) AddEntitlementsWithDiscoveredAt(ctx context.Context, en if err != nil { return err } - if err := b.entitlements.add(encodeEntitlementKey(rec.GetExternalId()), val); err != nil { + id, err := entitlementIdentityFromRecord(rec) + if err != nil { + // Missing resource ref: unrepresentable in the structured + // keyspace. Drop with a warning (counted, logged at Finish) — + // the in-place migration applies the same policy. + b.entitlementsSkippedMissingRefs.Add(1) + continue + } + if err := b.entitlements.add(encodeEntitlementIdentityKey(id), val); err != nil { return err } b.entitlementsByRT[rec.GetResource().GetResourceTypeId()]++ - if res := rec.GetResource(); res != nil && res.GetResourceId() != "" { - k := encodeEntitlementByResourceIndexKey( - res.GetResourceTypeId(), res.GetResourceId(), - rec.GetExternalId(), - ) - if err := b.idxEntitlementByResource.add(k, nil); err != nil { - return err - } - } } return nil } @@ -484,9 +509,10 @@ func (b *BulkSyncImport) AddEntitlementsWithDiscoveredAt(ctx context.Context, en // ComputedStats returns a SyncStatsRecord built from the counts the // import accumulated while streaming (primary record counts, resources // by resource type, entitlements by resource type, grants by entitlement -// resource type). Valid after -// all grant shards are Closed. Assets do not flow through the importer -// — the caller sets that count itself before stashing the record via +// resource type). Valid after all grant shards are Closed; call it after +// Finish so the counts reflect any duplicate-identity grant rows that +// folded at merge time. Assets do not flow through the importer — the +// caller sets that count itself before stashing the record via // Engine.StashComputedSyncStats. func (b *BulkSyncImport) ComputedStats() *v3.SyncStatsRecord { b.mu.Lock() @@ -494,16 +520,20 @@ func (b *BulkSyncImport) ComputedStats() *v3.SyncStatsRecord { var grants int64 grantsByEntRT := map[string]int64{} for _, s := range b.shards { - grants += int64(s.grants.count) + grants += s.grants.count for rt, n := range s.entRT { grantsByEntRT[rt] += n } } + grants -= b.grantDupRowsMerged + for rt, n := range b.grantDupRowsByEntRT { + grantsByEntRT[rt] -= n + } rec := &v3.SyncStatsRecord{ SyncId: b.syncID, ResourceTypes: int64(b.resourceTypes.count), Resources: int64(b.resources.count), - Entitlements: int64(b.entitlements.count), + Entitlements: b.entitlements.count, Grants: grants, ResourcesByResourceType: b.resourcesByRT, EntitlementsByResourceType: b.entitlementsByRT, @@ -524,11 +554,19 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { return errors.New("bulk sync import: already finished or aborted") } b.done = true - defer func() { _ = os.RemoveAll(b.dir) }() + // Full teardown, not a bare RemoveAll: Finish's error paths can return + // with the ordered SST writers still open (finish() failing on one + // leaves the other's file handle live) and with background chunk sorts + // still writing into the staging dir (nothing finalized the sorters + // yet). Removing the dir while a sort races its os.Create can strand + // the dir on disk, and Abort is a no-op once done is set — so this + // defer must do the closing and waiting itself. On success everything + // is already finished/finalized and teardown reduces to the RemoveAll. + defer b.teardown() start := time.Now() paths := make([]string, 0, 4+len(grantIndexFamilies)) - for _, w := range []*bulkSSTWriter{b.resourceTypes, b.resources, b.entitlements} { + for _, w := range []*bulkSSTWriter{b.resourceTypes, b.resources} { if err := w.finish(); err != nil { return err } @@ -537,8 +575,6 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { } } - // Per-shard grant primary SSTs: already final, already sorted, and - // pairwise disjoint by the shard range contract (Ingest verifies). b.mu.Lock() shards := b.shards b.mu.Unlock() @@ -549,23 +585,34 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { if s.err != nil { return s.err } - if s.grants.count > 0 { - paths = append(paths, s.grants.path) - } } // Build the merge units: each combines the sorted runs of one key - // family across every sorter that produced them. + // family across every sorter that produced them. Units with a resolve + // function tolerate duplicate keys by merging their values (legacy + // files can hold grant rows with distinct external ids but identical + // refs); units without one treat duplicates as corruption. type mergeUnit struct { name string sorters []*spillSorter + resolve func(key []byte, values [][]byte) ([]byte, error) } units := []mergeUnit{ + {name: "grants", resolve: b.resolveDuplicateGrants}, + {name: "entitlements", sorters: []*spillSorter{b.entitlements}}, {name: fmt.Sprintf("index-%02x", idxResourceByParent), sorters: []*spillSorter{b.idxResourceByParent}}, - {name: fmt.Sprintf("index-%02x", idxEntitlementByResource), sorters: []*spillSorter{b.idxEntitlementByResource}}, + } + for _, s := range shards { + units[0].sorters = append(units[0].sorters, s.grants) } for _, idx := range grantIndexFamilies { - u := mergeUnit{name: fmt.Sprintf("index-%02x", idx)} + u := mergeUnit{ + name: fmt.Sprintf("index-%02x", idx), + // Grant index keys are emitted per input row, so folded + // duplicate grants emit byte-identical (value-less) index keys; + // keep the first and drop the rest. + resolve: func(_ []byte, values [][]byte) ([]byte, error) { return values[0], nil }, + } for _, s := range shards { u.sorters = append(u.sorters, s.idx[idx]) } @@ -594,7 +641,13 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { return } sstPath := filepath.Join(b.dir, u.name+".sst") - if err := mergeSortedSpillChunksToSST(ctx, sstPath, u.name, chunks); err != nil { + var err error + if u.resolve != nil { + _, err = mergeSpillChunksToSSTResolvingDuplicates(ctx, sstPath, u.name, chunks, u.resolve) + } else { + err = mergeSortedSpillChunksToSST(ctx, sstPath, u.name, chunks) + } + if err != nil { results[slot].err = err return } @@ -610,6 +663,21 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { paths = append(paths, r.path) } } + if b.grantDupRowsMerged > 0 { + ctxzap.Extract(ctx).Warn("bulk sync import: merged legacy grant rows that share one structural identity", + zap.Int64("rows_merged_away", b.grantDupRowsMerged), + ) + } + if n := b.entitlementsSkippedMissingRefs.Load(); n > 0 { + ctxzap.Extract(ctx).Warn("bulk sync import: dropped legacy entitlements with no resource ref; they cannot be keyed in the structured layout", + zap.Int64("dropped", n), + ) + } + if n := b.grantsSkippedMissingRefs.Load(); n > 0 { + ctxzap.Extract(ctx).Warn("bulk sync import: dropped legacy grants with missing entitlement/principal refs; they cannot be keyed in the structured layout", + zap.Int64("dropped", n), + ) + } mergesDone := time.Now() if len(paths) == 0 { @@ -619,6 +687,12 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { if err := b.e.db.Ingest(ctx, paths); err != nil { return fmt.Errorf("bulk sync import: ingest: %w", err) } + b.e.noteEntitlementKeyspaceWrite() + // The data keyspaces are no longer provably empty: subsequent + // Put*Records calls in this sync must take their read-before-write + // paths so overwrites of imported identities clean up index entries. + _ = b.e.takeFreshGrantsEmpty() + _ = b.e.takeFreshResourcesEmpty() return nil }) if err != nil { @@ -632,6 +706,24 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { return nil } +// resolveDuplicateGrants merges the values of grant rows whose distinct +// legacy external ids fold to one structural identity, using the same +// policy as the in-place id-index migration, and records the fold so +// ComputedStats can correct its streaming per-row counters. +func (b *BulkSyncImport) resolveDuplicateGrants(key []byte, values [][]byte) ([]byte, error) { + merged, err := mergeDuplicateGrantValues(values) + if err != nil { + return nil, fmt.Errorf("bulk sync import: merge duplicate grant key %x: %w", key, err) + } + var rec v3.GrantRecord + if err := unmarshalRecord(merged, &rec); err != nil { + return nil, err + } + b.grantDupRowsMerged += int64(len(values) - 1) + b.grantDupRowsByEntRT[rec.GetEntitlement().GetResourceTypeId()] += int64(len(values) - 1) + return merged, nil +} + // Abort closes and removes all staged files without ingesting. Safe to // call after Finish (no-op) or on a partially-failed import. func (b *BulkSyncImport) Abort() { @@ -639,7 +731,17 @@ func (b *BulkSyncImport) Abort() { return } b.done = true - for _, w := range []*bulkSSTWriter{b.resourceTypes, b.resources, b.entitlements} { + b.teardown() +} + +// teardown closes both ordered SST writers, waits out every spill +// sorter's in-flight background chunk sorts, and then removes the +// staging directory. The waits must precede the RemoveAll: a chunk +// sort racing the removal can re-create a file mid-walk and strand the +// directory. Idempotent against already-finished writers and +// already-finalized sorters, so Finish can run it unconditionally. +func (b *BulkSyncImport) teardown() { + for _, w := range []*bulkSSTWriter{b.resourceTypes, b.resources} { if w != nil { _ = w.finish() } @@ -649,12 +751,12 @@ func (b *BulkSyncImport) Abort() { b.mu.Unlock() for _, s := range shards { s.closed = true - _ = s.grants.finish() + s.grants.abort() for _, w := range s.idx { w.abort() } } - for _, w := range []*spillSorter{b.idxResourceByParent, b.idxEntitlementByResource} { + for _, w := range []*spillSorter{b.entitlements, b.idxResourceByParent} { if w != nil { w.abort() } @@ -686,6 +788,13 @@ type spillSorter struct { dir string sem chan struct{} chunkBytes int + // free, when set, recycles chunk arenas through an explicit bounded + // freelist instead of the sync.Pool. Long single-producer builds (the + // synth-grant layer, the deferred index) allocate tens of GB per run + // through the pool because GC cycles clear it faster than arenas + // return; a plain channel survives GC and caps live arenas at the + // sort concurrency anyway (cutAndDispatch blocks on the sem). + free *spillArenaFreeList arena []byte views []kvView @@ -701,6 +810,68 @@ func newSpillSorter(dir, name string, sem chan struct{}, chunkBytes int) *spillS return &spillSorter{name: name, dir: dir, sem: sem, chunkBytes: chunkBytes} } +// spillArenaFreeList is a bounded, GC-proof recycler for chunk arenas of one +// size, shared by the sorters of one build (segments of a layer session, the +// deferred index sorter). Capacity should cover the maximum arenas live at +// once: one being filled by the producer plus one per concurrent sort slot. +type spillArenaFreeList struct { + size int + ch chan []byte +} + +func newSpillArenaFreeList(size, slots int) *spillArenaFreeList { + return &spillArenaFreeList{size: size, ch: make(chan []byte, slots)} +} + +func (f *spillArenaFreeList) get() []byte { + select { + case b := <-f.ch: + return b[:0] + default: + return make([]byte, 0, f.size) + } +} + +func (f *spillArenaFreeList) put(b []byte) { + if cap(b) < f.size { + return + } + select { + case f.ch <- b[:0]: + default: + } +} + +// spillArenaPool / spillViewsPool recycle the per-chunk key/value arena and its +// kvView slice across chunks. Without recycling, every cut allocates a fresh +// arena that grows from zero by append-doubling and is then GC'd; deferred index +// builds can otherwise churn hundreds of GB in short-lived buffers. +var spillArenaPool = sync.Pool{New: func() any { b := make([]byte, 0, bulkSpillKeyChunkBytes); return &b }} +var spillViewsPool = sync.Pool{New: func() any { v := make([]kvView, 0, 1<<16); return &v }} + +func getSpillArena() []byte { + bp := spillArenaPool.Get().(*[]byte) + return (*bp)[:0] +} + +func putSpillArena(b []byte) { + if cap(b) < bulkSpillKeyChunkBytes { + return + } + b = b[:0] + spillArenaPool.Put(&b) +} + +func getSpillViews() []kvView { + vp := spillViewsPool.Get().(*[]kvView) + return (*vp)[:0] +} + +func putSpillViews(v []kvView) { + v = v[:0] + spillViewsPool.Put(&v) +} + // add appends one entry. val may be nil (key-only spills). NOT // goroutine-safe — each sorter has exactly one producer. May block on // the sort semaphore when the arena fills (backpressure on this @@ -712,6 +883,22 @@ func (s *spillSorter) add(key, val []byte) error { if err := s.takeErr(); err != nil { return err } + if s.arena == nil { + if s.free != nil { + s.arena = s.free.get() + } else { + s.arena = getSpillArena() + if cap(s.arena) < s.chunkBytes { + // Pooled arenas are sized for bulkSpillKeyChunkBytes; a sorter + // with a larger chunk size (deferredIndexSpillChunkBytes) + // allocates its full arena up front instead of append-doubling + // through it, and returns the small one to the pool. + putSpillArena(s.arena) + s.arena = make([]byte, 0, s.chunkBytes) + } + } + s.views = getSpillViews() + } keyOff := len(s.arena) s.arena = append(s.arena, key...) s.arena = append(s.arena, val...) @@ -743,8 +930,11 @@ func (s *spillSorter) cutAndDispatch() { } func (s *spillSorter) sortAndWriteChunk(arena []byte, views []kvView) { - sort.Slice(views, func(i, j int) bool { - return bytes.Compare(arena[views[i].keyOff:views[i].keyEnd], arena[views[j].keyOff:views[j].keyEnd]) < 0 + // slices.SortFunc, not sort.Slice: typed swaps instead of the + // reflection-based Swapper. Chunk sorts run hot (57M+ keys per whale + // deferred index build) and the reflective swap was ~half the sort cost. + slices.SortFunc(views, func(a, b kvView) int { + return bytes.Compare(arena[a.keyOff:a.keyEnd], arena[b.keyOff:b.keyEnd]) }) s.chunkMu.Lock() chunkPath := filepath.Join(s.dir, fmt.Sprintf("%s-chunk-%04d.bin", s.name, len(s.chunks))) @@ -753,6 +943,12 @@ func (s *spillSorter) sortAndWriteChunk(arena []byte, views []kvView) { if err := writeSortedSpillChunk(chunkPath, arena, views); err != nil { s.setErr(err) } + if s.free != nil { + s.free.put(arena) + } else { + putSpillArena(arena) + } + putSpillViews(views) } func (s *spillSorter) setErr(err error) { @@ -915,6 +1111,8 @@ func readSpillEntry(r io.Reader, key, val *[]byte, lenBuf *[4]byte) (bool, error // single SST. Duplicate keys are corruption (the importer requires // globally unique tuples) and fail the merge. func mergeSortedSpillChunksToSST(ctx context.Context, sstPath, name string, chunks []string) error { + start := time.Now() + l := ctxzap.Extract(ctx) readers := make([]*os.File, 0, len(chunks)) defer func() { for _, r := range readers { @@ -954,10 +1152,9 @@ func mergeSortedSpillChunksToSST(ctx context.Context, sstPath, name string, chun } }() var last []byte + var written int64 + lastLog := start for len(*h) > 0 { - if err := ctx.Err(); err != nil { - return err - } item := h.pop() if bytes.Equal(item.key, last) { return fmt.Errorf("%w: bucket %s key %x", errBulkImportDuplicateKey, name, item.key) @@ -969,6 +1166,24 @@ func mergeSortedSpillChunksToSST(ctx context.Context, sstPath, name string, chun if err := writer.add(item.key, v); err != nil { return err } + written++ + // Throttle the per-entry bookkeeping: ctx.Err and time.Now on every + // one of 57M+ merged entries were measurable in profiles. + if written&0xFFFF == 0 { + if err := ctx.Err(); err != nil { + return err + } + if now := time.Now(); now.Sub(lastLog) >= 15*time.Second { + l.Info("spill sorter: merging chunks", + zap.String("name", name), + zap.Int("chunks", len(chunks)), + zap.Int("active_chunks", len(*h)+1), + zap.Int64("entries_written", written), + zap.Duration("elapsed", now.Sub(start)), + ) + lastLog = now + } + } last = append(last[:0], item.key...) ok, err := readSpillEntry(bufReaders[item.chunkIdx], &keyBufs[item.chunkIdx], &valBufs[item.chunkIdx], &lenBuf) if err != nil { @@ -981,6 +1196,169 @@ func mergeSortedSpillChunksToSST(ctx context.Context, sstPath, name string, chun if err := writer.finish(); err != nil { return err } + l.Info("spill sorter: merge complete", + zap.String("name", name), + zap.Int("chunks", len(chunks)), + zap.Int64("entries_written", written), + zap.Duration("elapsed", time.Since(start)), + ) success = true return nil } + +// mergeSpillChunksToSSTResolvingDuplicates heap-merges sorted chunk files +// into a single SST like mergeSortedSpillChunksToSST, but tolerates duplicate +// keys: the values of each duplicate-key group are handed to resolve and the +// resolved value is written once. Legacy sources can hold rows with distinct +// external ids that fold to one structural identity, so buckets keyed by +// identity use this variant instead of treating duplicates as corruption. +// +// Unlike the strict variant, every entry is copied into reused group scratch +// before its chunk buffer is advanced (one extra value memcpy per entry, no +// per-entry allocation) — that is what makes the one-group lookahead safe +// against chunk-buffer reuse. Returns the number of duplicate groups +// resolved. +func mergeSpillChunksToSSTResolvingDuplicates( + ctx context.Context, + sstPath, name string, + chunks []string, + resolve func(key []byte, values [][]byte) ([]byte, error), +) (int64, error) { + start := time.Now() + l := ctxzap.Extract(ctx) + readers := make([]*os.File, 0, len(chunks)) + defer func() { + for _, r := range readers { + _ = r.Close() + } + }() + bufReaders := make([]*bufio.Reader, len(chunks)) + keyBufs := make([][]byte, len(chunks)) + valBufs := make([][]byte, len(chunks)) + h := &spillChunkHeap{} + var lenBuf [4]byte + for i, chunk := range chunks { + f, err := os.Open(chunk) // #nosec G304 - staged under the import's MkdirTemp dir. + if err != nil { + return 0, err + } + readers = append(readers, f) + bufReaders[i] = bufio.NewReaderSize(f, bulkSpillBufferSize) + ok, err := readSpillEntry(bufReaders[i], &keyBufs[i], &valBufs[i], &lenBuf) + if err != nil { + return 0, err + } + if ok { + h.push(spillChunkItem{chunkIdx: i, key: keyBufs[i], val: valBufs[i]}) + } + } + + writer, err := newBulkSSTWriter(filepath.Dir(sstPath), name) + if err != nil { + return 0, err + } + success := false + defer func() { + _ = writer.finish() + if !success { + _ = os.Remove(sstPath) + } + }() + + type valSpan struct{ off, end int } + var ( + haveCur bool + curKey []byte // reused scratch: the pending group's key + curArena []byte // reused scratch: the pending group's values + curSpans []valSpan + ) + var dupGroups, written, scanned int64 + flushCur := func() error { + if !haveCur { + return nil + } + var val []byte + if len(curSpans) == 1 { + val = curArena[curSpans[0].off:curSpans[0].end] + } else { + dupGroups++ + values := make([][]byte, len(curSpans)) + for i, sp := range curSpans { + values[i] = curArena[sp.off:sp.end] + } + resolved, err := resolve(curKey, values) + if err != nil { + return err + } + val = resolved + } + if len(val) == 0 { + val = nil + } + if err := writer.add(curKey, val); err != nil { + return err + } + written++ + return nil + } + + lastLog := start + for len(*h) > 0 { + item := h.pop() + if haveCur && bytes.Equal(item.key, curKey) { + off := len(curArena) + curArena = append(curArena, item.val...) + curSpans = append(curSpans, valSpan{off: off, end: len(curArena)}) + } else { + if err := flushCur(); err != nil { + return dupGroups, err + } + curKey = append(curKey[:0], item.key...) + curArena = append(curArena[:0], item.val...) + curSpans = append(curSpans[:0], valSpan{off: 0, end: len(curArena)}) + haveCur = true + } + scanned++ + // Same throttled bookkeeping as the strict merge. + if scanned&0xFFFF == 0 { + if err := ctx.Err(); err != nil { + return dupGroups, err + } + if now := time.Now(); now.Sub(lastLog) >= 15*time.Second { + l.Info("spill sorter: merging chunks", + zap.String("name", name), + zap.Int("chunks", len(chunks)), + zap.Int("active_chunks", len(*h)+1), + zap.Int64("entries_scanned", scanned), + zap.Duration("elapsed", now.Sub(start)), + ) + lastLog = now + } + } + // Advance the popped chunk only AFTER the entry was consumed into + // the group scratch: readSpillEntry overwrites the buffers item + // aliases. + ok, err := readSpillEntry(bufReaders[item.chunkIdx], &keyBufs[item.chunkIdx], &valBufs[item.chunkIdx], &lenBuf) + if err != nil { + return dupGroups, err + } + if ok { + h.push(spillChunkItem{chunkIdx: item.chunkIdx, key: keyBufs[item.chunkIdx], val: valBufs[item.chunkIdx]}) + } + } + if err := flushCur(); err != nil { + return dupGroups, err + } + if err := writer.finish(); err != nil { + return dupGroups, err + } + l.Info("spill sorter: merge complete", + zap.String("name", name), + zap.Int("chunks", len(chunks)), + zap.Int64("entries_written", written), + zap.Int64("duplicate_groups_resolved", dupGroups), + zap.Duration("elapsed", time.Since(start)), + ) + success = true + return dupGroups, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go index 3740f79e..f412b874 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go @@ -35,6 +35,10 @@ func scopedRanges() [][2][]byte { {GrantByPrincipalLowerBound(), GrantByPrincipalUpperBound()}, {GrantByPrincipalResourceTypeLowerBound(), GrantByPrincipalResourceTypeUpperBound()}, {GrantByNeedsExpansionLowerBound(), GrantByNeedsExpansionUpperBound()}, + {GrantBySourceScopeLowerBound(), GrantBySourceScopeUpperBound()}, + {EntitlementBySourceScopeLowerBound(), EntitlementBySourceScopeUpperBound()}, + {ResourceBySourceScopeLowerBound(), ResourceBySourceScopeUpperBound()}, + {SourceCacheEntryLowerBound(), SourceCacheEntryUpperBound()}, {encodeAssetPrefix(), upperBoundOf(encodeAssetPrefix())}, // Stats sidecar — single key; the half-open range shape // contains exactly that one key. @@ -85,12 +89,16 @@ func (e *Engine) ResetForNewSync(ctx context.Context) error { {Start: []byte{versionV3, typeResourceType}, End: []byte{versionV3, typeEngineMeta}}, {Start: SyncStatsSidecarLowerBound(), End: SyncStatsSidecarUpperBound()}, } - return e.withWrite(func() error { + // AllowSealed: StartNewSync legitimately replaces a finished (sealed) + // sync; the wipe is the first step of leaving the sealed state. The + // engine stays sealed until MarkFreshSync unseals it right after. + return e.withWriteAllowSealed(func() error { for _, span := range spans { if err := e.db.Excise(ctx, span); err != nil { return fmt.Errorf("ResetForNewSync: excise [%x, %x): %w", span.Start, span.End, err) } } + e.noteEntitlementKeyspaceWrite() return nil }) } @@ -104,6 +112,18 @@ func (e *Engine) ResetForNewSync(ctx context.Context) error { // Errors are best-effort: a Compact failure on one range doesn't block // the others — pebble retries compaction in the background. ctx is // honored between ranges and surfaced verbatim on cancellation. +// +// Refuses with ErrEngineSealed after EndSync (via checkWritable): manual +// compactions go through the same CompactionScheduler as automatic ones, +// so on a sealed (paused) engine db.Compact would block forever waiting +// for a grant — and, because we hold writeWG, deadlock Engine.Close too. +// Bind a sync (SetCurrentSync) first. +// +// KNOWN LIMITATION: the gate only refuses calls made after the seal. A +// CompactAllRanges already inside its loop when EndSync pauses the +// scheduler blocks in db.Compact indefinitely (and holds writeWG, so a +// later Close hangs too). Do not run this concurrently with EndSync; no +// in-tree caller does. func (e *Engine) CompactAllRanges(ctx context.Context) error { if err := ctx.Err(); err != nil { return err diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/compaction_scheduler.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/compaction_scheduler.go new file mode 100644 index 00000000..cb7e0992 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/compaction_scheduler.go @@ -0,0 +1,199 @@ +package pebble + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/cockroachdb/pebble/v2" +) + +// pausableCompactionScheduler is pebble's ConcurrencyLimitScheduler with a +// pause switch. It exists because a fresh-sync store is saved (checkpoint + +// envelope) and closed moments after EndSync: any automatic compaction still +// running or scheduled in that window is discarded work that competes with +// the deferred index build and the envelope encode for CPU and IO. On the +// whale benchmark the post-expansion window ran 200+ compactions totaling +// ~2 minutes of compaction time whose output never survived to the saved +// artifact. +// +// Pause only stops NEW compactions from being granted; in-flight ones run to +// completion. Flushes are unaffected (they do not go through the +// CompactionScheduler), so memtables still drain to L0. A paused engine can +// accumulate L0 files toward L0StopWritesThreshold, which is why pausing is +// reserved for the EndSync-to-close window where writes are near zero, and +// why binding a new sync resumes the scheduler. +type pausableCompactionScheduler struct { + // db is set in Register, strictly before any other method is called. + db pebble.DBForCompaction + paused atomic.Bool + registered atomic.Bool + + mu struct { + sync.Mutex + runningCompactions int + // unregistered transitions once from false => true. + unregistered bool + // isGranting serializes granting from Done and the periodic granter, + // and lets Unregister wait out an in-flight grant loop. + isGranting bool + isGrantingCond *sync.Cond + lastAllowedWithoutPermission int + } + stopPeriodicGranterCh chan struct{} + pokePeriodicGranterCh chan struct{} +} + +var _ pebble.CompactionScheduler = (*pausableCompactionScheduler)(nil) +var _ pebble.CompactionGrantHandle = (*pausableCompactionScheduler)(nil) + +func newPausableCompactionScheduler() *pausableCompactionScheduler { + s := &pausableCompactionScheduler{ + stopPeriodicGranterCh: make(chan struct{}), + pokePeriodicGranterCh: make(chan struct{}, 1), + } + s.mu.isGrantingCond = sync.NewCond(&s.mu.Mutex) + return s +} + +// pause stops granting new compactions. In-flight compactions finish +// normally. +func (s *pausableCompactionScheduler) pause() { + s.paused.Store(true) +} + +// resume re-enables compaction granting and pokes the granter so a backlog +// is picked up promptly. +func (s *pausableCompactionScheduler) resume() { + if !s.paused.Swap(false) { + return + } + select { + case s.pokePeriodicGranterCh <- struct{}{}: + default: + } +} + +func (s *pausableCompactionScheduler) Register(numGoroutinesPerCompaction int, db pebble.DBForCompaction) { + s.db = db + s.registered.Store(true) + go s.periodicGranter() +} + +func (s *pausableCompactionScheduler) Unregister() { + // Guard against an Unregister without a successful Register (e.g. a + // failed Open): with no granter goroutine, the stop send would block + // forever. + if !s.registered.Swap(false) { + return + } + s.stopPeriodicGranterCh <- struct{}{} + s.mu.Lock() + defer s.mu.Unlock() + s.mu.unregistered = true + // Wait until isGranting becomes false. Since unregistered is now true, + // once isGranting is false no more granting can happen. + for s.mu.isGranting { + s.mu.isGrantingCond.Wait() + } +} + +func (s *pausableCompactionScheduler) TrySchedule() (bool, pebble.CompactionGrantHandle) { + if s.paused.Load() { + return false, nil + } + s.mu.Lock() + defer s.mu.Unlock() + if s.mu.unregistered { + return false, nil + } + s.mu.lastAllowedWithoutPermission = s.db.GetAllowedWithoutPermission() + if s.mu.lastAllowedWithoutPermission > s.mu.runningCompactions { + s.mu.runningCompactions++ + return true, s + } + return false, nil +} + +func (s *pausableCompactionScheduler) Started() {} +func (s *pausableCompactionScheduler) MeasureCPU(pebble.CompactionGoroutineKind) {} +func (s *pausableCompactionScheduler) CumulativeStats(pebble.CompactionGrantHandleStats) {} + +func (s *pausableCompactionScheduler) Done() { + s.mu.Lock() + s.mu.runningCompactions-- + s.tryGrantLockedAndUnlock() +} + +func (s *pausableCompactionScheduler) UpdateGetAllowedWithoutPermission() { + s.mu.Lock() + allowedWithoutPermission := s.db.GetAllowedWithoutPermission() + tryGrant := allowedWithoutPermission > s.mu.lastAllowedWithoutPermission + s.mu.lastAllowedWithoutPermission = allowedWithoutPermission + s.mu.Unlock() + if tryGrant { + select { + case s.pokePeriodicGranterCh <- struct{}{}: + default: + } + } +} + +// tryGrantLockedAndUnlock mirrors ConcurrencyLimitScheduler: grant as many +// waiting compactions as the DB's allowed concurrency permits. Callers hold +// s.mu; it is unlocked on return. +func (s *pausableCompactionScheduler) tryGrantLockedAndUnlock() { + defer s.mu.Unlock() + if s.mu.unregistered || s.paused.Load() { + return + } + // Wait for turn to grant. + for s.mu.isGranting { + s.mu.isGrantingCond.Wait() + } + if s.mu.unregistered || s.paused.Load() { + return + } + s.mu.lastAllowedWithoutPermission = s.db.GetAllowedWithoutPermission() + toGrant := s.mu.lastAllowedWithoutPermission - s.mu.runningCompactions + if toGrant <= 0 { + return + } + s.mu.isGranting = true + s.mu.Unlock() + // INVARIANT: loop exits with s.mu unlocked. + for toGrant > 0 && !s.paused.Load() { + waiting, _ := s.db.GetWaitingCompaction() + if !waiting { + break + } + if !s.db.Schedule(s) { + break + } + s.mu.Lock() + s.mu.runningCompactions++ + toGrant-- + s.mu.Unlock() + } + // Unlocked by the defer. + s.mu.Lock() + s.mu.isGranting = false + s.mu.isGrantingCond.Broadcast() +} + +func (s *pausableCompactionScheduler) periodicGranter() { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ticker.C: + s.mu.Lock() + s.tryGrantLockedAndUnlock() + case <-s.pokePeriodicGranterCh: + s.mu.Lock() + s.tryGrantLockedAndUnlock() + case <-s.stopPeriodicGranterCh: + return + } + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/deferred_index.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/deferred_index.go new file mode 100644 index 00000000..6fe511c9 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/deferred_index.go @@ -0,0 +1,548 @@ +package pebble + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "time" + + "github.com/cockroachdb/pebble/v2" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// deferredIndexSpillChunkBytes is the spill-chunk arena size for the deferred +// by_principal index build. The shared bulkSpillKeyChunkBytes (8MiB) is sized +// for the bulk import, where lanes × index-families sorters are alive at once +// and small arenas bound aggregate memory. The deferred build is the opposite +// shape — one producer, one sorter, nothing else running — and with 8MiB +// chunks a whale (57M+ index keys ≈ 6.4GB) produced an 801-way final merge: +// ~10 heap comparisons per entry plus 801 open chunk files with 1MiB readers +// (~800MB of buffers). 128MiB chunks cut that to ~50 runs. Peak memory is the +// active arena plus up to `sorters` chunks being sorted in background +// (~640MB), in the same ballpark as what the wide merge's read buffers used. +const deferredIndexSpillChunkBytes = 128 << 20 + +// appendGrantByPrincipalKeyFromPrimary builds the by_principal index key +// directly from a primary grant key by permuting its tuple segments, into +// dst. The primary tail is ent_rt|ent_rid|ent_kind|ent_name|p_rt|p_id and the +// index tail is p_rt|p_id|ent_rt|ent_rid|ent_kind|ent_name — the segments are +// already escaped, and the tuple escaping is canonical, so splicing the raw +// bytes is byte-identical to decode + re-encode +// (encodeGrantByPrincipalIdentityIndexKey ∘ decodeGrantIdentityKey) while +// skipping six string allocations and a fresh key buffer per row. That +// decode/re-encode pair was ~7GB of allocations per whale deferred build. +// Returns ok=false for keys that do not have exactly six segments. +// +// Pinned against the decode+re-encode path by +// TestAppendGrantByPrincipalKeyFromPrimary. +func appendGrantByPrincipalKeyFromPrimary(dst, primaryKey []byte) ([]byte, bool) { + const prefixLen = 3 // versionV3 | typeGrant | separator + if len(primaryKey) < prefixLen || primaryKey[0] != versionV3 || primaryKey[1] != typeGrant || primaryKey[2] != 0 { + return dst, false + } + tail := primaryKey[prefixLen:] + // Split into exactly six escaped segments on bare separator bytes (the + // escape rules guarantee segments contain no bare 0x00). + var segs [6][]byte + rest := tail + for i := 0; i < 5; i++ { + sep := bytes.IndexByte(rest, 0) + if sep < 0 { + return dst, false + } + segs[i] = rest[:sep] + rest = rest[sep+1:] + } + if bytes.IndexByte(rest, 0) >= 0 { + return dst, false + } + segs[5] = rest + + dst = append(dst, versionV3, typeIndex, idxGrantByPrincipal, 0) + for i, idx := range [6]int{4, 5, 0, 1, 2, 3} { // p_rt, p_id, ent_rt, ent_rid, ent_kind, ent_name + if i > 0 { + dst = append(dst, 0) + } + dst = append(dst, segs[idx]...) + } + return dst, true +} + +// deferredGrantStats carries the grant-keyspace stats accumulated during the +// BuildDeferredGrantIndexes scan: the same numbers computeSyncStats derives +// from its own full grant scan. Fusing them into the index scan removes a +// second O(grants) pass at EndSync. +type deferredGrantStats struct { + syncID string + grants int64 + grantsByEntitlementRT map[string]int64 +} + +// deferredRebuildCutBytes bounds the raw (key+value) bytes per rebuilt +// primary-grant SST. Cuts are strictly increasing in key order, so the files +// are disjoint and one IngestAndExcise call accepts them all. +const deferredRebuildCutBytes = 256 << 20 + +// rebuildBatch is one arena of raw rows handed from the scan thread to the +// rebuild writer goroutine. views index into arena (see kvView). +type rebuildBatch struct { + arena []byte + views []kvView +} + +// grantRebuildTee streams raw primary-grant rows from the deferred index scan +// to a background goroutine that builds the rolling rebuild SSTs, so SST +// block compression and file IO overlap the scan instead of adding to it (an +// inline tee inflated the whale scan from ~28s to ~63s). Rows are copied into +// pooled arenas (the iterator's key/value bytes are only valid until Next) +// and flushed in ~8MiB batches over a small bounded channel: the scan blocks +// when the writer falls more than a few batches behind. Single producer. +type grantRebuildTee struct { + dir string + + // Scan-thread state: the arena being filled. + arena []byte + views []kvView + rawBytes int64 + + ch chan rebuildBatch + done chan struct{} + + // Writer-goroutine state; read by finish()/abort() only after done. + writer *bulkSSTWriter + chunkBytes int64 + seq int + files []string + rowsWritten int64 + + mu sync.Mutex + err error +} + +func newGrantRebuildTee(dir string) *grantRebuildTee { + t := &grantRebuildTee{ + dir: dir, + ch: make(chan rebuildBatch, 4), + done: make(chan struct{}), + } + go t.run() + return t +} + +func (t *grantRebuildTee) setErr(err error) { + t.mu.Lock() + if t.err == nil { + t.err = err + } + t.mu.Unlock() +} + +func (t *grantRebuildTee) takeErr() error { + t.mu.Lock() + defer t.mu.Unlock() + return t.err +} + +func (t *grantRebuildTee) run() { + defer close(t.done) + for batch := range t.ch { + // After a failure, keep draining (recycling arenas) so the producer + // never blocks; the stored error surfaces at the next add/finish. + if t.takeErr() == nil { + if err := t.writeBatch(batch); err != nil { + t.setErr(err) + } + } + putSpillArena(batch.arena) + putSpillViews(batch.views) + } + if t.writer != nil { + if err := t.writer.finish(); err != nil { + t.setErr(err) + } else if t.takeErr() == nil && t.writer.count > 0 { + t.files = append(t.files, t.writer.path) + } + t.writer = nil + } +} + +func (t *grantRebuildTee) writeBatch(b rebuildBatch) error { + for _, v := range b.views { + if t.writer == nil { + w, err := newBulkSSTWriter(t.dir, fmt.Sprintf("grant-rebuild-%03d", t.seq)) + if err != nil { + return err + } + t.seq++ + t.writer = w + t.chunkBytes = 0 + } + if err := t.writer.add(b.arena[v.keyOff:v.keyEnd], b.arena[v.keyEnd:v.valEnd]); err != nil { + return err + } + t.rowsWritten++ + t.chunkBytes += int64(v.valEnd - v.keyOff) + if t.chunkBytes >= deferredRebuildCutBytes { + if err := t.writer.finish(); err != nil { + return err + } + t.files = append(t.files, t.writer.path) + t.writer = nil + } + } + return nil +} + +// add copies one raw row into the current arena, flushing a batch to the +// writer when the arena fills. +func (t *grantRebuildTee) add(key, val []byte) error { + if t.arena == nil { + t.arena = getSpillArena() + t.views = getSpillViews() + } + off := len(t.arena) + t.arena = append(t.arena, key...) + t.arena = append(t.arena, val...) + t.views = append(t.views, kvView{keyOff: off, keyEnd: off + len(key), valEnd: off + len(key) + len(val)}) + t.rawBytes += int64(len(key) + len(val)) + if len(t.arena) >= bulkSpillKeyChunkBytes { + return t.flushBatch() + } + return nil +} + +func (t *grantRebuildTee) flushBatch() error { + if err := t.takeErr(); err != nil { + return err + } + if len(t.views) == 0 { + return nil + } + t.ch <- rebuildBatch{arena: t.arena, views: t.views} + t.arena, t.views = nil, nil + return nil +} + +// rebuildResult is what a finished tee hands back: the rolling SST paths, +// the number of rows written into them (compared against the scan's row +// count before the excise is allowed to fire), and the total raw key+value +// bytes teed (logging only). +type rebuildResult struct { + files []string + rows int64 + rawBytes int64 +} + +// finish flushes the tail batch, waits the writer goroutine out, and returns +// the finished result. Call exactly once (use abort on early-error paths). +func (t *grantRebuildTee) finish() (rebuildResult, error) { + flushErr := t.flushBatch() + t.closeAndWait() + if err := t.takeErr(); err != nil { + return rebuildResult{}, err + } + if flushErr != nil { + return rebuildResult{}, flushErr + } + return rebuildResult{files: t.files, rows: t.rowsWritten, rawBytes: t.rawBytes}, nil +} + +// abort discards the tee: the writer goroutine drains and exits, partial SSTs +// are left for the caller's temp-dir cleanup. Safe after a failed flush; must +// not be called after finish. +func (t *grantRebuildTee) abort() { + t.setErr(errors.New("pebble: grant rebuild tee aborted")) + t.closeAndWait() +} + +func (t *grantRebuildTee) closeAndWait() { + close(t.ch) + <-t.done + if t.arena != nil { + putSpillArena(t.arena) + putSpillViews(t.views) + t.arena, t.views = nil, nil + } +} + +// BuildDeferredGrantIndexes rebuilds the remaining scattered expansion index +// family, by_principal, from entitlement-first primary grant keys. The expansion +// write path can skip by_principal inline because expansion reads by entitlement +// only; this method rewrites the whole by_principal range as one sorted SST at +// EndSync. +// +// The same scan also rebuilds the primary grant keyspace itself: every raw +// (key, value) is teed into rolling SSTs which replace the whole grant range +// via IngestAndExcise. Expansion's layer ingests are SSTs whose spans +// interleave, so they stack in the LSM's upper levels; consolidating them via +// compaction rewrites the data once per level hop (a whale measured ~60s / +// 431 compactions), while this tee rewrites each byte exactly once on top of +// a scan that is already being paid for the index build. After the excise the +// grant range is one flat sorted run in the bottom level: compaction debt ~0 +// without running the compactor at all. Requires no concurrent grant writers, +// which EndSync guarantees. +// +// The scan also accumulates the grant portion of the sync-stats sidecar +// (total count + per-entitlement-resource-type grouping) and stashes it so +// the subsequent PersistSyncStats call can skip its own full grant scan. The +// stats accumulation is best-effort: a value that fails the shallow field +// scan just disables the stash and PersistSyncStats falls back to scanning. +// +// The whole build runs under the engine write barrier (withWrite): the +// IngestAndExcise calls destroy everything in their key ranges, so a grant +// row committed by a concurrent writer after the scan's iterator snapshot +// would be silently erased. EndSync's callers are expected to have quiesced +// writers already — holding writeMu for the duration converts that +// convention into an enforced invariant (a straggler write blocks until the +// build finishes instead of racing the excise), and writeWG participation +// means Close waits the build out instead of tearing down e.db under it. +func (e *Engine) BuildDeferredGrantIndexes(ctx context.Context) error { + // AllowSealed: EndSync seals BEFORE running this build so no straggler + // record writer can slip a row in behind the scan (see Adapter.EndSync); + // the build itself is one of the sealed window's own steps. + return e.withWriteAllowSealed(func() error { + return e.buildDeferredGrantIndexesLocked(ctx) + }) +} + +func (e *Engine) buildDeferredGrantIndexesLocked(ctx context.Context) error { + // The scan below only polls ctx periodically; don't start an O(grants) + // pass (or its destructive IngestAndExcise) on an already-dead context. + if err := ctx.Err(); err != nil { + return err + } + start := time.Now() + l := ctxzap.Extract(ctx) + dir, err := os.MkdirTemp("", "pebble-deferred-idx-") + if err != nil { + return fmt.Errorf("BuildDeferredGrantIndexes: mkdir temp: %w", err) + } + defer os.RemoveAll(dir) + + sorters := min(4, max(2, runtime.GOMAXPROCS(0)/2)) + sem := make(chan struct{}, sorters) + principal := newSpillSorter(dir, fmt.Sprintf("index-%02x", idxGrantByPrincipal), sem, deferredIndexSpillChunkBytes) + principal.free = newSpillArenaFreeList(deferredIndexSpillChunkBytes, sorters+2) + // Every early return must wait out principal's background chunk sorts + // before the deferred RemoveAll(dir) deletes the directory they write + // into (mirrors the rebuild tee's guard below). Declared AFTER the + // RemoveAll defer so LIFO ordering runs the wait first; abort is a + // no-op once finalize has drained the sorter. + defer principal.abort() + + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: GrantLowerBound(), + UpperBound: GrantUpperBound(), + }) + if err != nil { + return fmt.Errorf("BuildDeferredGrantIndexes: iter: %w", err) + } + iterClosed := false + defer func() { + if !iterClosed { + _ = iter.Close() + } + }() + + statsOK := true + var totalKeys int64 + grantsByEntRTPtr := map[string]*int64{} + + rebuild := newGrantRebuildTee(dir) + rebuildFinished := false + defer func() { + if !rebuildFinished { + rebuild.abort() + } + }() + // A tee failure surfacing mid-scan (via add→flushBatch→takeErr) is the + // same class of failure finish() reports post-scan, and the GUARD below + // downgrades that to a loud skip-the-excise no-op. Honor the same + // contract here: remember the error, stop teeing, and keep scanning — + // by_principal and the stats stash don't depend on the rebuild. + var rebuildScanErr error + + var scanned, droppedMalformedKeys int64 + var idxKeyScratch []byte + lastLog := start + for iter.First(); iter.Valid(); iter.Next() { + totalKeys++ + if statsOK { + // Same shallow scan + grouping computeSyncStats performs; see + // sync_stats_sidecar.go for the map[string]*int64 rationale. + if entRT, serr := scanGrantEntitlementResourceTypeRaw(iter.Value()); serr != nil { + statsOK = false + } else if p, ok := grantsByEntRTPtr[string(entRT)]; ok { + *p++ + } else { + n := int64(1) + grantsByEntRTPtr[string(entRT)] = &n + } + } + // Tee the raw row into the primary-keyspace rebuild pipeline. The + // iterator yields globally sorted keys, so each SST is sorted and + // the cut files are mutually disjoint. + if rebuildScanErr == nil { + if err := rebuild.add(iter.Key(), iter.Value()); err != nil { + rebuildScanErr = err + } + } + idxKey, ok := appendGrantByPrincipalKeyFromPrimary(idxKeyScratch[:0], iter.Key()) + idxKeyScratch = idxKey + if !ok { + // Only possible on key-layout drift or corruption: every grant + // write path derives its key from the same 6-segment encoder. + // The row still rides the primary rebuild (totalKeys counts it), + // but it cannot be represented in by_principal — count and warn + // below so a principal silently losing grants is observable. + droppedMalformedKeys++ + continue + } + if err := principal.add(idxKey, nil); err != nil { + return err + } + scanned++ + // Throttle per-row bookkeeping; ctx.Err and time.Now are measurable + // at 57M+ rows. + if scanned&0xFFFF == 0 { + if err := ctx.Err(); err != nil { + return err + } + if now := time.Now(); now.Sub(lastLog) >= 15*time.Second { + l.Info("deferred grant index build: scanning primary grants", + zap.Int64("index_keys_added", scanned), + zap.Duration("elapsed", now.Sub(start)), + ) + lastLog = now + } + } + } + if err := iter.Error(); err != nil { + return fmt.Errorf("BuildDeferredGrantIndexes: iter error: %w", err) + } + if droppedMalformedKeys > 0 { + l.Error("deferred grant index build: grant primary keys did not decode as 6-segment identities; their rows are NOT represented in by_principal", + zap.Int64("dropped", droppedMalformedKeys), + ) + } + rebuildFinished = true + // The rebuild is an optimization: any failure — scan-time tee error or + // finish-time writer error — skips the excise and leaves the (correct, + // just unconsolidated) LSM alone rather than failing the sync. + var rebuilt rebuildResult + rebuildErr := rebuildScanErr + if rebuildErr == nil { + rebuilt, rebuildErr = rebuild.finish() + } else { + rebuild.abort() + } + if statsOK { + if syncID := codec.DecodeSyncID(e.currentSyncBytes()); syncID != "" { + grantsByEntitlementRT := make(map[string]int64, len(grantsByEntRTPtr)) + for rt, p := range grantsByEntRTPtr { + grantsByEntitlementRT[rt] = *p + } + e.stashDeferredGrantStats(&deferredGrantStats{ + syncID: syncID, + grants: totalKeys, + grantsByEntitlementRT: grantsByEntitlementRT, + }) + } + } + scanDone := time.Now() + l.Info("deferred grant index build: scan complete", + zap.Int64("index_keys_added", scanned), + zap.Duration("elapsed", scanDone.Sub(start)), + ) + + // Release the scan's pinned version before excising the range it read. + iterClosed = true + if err := iter.Close(); err != nil { + return fmt.Errorf("BuildDeferredGrantIndexes: close iter: %w", err) + } + + // Atomically replace the whole grant primary range with the rebuilt flat + // run. The excised span drops the stacked expansion-layer SSTs (and any + // batch-written grant rows in L0) in one version edit; the replacement + // files are disjoint and land in the bottom level. + // + // GUARD: the excise destroys everything in the range, so it only runs + // when the rebuilt SSTs verifiably contain every scanned row. A tee + // failure or a row-count mismatch downgrades to a loud no-op — the LSM + // stays correct, just unconsolidated. + switch { + case rebuildErr != nil: + l.Error("deferred grant index build: primary keyspace rebuild failed; skipping consolidation", + zap.Error(rebuildErr), + ) + case rebuilt.rows != totalKeys: + l.Error("deferred grant index build: rebuild row count mismatch; skipping consolidation", + zap.Int64("rows_scanned", totalKeys), + zap.Int64("rows_rebuilt", rebuilt.rows), + ) + case len(rebuilt.files) > 0: + if _, err := e.db.IngestAndExcise(ctx, rebuilt.files, nil, nil, pebble.KeyRange{ + Start: GrantLowerBound(), + End: GrantUpperBound(), + }); err != nil { + // Ingest is atomic: a failure applies nothing. Keep the sync alive. + l.Error("deferred grant index build: rebuild ingest/excise failed; skipping consolidation", + zap.Error(err), + ) + break + } + l.Info("deferred grant index build: primary grant keyspace rebuilt", + zap.Int("ssts", len(rebuilt.files)), + zap.Int64("rows", rebuilt.rows), + zap.Int64("raw_bytes", rebuilt.rawBytes), + zap.Duration("elapsed", time.Since(scanDone)), + ) + } + + chunks, err := principal.finalize() + if err != nil { + return err + } + if len(chunks) == 0 { + l.Info("deferred grant index build: no index chunks to ingest", + zap.Int64("index_keys_added", scanned), + zap.Duration("total", time.Since(start)), + ) + return nil + } + l.Info("deferred grant index build: merging sorted chunks", + zap.Int("chunks", len(chunks)), + zap.Int64("index_keys_added", scanned), + zap.Duration("scan", scanDone.Sub(start)), + ) + sstPath := filepath.Join(dir, fmt.Sprintf("index-%02x.sst", idxGrantByPrincipal)) + if err := mergeSortedSpillChunksToSST(ctx, sstPath, fmt.Sprintf("index-%02x", idxGrantByPrincipal), chunks); err != nil { + return err + } + mergeDone := time.Now() + + _, err = e.db.IngestAndExcise(ctx, []string{sstPath}, nil, nil, pebble.KeyRange{ + Start: GrantByPrincipalLowerBound(), + End: GrantByPrincipalUpperBound(), + }) + if err != nil { + return fmt.Errorf("BuildDeferredGrantIndexes: ingest/excise: %w", err) + } + + ctxzap.Extract(ctx).Info("deferred grant index build complete", + zap.Int64("index_keys_scanned", scanned), + zap.Duration("scan", scanDone.Sub(start)), + zap.Duration("merge", mergeDone.Sub(scanDone)), + zap.Duration("ingest", time.Since(mergeDone)), + zap.Duration("total", time.Since(start)), + ) + return nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go index 05644f28..fccb73db 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go @@ -45,7 +45,7 @@ type Engine struct { // can skip the read-before-write index-cleanup path because this // sync_id is guaranteed to be empty. freshSync bool - // freshGrantsEmpty / freshResourcesEmpty / freshEntitlementsEmpty + // freshGrantsEmpty / freshResourcesEmpty // are one-shot bits guarded by currentSyncMu. MarkFreshSync sets // each to true; the first PutXxxRecords call of the fresh sync // reads the value via takeFreshXxxEmpty() which returns it and @@ -53,9 +53,8 @@ type Engine struct { // call only" — subsequent calls in the same fresh sync must // still read-before-write to clean up cross-call duplicate index // entries. - freshGrantsEmpty bool - freshResourcesEmpty bool - freshEntitlementsEmpty bool + freshGrantsEmpty bool + freshResourcesEmpty bool // writeWG tracks in-flight writes. Incremented at the start of // every Writer method, decremented in defer. @@ -71,6 +70,90 @@ type Engine struct { // record they wrote. computedStatsMu sync.Mutex computedStats map[string]*v3.SyncStatsRecord + + // deferredGrantStats holds the grant counts BuildDeferredGrantIndexes + // accumulated while scanning the whole grant primary keyspace, so + // computeSyncStats can skip its own O(grants) scan at EndSync. Consumed + // once, guarded by sync_id. + deferredGrantStatsMu sync.Mutex + deferredGrantStats *deferredGrantStats + + // deferredIdxPending is set by grant writes that skipped the inline + // by_principal index write (all of them: the index family is scattered + // relative to the entitlement-first write order, so it is always built + // as one sorted SST at EndSync — see BuildDeferredGrantIndexes). + deferredIdxPending atomic.Bool + + // synthLayer is the open wave-scoped layer session, if any (see + // BeginSynthesizedGrantLayer). Single producer: the expansion driver + // opens/adds/finishes sessions strictly sequentially. synthLayerMu + // guards the pointer itself — Abort and Close read/nil it without the + // engine write barrier, so pointer access needs its own lock even + // though the session contents are only ever touched by one goroutine. + synthLayerMu sync.Mutex + synthLayer *synthGrantLayerSession + + // checkpointMu is the barrier between CheckpointTo's Flush→Checkpoint→ + // WAL-truncate window (write lock) and DB mutations that bypass writeMu + // (read lock). The full bypass inventory: + // - the synth-layer worker's background SST ingest (takes the read + // lock). It cannot take writeMu: an Add holding writeMu blocks on + // the worker's bounded segment channel, so worker-needs-writeMu + // would deadlock. A dedicated RWMutex gives CheckpointTo exclusion + // without that cycle. + // - CompactAllRanges/Flush (cleanup.go): deliberately writeMu-free + // long operations; safe against checkpoints because pebble + // compactions/flushes are internally consistent with Checkpoint + // and LogData(nil) carries no keys. + // - the compactor's raw DB() writes: fenced by call ordering (the + // merge completes before the store's save/CheckpointTo runs). + // Everything else (record writes, sessions, the stats sidecar, the + // deferred index build, bulk-import ingest) holds writeMu via + // withWrite*, which CheckpointTo also takes. + checkpointMu sync.RWMutex + + // sealed is the explicit post-EndSync lifecycle state. A successful + // EndSync seals the engine: record writes fail with ErrEngineSealed and + // the compaction scheduler is paused, because the only work left before + // save/close (checkpoint + envelope encode) never benefits from either. + // Binding a sync again (SetCurrentSync / MarkFreshSync) unseals and + // resumes compactions. Sync-run metadata writes (PutSyncRunRecord and + // friends) are exempt — callers legitimately stamp ended_at overrides, + // diff links, and supports_diff markers on a finished sync. Without this + // state the "no writes while compactions are paused" invariant was + // convention only, and a caller that kept writing after EndSync would + // silently accumulate L0 until pebble stalled writes at + // L0StopWritesThreshold with nothing left to resume the scheduler. + sealed atomic.Bool + // sealMu makes the (sealed, compactions-paused) pair transition + // atomically: an interleaved seal/unseal could otherwise end at + // sealed=false with the scheduler paused — writes allowed with nothing + // draining L0, the silent stall state sealing exists to eliminate. + sealMu sync.Mutex + + // compactionScheduler is the engine's pausable compaction scheduler, + // installed by newPebbleOptions. Pause/resume via pauseCompactions / + // resumeCompactions (package-private; see those funcs for why). + compactionScheduler *pausableCompactionScheduler + + // Lazy bare-id entitlement lookup (see lookup.go). entIDLookupGen is + // bumped by every entitlement-keyspace mutation; the map rebuilds on the + // next lookup when its built generation is stale. + entIDLookupGen atomic.Uint64 + entIDLookupMu sync.Mutex + entIDLookup map[string][]entitlementIdentity + entIDLookupBuiltGen uint64 + + // migratedOnOpen reports that this Open ran the in-place id-index + // migration. The store layer uses it to mark a writable store dirty so + // the migrated layout is saved back into the c1z once, instead of + // re-running the O(rows) migration on every subsequent open. + migratedOnOpen bool + + expandedWriteCalls atomic.Int64 + expandedWriteRows atomic.Int64 + synthesizedWriteCalls atomic.Int64 + synthesizedWriteRows atomic.Int64 } // Open creates or opens a Pebble engine rooted at dir. If dir does @@ -103,6 +186,9 @@ func Open(ctx context.Context, dir string, opts ...Option) (*Engine, error) { opts: o, pebbleOpts: pebbleOpts, } + if s, ok := pebbleOpts.Experimental.CompactionScheduler.(*pausableCompactionScheduler); ok { + e.compactionScheduler = s + } // Enforce the single-sync key-layout contract before touching any // keys: reject an old multi-sync-layout file (which the current // encoders would silently mis-decode) and stamp a fresh writable @@ -112,6 +198,20 @@ func Open(ctx context.Context, dir string, opts ...Option) (*Engine, error) { _ = e.Close() return nil, err } + if err := e.verifyOrStampIDIndexFormat(ctx); err != nil { + _ = e.Close() + return nil, err + } + // Restore the durable deferred-index marker (see + // encodeDeferredIdxPendingKey): a prior process may have deferred + // by_principal writes and been interrupted before the EndSync rebuild. + if _, closer, err := e.db.Get(encodeDeferredIdxPendingKey()); err == nil { + closer.Close() + e.deferredIdxPending.Store(true) + } else if !errors.Is(err, pebble.ErrNotFound) { + _ = e.Close() + return nil, err + } // Run secondary-index migrations before returning. Migrations // are skipped for read-only opens (the on-disk file is // immutable, so we'd error out trying to backfill). @@ -132,6 +232,21 @@ func (e *Engine) Close() error { } e.closing.Store(true) e.writeWG.Wait() + // A leaked synthesized-grant layer session (possible only if a panic + // unwound past the expansion driver's Abort) has a background worker + // ingesting through e.db; drain it before tearing the DB down. This + // runs AFTER the closing/writeWG barrier so no in-flight Add/Finish + // (which run under withWrite) can be touching the session concurrently + // — Abort itself takes no write barrier, only synthLayerMu for the + // pointer handoff, and is a no-op when no session is open. + _ = e.AbortSynthesizedGrantLayer(context.Background()) + // Hold writeMu for the teardown: writeWG only covers withWrite users, + // while CheckpointTo takes writeMu directly (no WG participation). A + // CheckpointTo that passed its closing check but hasn't locked yet must + // find either the mutex held or db nil'd under the lock — never a db + // torn down mid-checkpoint. + e.writeMu.Lock() + defer e.writeMu.Unlock() // Invariant: flush before close on any write path. This drives the // memtable out to an SST so a Close is never the step that leaves // un-materialized writes behind — independent of whether EndSync or @@ -168,11 +283,64 @@ func (e *Engine) SetCurrentSync(syncID string) error { e.freshSync = false e.freshGrantsEmpty = false e.freshResourcesEmpty = false - e.freshEntitlementsEmpty = false e.currentSyncMu.Unlock() + // Binding a sync means more writes are coming; leave the sealed state + // and resume compactions so L0 keeps draining (see seal). + e.unseal() return nil } +// pauseCompactions stops the engine from granting new automatic compactions. +// In-flight compactions finish; flushes are unaffected. Intended for the +// EndSync-to-close window, where compaction output never survives to the +// saved artifact but competes with the deferred index build and envelope +// encode. +// +// Deliberately unexported, as is resumeCompactions: the only way for a +// caller outside this package to restart compactions is to bind a sync +// (StartNewSync / ResumeSync / SetCurrentSync), which also unseals the +// engine. Pause without seal (or resume without a bound sync) is how the +// "writes on a paused scheduler stall at L0StopWritesThreshold" hang +// happens, so the two transitions are only available as a pair. +func (e *Engine) pauseCompactions() { + if e.compactionScheduler != nil { + e.compactionScheduler.pause() + } +} + +// resumeCompactions re-enables automatic compaction granting. See +// pauseCompactions for why this is unexported. +func (e *Engine) resumeCompactions() { + if e.compactionScheduler != nil { + e.compactionScheduler.resume() + } +} + +// seal moves the engine into the explicit post-EndSync state: record +// writes fail with ErrEngineSealed and automatic compactions stop. Called +// by Adapter.EndSync after a successful finalize; undone by binding a sync +// (SetCurrentSync / MarkFreshSync → unseal). See the sealed field doc for +// why this is a hard state rather than a convention. +func (e *Engine) seal() { + e.sealMu.Lock() + defer e.sealMu.Unlock() + e.sealed.Store(true) + e.pauseCompactions() +} + +// unseal leaves the sealed state and resumes automatic compactions. +func (e *Engine) unseal() { + e.sealMu.Lock() + defer e.sealMu.Unlock() + e.sealed.Store(false) + e.resumeCompactions() +} + +// IsSealed reports whether the engine is in the post-EndSync sealed state. +func (e *Engine) IsSealed() bool { + return e.sealed.Load() +} + // MarkFreshSync sets currentSync AND flags the sync as freshly // started (no prior records under this sync_id). The engine then // takes the perf-fast write path: pebble.NoSync per commit and skip @@ -193,8 +361,10 @@ func (e *Engine) MarkFreshSync(syncID string) error { e.freshSync = true e.freshGrantsEmpty = true e.freshResourcesEmpty = true - e.freshEntitlementsEmpty = true e.currentSyncMu.Unlock() + // A fresh sync writes heavily; leave the sealed state and resume + // compactions so L0 keeps draining (see seal). + e.unseal() return nil } @@ -208,7 +378,6 @@ func (e *Engine) clearCurrentSync() { e.freshSync = false e.freshGrantsEmpty = false e.freshResourcesEmpty = false - e.freshEntitlementsEmpty = false e.currentSyncMu.Unlock() } @@ -220,9 +389,9 @@ func (e *Engine) IsFreshSync() bool { return e.freshSync } -// takeFreshGrantsEmpty / takeFreshResourcesEmpty / -// takeFreshEntitlementsEmpty return true exactly once per fresh -// sync, for the first PutXxxRecords call of that type after +// takeFreshGrantsEmpty / takeFreshResourcesEmpty return true +// exactly once per fresh sync, for the first PutXxxRecords call +// of that type after // MarkFreshSync. Subsequent calls (and any call after EndSync) see // false. PutXxxRecords uses these to safely skip the // read-before-write Get on the first bulk write of each type: @@ -248,40 +417,35 @@ func (e *Engine) takeFreshResourcesEmpty() bool { return true } -func (e *Engine) takeFreshEntitlementsEmpty() bool { - e.currentSyncMu.Lock() - defer e.currentSyncMu.Unlock() - if !e.freshEntitlementsEmpty { - return false - } - e.freshEntitlementsEmpty = false - return true -} - // EndFreshSync clears the fresh-sync flag and flushes the memtable // + fsyncs the WAL so the data written during the sync is on disk // before the caller returns. Called by Adapter.EndSync. +// +// Uses withWrite (not a bare writeMu) so the flush participates in the +// closing check and writeWG: Close tears e.db down after writeWG.Wait, +// and a bare-mutex EndFreshSync racing Close would flush a nil db. func (e *Engine) EndFreshSync(ctx context.Context) error { - e.writeMu.Lock() - defer e.writeMu.Unlock() - - e.currentSyncMu.RLock() - wasFresh := e.freshSync - e.currentSyncMu.RUnlock() - if !wasFresh { + // AllowSealed: this is the last step of EndSync's sealed finalize + // window (see Adapter.EndSync). + return e.withWriteAllowSealed(func() error { + e.currentSyncMu.RLock() + wasFresh := e.freshSync + e.currentSyncMu.RUnlock() + if !wasFresh { + e.clearCurrentSync() + return nil + } + // Flush the memtable (turns NoSync-buffered writes into on-disk + // SSTs) and let pebble.LogData with Sync force-fsync the WAL tail. + if err := e.db.Flush(); err != nil { + return fmt.Errorf("EndFreshSync: flush: %w", err) + } + if err := e.db.LogData(nil, pebble.Sync); err != nil { + return fmt.Errorf("EndFreshSync: fsync WAL: %w", err) + } e.clearCurrentSync() return nil - } - // Flush the memtable (turns NoSync-buffered writes into on-disk - // SSTs) and let pebble.LogData with Sync force-fsync the WAL tail. - if err := e.db.Flush(); err != nil { - return fmt.Errorf("EndFreshSync: flush: %w", err) - } - if err := e.db.LogData(nil, pebble.Sync); err != nil { - return fmt.Errorf("EndFreshSync: fsync WAL: %w", err) - } - e.clearCurrentSync() - return nil + }) } // currentSyncBytes returns the engine's tracked sync_id (raw bytes) @@ -311,9 +475,23 @@ func (e *Engine) requireCurrentSync() error { return nil } -// checkWritable returns ErrEngineClosing if the engine has been -// closed. Called at the start of every Writer method. +// checkWritable returns ErrEngineClosing if the engine has been closed, +// and ErrEngineSealed after a successful EndSync until a sync is bound +// again. Called at the start of every Writer method. func (e *Engine) checkWritable() error { + if err := e.checkWritableAllowSealed(); err != nil { + return err + } + if e.sealed.Load() { + return ErrEngineSealed + } + return nil +} + +// checkWritableAllowSealed is checkWritable without the sealed check, for +// the few write paths that legitimately run on a finished sync (sync-run +// metadata updates and the pre-StartNewSync wipe). +func (e *Engine) checkWritableAllowSealed() error { if e.closing.Load() { return ErrEngineClosing } @@ -327,9 +505,33 @@ func (e *Engine) checkWritable() error { } // withWrite wraps a writer function with WaitGroup tracking + the -// closing check. The closure runs only if the engine is open. +// closing and sealed checks. The closure runs only if the engine is open +// and a sync is bound (not sealed). func (e *Engine) withWrite(fn func() error) error { - if err := e.checkWritable(); err != nil { + if e.sealed.Load() { + return ErrEngineSealed + } + return e.withWriteAllowSealed(func() error { + // Re-check under writeMu: the first check is lock-free, so a writer + // that passed it and then blocked on writeMu (e.g. behind the + // EndSync finalize steps) must not commit once the engine sealed in + // the meantime — a grant landing in that window would be + // permanently missing from by_principal (the deferred rebuild + // already ran and the pending marker was cleared). + if e.sealed.Load() { + return ErrEngineSealed + } + return fn() + }) +} + +// withWriteAllowSealed is withWrite without the sealed check. Reserved for +// writes that are part of the sealed lifecycle itself: sync-run metadata +// stamps on a finished sync (ended_at overrides, diff links, supports_diff) +// and ResetForNewSync's wipe on the way into a new sync. Record-data writes +// must use withWrite. +func (e *Engine) withWriteAllowSealed(fn func() error) error { + if err := e.checkWritableAllowSealed(); err != nil { return err } e.writeWG.Add(1) @@ -390,9 +592,20 @@ func (e *Engine) CurrentDBSizeBytes() (int64, error) { // DB returns the underlying *pebble.DB. Exported for the // synccompactor/pebble package; callers must not Close it directly // (use Engine.Close) and must respect the engine's lifecycle. Returns -// nil after Close. +// nil after Close. Callers that write the entitlement keyspace through +// this handle (ingests, excises) must call InvalidateBareIDLookups +// afterwards. func (e *Engine) DB() *pebble.DB { return e.db } +// InvalidateBareIDLookups invalidates the lazily built bare-id lookup +// state (see lookup.go). Engine write paths call this internally; it is +// exported for callers that mutate the keyspace through DB() directly. +func (e *Engine) InvalidateBareIDLookups() { e.noteEntitlementKeyspaceWrite() } + +// MigratedOnOpen reports whether this Open ran the in-place id-index +// migration (see migratedOnOpen). +func (e *Engine) MigratedOnOpen() bool { return e.migratedOnOpen } + // CheckpointTo writes a self-contained Pebble directory snapshot to // destDir. destDir must not exist yet. Pebble creates it and // hard-links SSTs where possible. @@ -416,6 +629,10 @@ func (e *Engine) DB() *pebble.DB { return e.db } // Flush→Checkpoint→truncate window. That prevents a write from // committing between the Flush and Checkpoint — such a write would // otherwise exist only in the WAL, which truncateCheckpointWALs discards. +// It also takes checkpointMu exclusively for the same window: the +// synth-layer session's background worker ingests SSTs outside writeMu +// (see ingestSynthLayerSegment), and a flushable ingest landing mid-window +// would be a WAL-only record the truncate discards. func (e *Engine) CheckpointTo(ctx context.Context, destDir string) error { // Wait for all in-flight writes to complete. e.writeWG.Wait() @@ -425,6 +642,13 @@ func (e *Engine) CheckpointTo(ctx context.Context, destDir string) error { } e.writeMu.Lock() defer e.writeMu.Unlock() + e.checkpointMu.Lock() + defer e.checkpointMu.Unlock() + // Re-check under the lock: Close (which also takes writeMu for its + // teardown) may have won the race and nil'd e.db. + if e.closing.Load() || e.db == nil { + return ErrEngineClosing + } if e.opts.readOnly { return copyReadOnlyDBDir(e.dbDir, destDir) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go index 8142a5f3..fd15e349 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go @@ -1,5 +1,5 @@ // Package pebble is the v3 storage engine for baton-sdk. It is the -// implementation behind dotc1z.EnginePebble and the v3 envelope. +// implementation behind c1zstore.EnginePebble and the v3 envelope. package pebble import ( @@ -40,7 +40,14 @@ var ( ErrEnvelopeTruncated = sentinel(codes.DataLoss, "pebble engine: v3 envelope truncated") ErrDiskFull = sentinel(codes.ResourceExhausted, "pebble engine: disk full (ENOSPC)") ErrNoCurrentSync = sentinel(codes.FailedPrecondition, "pebble engine: no current sync") + ErrEngineSealed = sentinel(codes.FailedPrecondition, "pebble engine: sealed after EndSync; bind a sync (StartNewSync/ResumeSync/SetCurrentSync) before writing") ErrSaveDestExists = sentinel(codes.AlreadyExists, "pebble engine: save destination already exists") ErrCrossFilesystem = sentinel(codes.InvalidArgument, "pebble engine: save tmpDir and dest must be on the same filesystem") ErrInvalidPageToken = sentinel(codes.InvalidArgument, "pebble engine: invalid page token") + // ErrAmbiguousExternalID is returned by the bare-id lookup edge when a + // lossy public id string matches more than one record (or has too many + // candidate parses to certify uniqueness). Callers that hold structured + // refs never hit this; it exists so a lossy string can never silently + // address the wrong record. + ErrAmbiguousExternalID = sentinel(codes.FailedPrecondition, "pebble engine: external id is ambiguous") ) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go index a470029f..7037b4db 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go @@ -18,15 +18,11 @@ func (e *Engine) PutEntitlementRecord(ctx context.Context, r *v3.EntitlementReco return e.PutEntitlementRecords(ctx, r) } -// PutEntitlementRecords writes N entitlements in two pebble.Batches -// — primary keys in one, by_resource index keys in the other. -// Mirrors the PutGrantRecords pattern (RFC §3a Tier-B/C): -// - within-call dedup pre-pass keyed by external_id drops earlier -// occurrences; -// - the first PutEntitlementRecords call of a fresh sync skips -// the read-before-write Get (keyspace provably empty); -// - subsequent calls fall back to read-before-write so cross-call -// duplicates can clean up the prior call's index entries. +// PutEntitlementRecords writes N entitlements by structured primary key. +// The identity key is a pure function of the record (it contains the raw +// external id), so overwrites are idempotent and no read-before-write or +// index cleanup is needed; a within-call dedup pre-pass keeps last-wins +// semantics for same-identity duplicates in one batch. func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.EntitlementRecord) error { if len(records) == 0 { return nil @@ -37,14 +33,11 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit } priBatch := e.db.NewBatch() defer priBatch.Close() - idxBatch := e.db.NewBatch() - defer idxBatch.Close() fresh := e.IsFreshSync() - skipGet := e.takeFreshEntitlementsEmpty() type dedupKey struct { - extID string + id entitlementIdentity } var dedup map[dedupKey]int if len(records) > 1 { @@ -53,7 +46,11 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit if r == nil { continue } - dedup[dedupKey{r.GetExternalId()}] = i + id, err := entitlementIdentityFromRecord(r) + if err != nil { + return err + } + dedup[dedupKey{id}] = i } } @@ -61,36 +58,34 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit if r == nil { continue } + id, err := entitlementIdentityFromRecord(r) + if err != nil { + return err + } if dedup != nil { - if dedup[dedupKey{r.GetExternalId()}] != i { + if dedup[dedupKey{id}] != i { continue } } - key := encodeEntitlementKey(r.GetExternalId()) + key := encodeEntitlementIdentityKey(id) val, err := marshalRecord(r) if err != nil { return err } - if !skipGet { - oldVal, closer, getErr := e.db.Get(key) - switch { - case getErr == nil: - if err := e.deleteEntitlementIndexesRaw(idxBatch, r.GetExternalId(), oldVal); err != nil { - closer.Close() - return err - } - closer.Close() - case errors.Is(getErr, pebble.ErrNotFound): - // no prior — write unconditionally - default: - return fmt.Errorf("PutEntitlementRecords: get old: %w", getErr) - } - } if err := priBatch.Set(key, val, nil); err != nil { return err } - if err := e.writeEntitlementIndexes(idxBatch, r); err != nil { - return err + // by_source_scope is the only entitlement secondary index. + // No read-before-write cleanup here (matching this method's + // no-Get philosophy): a same-identity rewrite under a + // different scope within one sync can leave a stale index + // entry, which replay tolerates — the copied record carries + // its true scope and the file holds a single sync, so the + // staleness cannot outlive it. + if sh := r.GetSourceScopeHash(); sh != "" { + if err := priBatch.Set(encodeEntitlementBySourceScopeIndexKey(sh, id), nil, nil); err != nil { + return err + } } } opts := writeOpts(e.opts.durability) @@ -100,12 +95,19 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit if err := priBatch.Commit(opts); err != nil { return err } - return idxBatch.Commit(opts) + e.noteEntitlementKeyspaceWrite() + return nil }) } +// GetEntitlementRecord fetches an entitlement by its raw public id via the +// bare-id lookup (exact string-match, exactly-one rule — see lookup.go). func (e *Engine) GetEntitlementRecord(ctx context.Context, externalID string) (*v3.EntitlementRecord, error) { - val, closer, err := e.db.Get(encodeEntitlementKey(externalID)) + id, err := e.resolveEntitlementIdentityByExternalID(ctx, externalID) + if err != nil { + return nil, err + } + val, closer, err := e.db.Get(encodeEntitlementIdentityKey(id)) if err != nil { return nil, err } @@ -117,40 +119,46 @@ func (e *Engine) GetEntitlementRecord(ctx context.Context, externalID string) (* return r, nil } +// DeleteEntitlementRecord deletes by raw public id. A missing id is a +// no-op; an ambiguous id is an error (a lossy string must never guess a +// delete). func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) error { return e.withWrite(func() error { - key := encodeEntitlementKey(externalID) - batch := e.db.NewBatch() - defer batch.Close() - oldVal, closer, err := e.db.Get(key) + id, err := e.resolveEntitlementIdentityByExternalID(ctx, externalID) if err != nil { if errors.Is(err, pebble.ErrNotFound) { return nil } return err } - if err := e.deleteEntitlementIndexesRaw(batch, externalID, oldVal); err != nil { + key := encodeEntitlementIdentityKey(id) + batch := e.db.NewBatch() + defer batch.Close() + // Clean up the source-scope index entry (the only entitlement + // secondary index) so a replayed scope can't resurrect the row. + if oldVal, closer, getErr := e.db.Get(key); getErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) closer.Close() - return err + if scanErr != nil { + return scanErr + } + if oldScope != "" { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr } - closer.Close() if err := batch.Delete(key, nil); err != nil { return err } - return batch.Commit(writeOpts(e.opts.durability)) - }) -} - -func (e *Engine) writeEntitlementIndexes(batch *pebble.Batch, r *v3.EntitlementRecord) error { - res := r.GetResource() - if res == nil || res.GetResourceId() == "" { + if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { + return err + } + e.noteEntitlementKeyspaceWrite() return nil - } - k := encodeEntitlementByResourceIndexKey( - res.GetResourceTypeId(), res.GetResourceId(), - r.GetExternalId(), - ) - return batch.Set(k, nil, nil) + }) } func (e *Engine) IterateEntitlements(ctx context.Context, yield func(*v3.EntitlementRecord) bool) error { @@ -176,7 +184,7 @@ func (e *Engine) IterateEntitlements(ctx context.Context, yield func(*v3.Entitle } func (e *Engine) IterateEntitlementsByResource(ctx context.Context, resourceTypeID, resourceID string, yield func(*v3.EntitlementRecord) bool) error { - indexPrefix := encodeEntitlementByResourcePrefix(resourceTypeID, resourceID) + indexPrefix := encodeEntitlementPrimaryResourcePrefix(resourceTypeID, resourceID) iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: indexPrefix, UpperBound: upperBoundOf(indexPrefix), @@ -186,21 +194,8 @@ func (e *Engine) IterateEntitlementsByResource(ctx context.Context, resourceType } defer iter.Close() for iter.First(); iter.Valid(); iter.Next() { - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { - continue - } - val, closer, err := e.db.Get(encodeEntitlementKey(externalID)) - if err != nil { - if errors.Is(err, pebble.ErrNotFound) { - continue - } - return err - } r := &v3.EntitlementRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { + if err := unmarshalRecord(iter.Value(), r); err != nil { return err } if !yield(r) { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/expand_write_path_stats.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/expand_write_path_stats.go new file mode 100644 index 00000000..92663051 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/expand_write_path_stats.go @@ -0,0 +1,17 @@ +package pebble + +type ExpandWritePathStats struct { + ExpandedCalls int64 + ExpandedRows int64 + SynthesizedCalls int64 + SynthesizedRows int64 +} + +func (e *Engine) ExpandWritePathStats() ExpandWritePathStats { + return ExpandWritePathStats{ + ExpandedCalls: e.expandedWriteCalls.Load(), + ExpandedRows: e.expandedWriteRows.Load(), + SynthesizedCalls: e.synthesizedWriteCalls.Load(), + SynthesizedRows: e.synthesizedWriteRows.Load(), + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go index 1365990a..a3758116 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go @@ -4,7 +4,10 @@ import ( "context" "errors" "fmt" + "os" + "path/filepath" "runtime" + "strconv" "sync" "sync/atomic" @@ -13,10 +16,11 @@ import ( v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" ) -// PutGrantRecord writes a grant record + its by_entitlement and -// by_principal index entries, atomically in a single pebble.Batch. +// PutGrantRecord writes a grant record + its remaining secondary index entries, +// atomically in a single pebble.Batch. // // This is the engine's canonical write path; other record types // follow the same shape (read the previous primary if any → delete @@ -85,11 +89,11 @@ func (e *Engine) PutGrantRecords(ctx context.Context, records ...*v3.GrantRecord skipGet := e.takeFreshGrantsEmpty() // Dedup pre-pass: keep only the LAST occurrence of each - // external_id. The map value is the records[] + // structured grant identity. The map value is the records[] // index — when we re-iterate, we process record i only if // dedup[ext] == i. type dedupKey struct { - extID string + id grantIdentity } var dedup map[dedupKey]int if len(records) > 1 { @@ -98,7 +102,11 @@ func (e *Engine) PutGrantRecords(ctx context.Context, records ...*v3.GrantRecord if r == nil { continue } - dedup[dedupKey{r.GetExternalId()}] = i + id, err := grantIdentityFromRecord(r) + if err != nil { + return err + } + dedup[dedupKey{id}] = i } } @@ -106,12 +114,16 @@ func (e *Engine) PutGrantRecords(ctx context.Context, records ...*v3.GrantRecord if r == nil { continue } + id, err := grantIdentityFromRecord(r) + if err != nil { + return err + } if dedup != nil { - if dedup[dedupKey{r.GetExternalId()}] != i { + if dedup[dedupKey{id}] != i { continue } } - key := encodeGrantKey(r.GetExternalId()) + key := encodeGrantIdentityKey(id) val, err := marshalRecord(r) if err != nil { return err @@ -142,10 +154,14 @@ func (e *Engine) PutGrantRecords(ctx context.Context, records ...*v3.GrantRecord if fresh { opts = pebble.NoSync } - if err := priBatch.Commit(opts); err != nil { + // One atomic commit: folding the index batch into the primary batch + // closes the durability gap where the primary commit landed but the + // index commit failed — a divergence that would persist in the saved + // artifact if the caller shipped it anyway. + if err := priBatch.Apply(idxBatch, nil); err != nil { return err } - return idxBatch.Commit(opts) + return priBatch.Commit(opts) }) } @@ -178,6 +194,8 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran if len(records) == 0 { return nil } + e.expandedWriteCalls.Add(1) + e.expandedWriteRows.Add(int64(len(records))) return e.withWrite(func() error { if err := e.requireCurrentSync(); err != nil { return err @@ -190,19 +208,23 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran now := timestamppb.Now() // Dedup pre-pass: keep only the LAST occurrence of each - // external_id. The expander appends the same deterministic + // structured grant identity. The expander appends the same deterministic // grant id more than once when it merges sources for a // principal across a source page, so without this the earlier // occurrences would leak orphan index entries (db.Get can't see // in-batch writes). Same safety net as PutGrantRecords. - var dedup map[string]int + var dedup map[grantIdentity]int if len(records) > 1 { - dedup = make(map[string]int, len(records)) + dedup = make(map[grantIdentity]int, len(records)) for i, r := range records { if r == nil { continue } - dedup[r.GetExternalId()] = i + id, err := grantIdentityFromRecord(r) + if err != nil { + return err + } + dedup[id] = i } } @@ -217,11 +239,15 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran if r == nil { continue } - if dedup != nil && dedup[r.GetExternalId()] != i { + id, err := grantIdentityFromRecord(r) + if err != nil { + return err + } + if dedup != nil && dedup[id] != i { continue } ext := r.GetExternalId() - keyScratch = appendGrantKey(keyScratch[:0], ext) + keyScratch = appendGrantIdentityKey(keyScratch[:0], id) oldVal, closer, getErr := e.db.Get(keyScratch) switch { @@ -237,6 +263,12 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran r.SetExpansion(prior.GetExpansion()) r.SetNeedsExpansion(prior.GetNeedsExpansion()) r.SetDiscoveredAt(prior.GetDiscoveredAt()) + // Preserve the source-cache scope stamp exactly like the + // expansion side-state: the expander rewrites existing + // direct grants to bake in Sources, and clobbering the + // stamp here would silently drop every expander-touched + // grant from the next sync's replay of its scope. + r.SetSourceScopeHash(prior.GetSourceScopeHash()) var err error idxScratch, err = e.deleteGrantIndexesScratch(idxBatch, ext, oldVal, idxScratch) if err != nil { @@ -268,10 +300,481 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran } } - if err := priBatch.Commit(pebble.NoSync); err != nil { + // One atomic commit: folding the index batch into the primary batch + // closes the durability gap where the primary commit landed but the + // index commit failed — a divergence that would persist in the saved + // artifact if the caller shipped it anyway. + if err := priBatch.Apply(idxBatch, nil); err != nil { + return err + } + return priBatch.Commit(pebble.NoSync) + }) +} + +// PutSynthesizedGrantRecords writes expander-synthesized grants that the caller +// guarantees are brand-new by structured grant identity. It skips the +// read-before-write Get in PutExpandedGrantRecords because there is no prior +// value whose Expansion/NeedsExpansion/DiscoveredAt or index entries must be +// preserved/cleaned. +func (e *Engine) PutSynthesizedGrantRecords(ctx context.Context, records []*v3.GrantRecord) error { + if len(records) == 0 { + return nil + } + e.synthesizedWriteCalls.Add(1) + e.synthesizedWriteRows.Add(int64(len(records))) + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + priBatch := e.db.NewBatch() + defer priBatch.Close() + idxBatch := e.db.NewBatch() + defer idxBatch.Close() + + now := timestamppb.Now() + var keyScratch, valScratch, idxScratch []byte + for _, r := range records { + if r == nil { + continue + } + id, err := grantIdentityFromRecord(r) + if err != nil { + return err + } + keyScratch = appendGrantIdentityKey(keyScratch[:0], id) + if r.GetDiscoveredAt() == nil { + r.SetDiscoveredAt(now) + } + val, err := marshalRecordAppend(valScratch[:0], r) + if err != nil { + return err + } + valScratch = val + if err := priBatch.Set(keyScratch, val, nil); err != nil { + return err + } + idxScratch, err = e.writeGrantIndexesScratch(idxBatch, r, idxScratch) + if err != nil { + return err + } + } + // One atomic commit: folding the index batch into the primary batch + // closes the durability gap where the primary commit landed but the + // index commit failed — a divergence that would persist in the saved + // artifact if the caller shipped it anyway. + if err := priBatch.Apply(idxBatch, nil); err != nil { + return err + } + return priBatch.Commit(pebble.NoSync) + }) +} + +type synthesizedGrantRecord struct { + id grantIdentity + entitlement *v3.EntitlementRef + principal *v3.PrincipalRef + sources batonGrant.Sources +} + +// PutSynthesizedGrantContributions batch-writes one destination's synthesized +// contributions. It is the fallback for stores/engines that cannot run a +// layer-scoped layer session (see BeginSynthesizedGrantLayer); the layer path +// is preferred because it publishes sorted SSTs instead of out-of-order batch +// commits. +func (e *Engine) PutSynthesizedGrantContributions(ctx context.Context, records []synthesizedGrantRecord) error { + if len(records) == 0 { + return nil + } + e.synthesizedWriteCalls.Add(1) + e.synthesizedWriteRows.Add(int64(len(records))) + return e.putSynthesizedGrantContributionsBatch(ctx, records) +} + +// synthLayerSegmentRows is the default row count at which an open layer +// session cuts its current segment and hands it to the background worker for +// merge + SST ingest. Cutting mid-layer is safe: nothing reads a layer's rows +// until the next layer begins, and keys are globally unique, so segments may +// cover overlapping key ranges without conflict. Segments keep the merge +// fan-in small, bound temp-disk usage, and overlap merge/ingest work with the +// producer's compute instead of serializing it at the layer boundary. +const synthLayerSegmentRows = 8_000_000 + +func synthLayerSegmentLimit() int64 { + if raw := os.Getenv("BATON_PEBBLE_SYNTH_LAYER_SEGMENT_ROWS"); raw != "" { + if parsed, err := strconv.ParseInt(raw, 10, 64); err == nil && parsed > 0 { + return parsed + } + } + return synthLayerSegmentRows +} + +// synthLayerSegment is one finalized batch of sorted spill chunks awaiting +// merge + ingest on the session's background worker. +type synthLayerSegment struct { + name string + chunks []string +} + +// synthGrantLayerSession accumulates one topological layer's synthesized grant +// rows as encoded (key, value) pairs in a background-sorted spill sorter. +// Every segLimit rows the current sorter is finalized into a segment and +// queued for the background worker, which k-way merges the segment's chunks +// into one SST and ingests it while the producer keeps encoding. Finish +// flushes the tail segment and waits the worker out. Encoding happens at Add +// time, so the session never retains references into the caller's reused +// buffers. One session at a time; the expansion driver is the single +// producer. +type synthGrantLayerSession struct { + dir string + sorter *spillSorter + sortSem chan struct{} + segIdx int + segRows int64 + segLimit int64 + + // segCh carries finalized segments to the worker; its capacity bounds + // how many merged-but-uningested segments can stack up before Add blocks + // (backpressure). segErr holds the worker's first failure, surfaced on + // the next Add/Finish. + segCh chan synthLayerSegment + segWG sync.WaitGroup + segMu sync.Mutex + segErr error + + // arenaFree recycles the 128MiB chunk arenas across all of the + // session's segment sorters (see spillArenaFreeList). + arenaFree *spillArenaFreeList + + now *timestamppb.Timestamp + keyScratch []byte + valScratch []byte + srcScratch batonGrant.Sources + rec *v3.GrantRecord // reused across rows; see fillSynthGrantRecord + rows int64 +} + +func (s *synthGrantLayerSession) setErr(err error) { + s.segMu.Lock() + if s.segErr == nil { + s.segErr = err + } + s.segMu.Unlock() +} + +func (s *synthGrantLayerSession) takeErr() error { + s.segMu.Lock() + defer s.segMu.Unlock() + return s.segErr +} + +func (s *synthGrantLayerSession) segName() string { + return fmt.Sprintf("synth-grant-layer-seg%03d", s.segIdx) +} + +// cutSegment finalizes the current sorter into a segment for the worker and +// starts a fresh sorter for the next segment. Blocks when the worker is more +// than one segment behind. +func (s *synthGrantLayerSession) cutSegment() error { + chunks, err := s.sorter.finalize() + if err != nil { + return err + } + name := s.segName() + s.segIdx++ + s.segRows = 0 + s.sorter = newSpillSorter(s.dir, s.segName(), s.sortSem, deferredIndexSpillChunkBytes) + s.sorter.free = s.arenaFree + if len(chunks) == 0 { + return nil + } + s.segCh <- synthLayerSegment{name: name, chunks: chunks} + return nil +} + +// BeginSynthesizedGrantLayer opens a layer-scoped layer session. The ingested +// SSTs carry primary rows only; the by_principal index is always rebuilt at +// EndSync, so no inline index maintenance is skipped by taking this path. +// The boolean is part of the store-level contract (non-Pebble stores report +// false and callers fall back to StoreNewExpandedGrantContributions). +func (e *Engine) BeginSynthesizedGrantLayer(ctx context.Context) (bool, error) { + if err := e.checkWritable(); err != nil { + return false, err + } + if err := e.requireCurrentSync(); err != nil { + return false, err + } + e.synthLayerMu.Lock() + defer e.synthLayerMu.Unlock() + if e.synthLayer != nil { + return false, errors.New("pebble: synthesized grant layer session already open") + } + e.synthLayer = &synthGrantLayerSession{ + now: timestamppb.Now(), + rec: &v3.GrantRecord{}, + segLimit: synthLayerSegmentLimit(), + } + return true, nil +} + +// loadSynthLayer returns the open layer session (or nil) under synthLayerMu. +func (e *Engine) loadSynthLayer() *synthGrantLayerSession { + e.synthLayerMu.Lock() + defer e.synthLayerMu.Unlock() + return e.synthLayer +} + +// takeSynthLayer detaches and returns the open layer session (or nil) under +// synthLayerMu, so Finish/Abort can tear it down without racing each other +// or Close. +func (e *Engine) takeSynthLayer() *synthGrantLayerSession { + e.synthLayerMu.Lock() + defer e.synthLayerMu.Unlock() + s := e.synthLayer + e.synthLayer = nil + return s +} + +// initSynthLayerSession lazily allocates the session's temp dir, first +// sorter, and background merge+ingest worker on the first Add. +func (e *Engine) initSynthLayerSession(ctx context.Context, s *synthGrantLayerSession) error { + dir, err := os.MkdirTemp("", "pebble-synth-grant-layer-") + if err != nil { + return fmt.Errorf("synth grant layer: mkdir temp: %w", err) + } + s.dir = dir + sorters := min(4, max(2, runtime.GOMAXPROCS(0)/2)) + s.sortSem = make(chan struct{}, sorters) + s.arenaFree = newSpillArenaFreeList(deferredIndexSpillChunkBytes, sorters+2) + // deferredIndexSpillChunkBytes, not bulkSpillKeyChunkBytes: like the + // deferred index build, the layer session is one producer with one live + // sorter carrying full grant records. 8MiB chunks turned one whale layer + // into a 3,400-way merge with ~3.4GB of read buffers; 128MiB chunks keep + // each segment's merge fan-in small. + s.sorter = newSpillSorter(dir, s.segName(), s.sortSem, deferredIndexSpillChunkBytes) + s.sorter.free = s.arenaFree + s.segCh = make(chan synthLayerSegment, 1) + s.segWG.Add(1) + go func() { + defer s.segWG.Done() + for seg := range s.segCh { + // After a failure (or abort), drain remaining segments without + // touching the DB; Finish surfaces the stored error. + if s.takeErr() != nil { + continue + } + if err := e.ingestSynthLayerSegment(ctx, s.dir, seg); err != nil { + s.setErr(err) + } + } + }() + return nil +} + +// ingestSynthLayerSegment merges one segment's sorted chunks into an SST and +// ingests it. Chunk files are deleted once merged; the SST path is left for +// the session's final dir cleanup (Pebble links/copies it on ingest). +// +// Runs on the session's background worker, which deliberately bypasses the +// engine write barrier (an Add holding writeMu can block on the bounded +// segment channel waiting for this worker, so worker-takes-writeMu would +// deadlock). The db itself stays valid for the worker's whole life — Close +// drains the worker via AbortSynthesizedGrantLayer before tearing the db +// down — but CheckpointTo's Flush→Checkpoint→WAL-truncate window must not +// see an ingest land in the middle: pebble's flushable-ingest path writes a +// WAL record referencing the ingested SSTs, and truncateCheckpointWALs would +// discard it from the snapshot. checkpointMu is that barrier. +func (e *Engine) ingestSynthLayerSegment(ctx context.Context, dir string, seg synthLayerSegment) error { + sstPath := filepath.Join(dir, seg.name+".sst") + if err := mergeSortedSpillChunksToSST(ctx, sstPath, seg.name, seg.chunks); err != nil { + return err + } + for _, chunk := range seg.chunks { + _ = os.Remove(chunk) + } + e.checkpointMu.RLock() + defer e.checkpointMu.RUnlock() + if err := e.db.Ingest(ctx, []string{sstPath}); err != nil { + return fmt.Errorf("synth grant layer: ingest segment %s: %w", seg.name, err) + } + return nil +} + +// AddSynthesizedGrantLayerContributions encodes records into the open layer +// session. Rows become readable as their segment is ingested; callers must +// not rely on visibility before FinishSynthesizedGrantLayer returns. +func (e *Engine) AddSynthesizedGrantLayerContributions(ctx context.Context, records []synthesizedGrantRecord) error { + if len(records) == 0 { + return nil + } + return e.withWrite(func() error { + s := e.loadSynthLayer() + if s == nil { + return errors.New("pebble: no open synthesized grant layer session") + } + if err := s.takeErr(); err != nil { + return err + } + if s.sorter == nil { + // Segment SSTs carry primary rows only; make sure EndSync builds + // the deferred by_principal index even if no batch write set the + // flag. Arm the marker BEFORE initializing the session: if the + // marker lands but init fails, the worst case is a spurious + // (cheap) rebuild at EndSync. The reverse order could leave an + // initialized session whose later Adds skip this block, ingesting + // rows with the rebuild unarmed. + if err := e.markDeferredIdxPending(); err != nil { + return err + } + if err := e.initSynthLayerSession(ctx, s); err != nil { + return err + } + } + e.synthesizedWriteCalls.Add(1) + e.synthesizedWriteRows.Add(int64(len(records))) + for i := range records { + rec := &records[i] + if rec.entitlement == nil || rec.principal == nil || len(rec.sources) == 0 { + continue + } + s.keyScratch = appendGrantIdentityKey(s.keyScratch[:0], rec.id) + // external_id (field 2) is hand-encoded FIRST — it is the + // lowest-numbered field these records carry, so emitting it + // before the base marshal preserves canonical field order + // without materializing a per-row concat string. + s.valScratch = appendSynthGrantExternalIDWire(s.valScratch[:0], rec) + fillSynthGrantRecord(s.rec, rec, s.now) + val, err := marshalRecordAppend(s.valScratch, s.rec) + if err != nil { + return err + } + // sources (field 9) is the record's highest field, so appending + // it after the base marshal matches the deterministic byte order. + val, s.srcScratch = appendGrantSourcesWire(val, s.srcScratch, rec.sources) + s.valScratch = val + if err := s.sorter.add(s.keyScratch, val); err != nil { + return err + } + s.rows++ + s.segRows++ + } + if s.segLimit > 0 && s.segRows >= s.segLimit { + if err := s.cutSegment(); err != nil { + return err + } + } + return nil + }) +} + +// FinishSynthesizedGrantLayer flushes the session's tail segment, waits for +// the background worker to merge and ingest every queued segment, and closes +// the session. No-op if no session is open or the session saw no rows. +func (e *Engine) FinishSynthesizedGrantLayer(ctx context.Context) error { + return e.withWrite(func() error { + s := e.takeSynthLayer() + if s == nil { + return nil + } + if s.dir != "" { + defer os.RemoveAll(s.dir) + } + if s.sorter == nil { + return nil + } + var finishErr error + chunks, err := s.sorter.finalize() + if err != nil { + finishErr = err + } else if len(chunks) > 0 { + s.segCh <- synthLayerSegment{name: s.segName(), chunks: chunks} + } + close(s.segCh) + s.segWG.Wait() + if err := s.takeErr(); err != nil && finishErr == nil { + finishErr = err + } + return finishErr + }) +} + +// AbortSynthesizedGrantLayer discards an in-flight layer session: already +// ingested segments remain in the DB (their rows are idempotent overwrites on +// retry), staged chunks are dropped. Safe to call with no open session. +// +// Deliberately does NOT take the engine write barrier — Close calls it after +// setting the closing flag (withWrite would refuse), and it must stay callable +// as a cleanup path when a writer holding writeMu panicked. The synthLayerMu +// take keeps the pointer handoff race-free against Begin/Add/Finish/Close. +func (e *Engine) AbortSynthesizedGrantLayer(ctx context.Context) error { + s := e.takeSynthLayer() + if s == nil { + return nil + } + if s.sorter != nil { + s.sorter.abort() + } + if s.segCh != nil { + // Make the worker drain queued segments without ingesting them. + s.setErr(errors.New("pebble: synthesized grant layer session aborted")) + close(s.segCh) + s.segWG.Wait() + } + if s.dir != "" { + _ = os.RemoveAll(s.dir) + } + return nil +} + +func (e *Engine) putSynthesizedGrantContributionsBatch(ctx context.Context, records []synthesizedGrantRecord) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + priBatch := e.db.NewBatch() + defer priBatch.Close() + idxBatch := e.db.NewBatch() + defer idxBatch.Close() + + now := timestamppb.Now() + var keyScratch, valScratch, idxScratch []byte + var srcScratch batonGrant.Sources + r := &v3.GrantRecord{} // reused across rows; see fillSynthGrantRecord + for i := range records { + rec := &records[i] + if rec.entitlement == nil || rec.principal == nil || len(rec.sources) == 0 { + continue + } + keyScratch = appendGrantIdentityKey(keyScratch[:0], rec.id) + // external_id first; see AddSynthesizedGrantLayerContributions. + valScratch = appendSynthGrantExternalIDWire(valScratch[:0], rec) + fillSynthGrantRecord(r, rec, now) + val, err := marshalRecordAppend(valScratch, r) + if err != nil { + return err + } + // sources (field 9) is the highest field this record carries, so + // appending it after the base marshal matches the deterministic + // byte order. (source_scope_hash is field 10, but synthesized + // grants never carry a scope stamp — fillSynthGrantRecord leaves + // it unset — so field 9 stays last on the wire.) + val, srcScratch = appendGrantSourcesWire(val, srcScratch, rec.sources) + valScratch = val + if err := priBatch.Set(keyScratch, val, nil); err != nil { + return err + } + idxScratch, err = e.writeGrantIndexesForIdentityScratch(idxBatch, rec.id, false, "", idxScratch) + if err != nil { + return err + } + } + // One atomic commit: folding the index batch into the primary batch + // closes the durability gap where the primary commit landed but the + // index commit failed — a divergence that would persist in the saved + // artifact if the caller shipped it anyway. + if err := priBatch.Apply(idxBatch, nil); err != nil { return err } - return idxBatch.Commit(pebble.NoSync) + return priBatch.Commit(pebble.NoSync) }) } @@ -297,6 +800,11 @@ func (e *Engine) UnsafePutUniqueGrantRecords(ctx context.Context, records ...*v3 if !e.IsFreshSync() { return errors.New("UnsafePutUniqueGrantRecords: sync is not fresh") } + // Consume the fresh-grants-empty bit: the keyspace is no longer + // provably empty, so a subsequent PutGrantRecords in this sync must + // take its read-before-write path to clean up index entries on + // overwrites of identities written here. + _ = e.takeFreshGrantsEmpty() type encoded struct { priKey []byte @@ -346,8 +854,18 @@ func (e *Engine) UnsafePutUniqueGrantRecords(ctx context.Context, records ...*v3 failed.Store(true) return } + id, err := grantIdentityFromRecord(r) + if err != nil { + errMu.Lock() + if encErr == nil { + encErr = err + } + errMu.Unlock() + failed.Store(true) + return + } enc[i] = encoded{ - priKey: encodeGrantKey(r.GetExternalId()), + priKey: encodeGrantIdentityKey(id), priVal: val, idxKeys: grantIndexKeys(r), } @@ -381,16 +899,25 @@ func (e *Engine) UnsafePutUniqueGrantRecords(ctx context.Context, records ...*v3 if e.IsFreshSync() { opts = pebble.NoSync } - if err := priBatch.Commit(opts); err != nil { + // One atomic commit: folding the index batch into the primary batch + // closes the durability gap where the primary commit landed but the + // index commit failed — a divergence that would persist in the saved + // artifact if the caller shipped it anyway. + if err := priBatch.Apply(idxBatch, nil); err != nil { return err } - return idxBatch.Commit(opts) + return priBatch.Commit(opts) }) } -// GetGrantRecord fetches a grant record by external_id. +// GetGrantRecord fetches a grant record by its raw public id via the +// bare-id lookup (candidate-split probing, exactly-one rule — lookup.go). func (e *Engine) GetGrantRecord(ctx context.Context, externalID string) (*v3.GrantRecord, error) { - key := encodeGrantKey(externalID) + id, err := e.resolveGrantIdentityByExternalID(ctx, externalID) + if err != nil { + return nil, err + } + key := encodeGrantIdentityKey(id) val, closer, err := e.db.Get(key) if err != nil { return nil, err @@ -403,63 +930,78 @@ func (e *Engine) GetGrantRecord(ctx context.Context, externalID string) (*v3.Gra return r, nil } -// DeleteGrantRecord removes a grant and its index entries. +// DeleteGrantRecord removes a grant and its index entries by raw public id. +// A missing/unresolvable id is a no-op; an ambiguous id is an error (a +// lossy string must never guess a delete). func (e *Engine) DeleteGrantRecord(ctx context.Context, externalID string) error { return e.withWrite(func() error { - key := encodeGrantKey(externalID) - - batch := e.db.NewBatch() - defer batch.Close() - - oldVal, closer, err := e.db.Get(key) + id, err := e.resolveGrantIdentityByExternalID(ctx, externalID) if err != nil { if errors.Is(err, pebble.ErrNotFound) { - return nil // delete of non-existent is a no-op + return nil } return err } - if err := e.deleteGrantIndexesRaw(batch, externalID, oldVal); err != nil { - closer.Close() - return err - } - closer.Close() + return e.deleteGrantByIdentityLocked(id) + }) +} - if err := batch.Delete(key, nil); err != nil { - return err - } - return batch.Commit(writeOpts(e.opts.durability)) +// DeleteGrantByIdentityRefs removes a grant addressed by its structural +// refs — the exact delete path for callers that hold the full grant. No +// lossy id string is involved. Deleting a non-existent grant is a no-op. +func (e *Engine) DeleteGrantByIdentityRefs(ctx context.Context, r *v3.GrantRecord) error { + id, err := grantIdentityFromRecord(r) + if err != nil { + return fmt.Errorf("DeleteGrantByIdentityRefs: %w", err) + } + return e.withWrite(func() error { + return e.deleteGrantByIdentityLocked(id) }) } -// grantIndexKeys returns the secondary-index keys for r. The set mirrors the -// index keyspaces documented on writeGrantIndexes: -// - by_entitlement: (entitlement_id, principal) — needs both ent + principal. -// - by_entitlement_resource: the resource side of the entitlement (drives -// ListGrants with req.Resource set). -// - by_principal and by_principal_resource_type: the principal side. -// - needs_expansion: only when the grant currently carries the flag. -func grantIndexKeys(r *v3.GrantRecord) [][]byte { - ent := r.GetEntitlement() - princ := r.GetPrincipal() - ext := r.GetExternalId() +// deleteGrantByIdentityLocked deletes one grant row and its index entries. +// Caller holds the engine write lock (withWrite). +func (e *Engine) deleteGrantByIdentityLocked(id grantIdentity) error { + key := encodeGrantIdentityKey(id) - keys := make([][]byte, 0, 4) - if ent != nil && princ != nil { - keys = append(keys, encodeGrantByEntitlementIndexKey( - ent.GetEntitlementId(), princ.GetResourceTypeId(), princ.GetResourceId(), ext)) + batch := e.db.NewBatch() + defer batch.Close() + + oldVal, closer, err := e.db.Get(key) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return nil // delete of non-existent is a no-op + } + return err } - if ent != nil && ent.GetResourceId() != "" { - keys = append(keys, encodeGrantByEntitlementResourceIndexKey( - ent.GetResourceTypeId(), ent.GetResourceId(), ext)) + if err := e.deleteGrantIndexesRaw(batch, "", oldVal); err != nil { + closer.Close() + return err } - if princ != nil { - keys = append(keys, encodeGrantByPrincipalIndexKey( - princ.GetResourceTypeId(), princ.GetResourceId(), ext)) - keys = append(keys, encodeGrantByPrincipalResourceTypeIndexKey( - princ.GetResourceTypeId(), ext)) + closer.Close() + + if err := batch.Delete(key, nil); err != nil { + return err + } + return batch.Commit(writeOpts(e.opts.durability)) +} + +// grantIndexKeys returns the secondary-index keys for r. Folded families +// (by_entitlement, by_entitlement_resource, and by_principal_resource_type) are +// served by primary or by_principal prefix scans and are deliberately not +// written. +func grantIndexKeys(r *v3.GrantRecord) [][]byte { + id, err := grantIdentityFromRecord(r) + if err != nil { + return nil } + keys := make([][]byte, 0, 3) + keys = append(keys, encodeGrantByPrincipalIdentityIndexKey(id)) if r.GetNeedsExpansion() { - keys = append(keys, encodeGrantByNeedsExpansionIndexKey(ext)) + keys = append(keys, encodeGrantByNeedsExpansionIdentityIndexKey(id)) + } + if sh := r.GetSourceScopeHash(); sh != "" { + keys = append(keys, encodeGrantBySourceScopeIndexKey(sh, id)) } return keys } @@ -481,34 +1023,68 @@ func (e *Engine) writeGrantIndexes(batch *pebble.Batch, r *v3.GrantRecord) error // index set and conditions MUST stay in lockstep with grantIndexKeys. // Returns the (possibly grown) scratch buffer for the next record. func (e *Engine) writeGrantIndexesScratch(batch *pebble.Batch, r *v3.GrantRecord, scratch []byte) ([]byte, error) { - ent := r.GetEntitlement() - princ := r.GetPrincipal() - ext := r.GetExternalId() + id, err := grantIdentityFromRecord(r) + if err != nil { + return scratch, err + } + return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), r.GetSourceScopeHash(), scratch) +} - if ent != nil && princ != nil { - scratch = appendGrantByEntitlementIndexKey(scratch[:0], ent.GetEntitlementId(), princ.GetResourceTypeId(), princ.GetResourceId(), ext) - if err := batch.Set(scratch, nil, nil); err != nil { - return scratch, err - } +// markDeferredIdxPending arms the deferred by_principal rebuild, durably. +// The atomic flag serves in-process checks; the engine-meta marker survives +// a process restart so a resumed sync still runs the rebuild at EndSync +// even when the resume itself writes nothing (see +// encodeDeferredIdxPendingKey). The CAS keeps the durable write to one per +// arming, off the per-record hot path. +func (e *Engine) markDeferredIdxPending() error { + if !e.deferredIdxPending.CompareAndSwap(false, true) { + return nil } - if ent != nil && ent.GetResourceId() != "" { - scratch = appendGrantByEntitlementResourceIndexKey(scratch[:0], ent.GetResourceTypeId(), ent.GetResourceId(), ext) - if err := batch.Set(scratch, nil, nil); err != nil { - return scratch, err - } + if err := e.db.Set(encodeDeferredIdxPendingKey(), nil, pebble.Sync); err != nil { + // Roll the CAS back so a retried write attempts the durable marker + // again. Leaving the flag armed with no durable key would make the + // in-process EndSync build the index (flag true) while a + // crash+resume silently skipped it (key absent) — exactly the + // divergence the durable marker exists to close. + e.deferredIdxPending.Store(false) + return fmt.Errorf("arm deferred index marker: %w", err) } - if princ != nil { - scratch = appendGrantByPrincipalIndexKey(scratch[:0], princ.GetResourceTypeId(), princ.GetResourceId(), ext) - if err := batch.Set(scratch, nil, nil); err != nil { - return scratch, err - } - scratch = appendGrantByPrincipalResourceTypeIndexKey(scratch[:0], princ.GetResourceTypeId(), ext) + return nil +} + +// clearDeferredIdxPending drops both forms of the marker after a successful +// rebuild. Runs under the write barrier so the flag reset and durable +// delete can't interleave with a concurrent writer's markDeferredIdxPending +// (which would leave the atomic flag armed but the durable marker deleted, +// or vice versa). +func (e *Engine) clearDeferredIdxPending() error { + // AllowSealed: runs inside EndSync's sealed finalize window, right + // after the deferred build (see Adapter.EndSync). + return e.withWriteAllowSealed(func() error { + e.deferredIdxPending.Store(false) + return e.db.Delete(encodeDeferredIdxPendingKey(), pebble.Sync) + }) +} + +// writeGrantIndexesForIdentityScratch writes the inline-maintained index keys +// for one grant. by_principal is never written inline: it is scattered +// relative to the entitlement-first write order, so it is always rebuilt as +// one sorted SST at EndSync (deferredIdxPending → BuildDeferredGrantIndexes). +// by_needs_expansion and by_source_scope are written inline: both share the +// entitlement-first ordering of the primary keyspace, so their writes stay +// sorted. +func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, sourceScopeHash string, scratch []byte) ([]byte, error) { + if err := e.markDeferredIdxPending(); err != nil { + return scratch, err + } + if needsExpansion { + scratch = appendGrantByNeedsExpansionIdentityIndexKey(scratch[:0], id) if err := batch.Set(scratch, nil, nil); err != nil { return scratch, err } } - if r.GetNeedsExpansion() { - scratch = appendGrantByNeedsExpansionIndexKey(scratch[:0], ext) + if sourceScopeHash != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) if err := batch.Set(scratch, nil, nil); err != nil { return scratch, err } @@ -516,43 +1092,37 @@ func (e *Engine) writeGrantIndexesScratch(batch *pebble.Batch, r *v3.GrantRecord return scratch, nil } -// deleteGrantIndexesScratch is deleteGrantIndexesRaw that encodes the -// delete keys into a single reused scratch buffer. value is the prior -// record's marshaled bytes (borrowed from the caller's pebble Get -// closer, which must outlive this call). The delete set MUST stay in -// lockstep with deleteGrantIndexesRaw. Returns the (possibly grown) -// scratch buffer. +// deleteGrantIndexesScratch is the overwrite-cleanup counterpart of +// writeGrantIndexesForIdentityScratch, encoding delete keys into a reused +// scratch buffer. value is the prior record's marshaled bytes (borrowed from +// the caller's pebble Get closer, which must outlive this call). by_principal +// is not deleted inline for the same reason it is not written inline: the +// whole family is excised and rebuilt from the primary keyspace at EndSync, +// which also clears any stale entries an overwrite would have left. Returns +// the (possibly grown) scratch buffer. func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID string, value, scratch []byte) ([]byte, error) { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) if err != nil { return scratch, err } - if entID != "" && principalRT != "" && principalID != "" { - scratch = appendGrantByEntitlementIndexKey(scratch[:0], entID, principalRT, principalID, externalID) - if err := batch.Delete(scratch, nil); err != nil { - return scratch, err - } + if entID == "" || entRT == "" || entRID == "" || principalRT == "" || principalID == "" { + return scratch, nil } - if entRID != "" { - scratch = appendGrantByEntitlementResourceIndexKey(scratch[:0], entRT, entRID, externalID) - if err := batch.Delete(scratch, nil); err != nil { - return scratch, err - } + id := grantIdentity{ + entitlement: entitlementIdentityFromParts(entRT, entRID, entID), + principalTypeID: principalRT, + principalID: principalID, } - if principalRT != "" && principalID != "" { - scratch = appendGrantByPrincipalIndexKey(scratch[:0], principalRT, principalID, externalID) - if err := batch.Delete(scratch, nil); err != nil { - return scratch, err - } - scratch = appendGrantByPrincipalResourceTypeIndexKey(scratch[:0], principalRT, externalID) + scratch = appendGrantByNeedsExpansionIdentityIndexKey(scratch[:0], id) + if err := batch.Delete(scratch, nil); err != nil { + return scratch, err + } + if sourceScopeHash != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) if err := batch.Delete(scratch, nil); err != nil { return scratch, err } } - scratch = appendGrantByNeedsExpansionIndexKey(scratch[:0], externalID) - if err := batch.Delete(scratch, nil); err != nil { - return scratch, err - } return scratch, nil } @@ -580,11 +1150,19 @@ func (e *Engine) IterateGrants(ctx context.Context, yield func(*v3.GrantRecord) return iter.Error() } -// IterateGrantsByEntitlement iterates the by_entitlement index for -// the given entitlement_id, yielding each grant in encoded principal- -// key order. yield returns false to stop. +// IterateGrantsByEntitlement iterates the primary grant keyspace for the given +// raw entitlement id, yielding each grant in encoded principal-key order. +// The id resolves through the bare-id lookup; an id matching no entitlement +// iterates nothing. yield returns false to stop. func (e *Engine) IterateGrantsByEntitlement(ctx context.Context, entitlementID string, yield func(*v3.GrantRecord) bool) error { - indexPrefix := encodeGrantByEntitlementPrefix(entitlementID) + entID, err := e.resolveGrantScanEntitlementIdentity(ctx, entitlementID) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return nil + } + return err + } + indexPrefix := encodeGrantPrimaryEntitlementPrefix(entID) iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: indexPrefix, UpperBound: upperBoundOf(indexPrefix), @@ -594,28 +1172,8 @@ func (e *Engine) IterateGrantsByEntitlement(ctx context.Context, entitlementID s } defer iter.Close() for iter.First(); iter.Valid(); iter.Next() { - // Index key tail is the grant's external_id. Need to decode it - // to look up the primary record. The tail is the last - // tuple-encoded component in the index key. - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { - continue - } - key := encodeGrantKey(externalID) - val, closer, err := e.db.Get(key) - if err != nil { - if errors.Is(err, pebble.ErrNotFound) { - // Index entry references a primary that's gone — a - // transient orphan during overwrite, or a real - // consistency issue. Skip; fsck reconciles. - continue - } - return err - } r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { + if err := unmarshalRecord(iter.Value(), r); err != nil { return fmt.Errorf("iterate by entitlement: %w", err) } if !yield(r) { @@ -637,24 +1195,26 @@ func (e *Engine) IterateGrantsByPrincipal(ctx context.Context, principalRT, prin } defer iter.Close() for iter.First(); iter.Valid(); iter.Next() { - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { + components, ok := decodeTupleComponents(iter.Key(), indexPrefix, 4) + if !ok { continue } - key := encodeGrantKey(externalID) - val, closer, err := e.db.Get(key) + r, err := getGrantByIdentity(ctx, e.db, grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: components[0], + resourceID: components[1], + stripped: components[2] == idFlagStripped, + tail: components[3], + }, + principalTypeID: principalRT, + principalID: principalID, + }) if err != nil { if errors.Is(err, pebble.ErrNotFound) { continue } return err } - r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { - return err - } if !yield(r) { return nil } @@ -662,12 +1222,11 @@ func (e *Engine) IterateGrantsByPrincipal(ctx context.Context, principalRT, prin return iter.Error() } -// IterateGrantsByPrincipalResourceType iterates the by-principal-RT -// index. Yields each grant whose principal carries the given -// resource_type, in encoded external_id order. Stops when yield -// returns false. +// IterateGrantsByPrincipalResourceType iterates the by_principal index narrowed +// to a principal resource type. Yields each grant whose principal carries the +// given resource_type. Stops when yield returns false. func (e *Engine) IterateGrantsByPrincipalResourceType(ctx context.Context, principalRT string, yield func(*v3.GrantRecord) bool) error { - indexPrefix := encodeGrantByPrincipalResourceTypePrefix(principalRT) + indexPrefix := encodeGrantByPrincipalResourceTypeIdentityPrefix(principalRT) iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: indexPrefix, UpperBound: upperBoundOf(indexPrefix), @@ -677,22 +1236,24 @@ func (e *Engine) IterateGrantsByPrincipalResourceType(ctx context.Context, princ } defer iter.Close() for iter.First(); iter.Valid(); iter.Next() { - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { + components, ok := decodeTupleComponents(iter.Key(), indexPrefix, 5) + if !ok { continue } - key := encodeGrantKey(externalID) - val, closer, err := e.db.Get(key) + r, err := getGrantByIdentity(ctx, e.db, grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: components[1], + resourceID: components[2], + stripped: components[3] == idFlagStripped, + tail: components[4], + }, + principalTypeID: principalRT, + principalID: components[0], + }) if err != nil { if errors.Is(err, pebble.ErrNotFound) { continue } - return err - } - r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { return fmt.Errorf("iterate by principal_rt: %w", err) } if !yield(r) { @@ -720,24 +1281,24 @@ func (e *Engine) IterateGrantsByNeedsExpansion(ctx context.Context, yield func(* } defer iter.Close() for iter.First(); iter.Valid(); iter.Next() { - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { + components, ok := decodeTupleComponents(iter.Key(), indexPrefix, 6) + if !ok { continue } - key := encodeGrantKey(externalID) - val, closer, err := e.db.Get(key) + r, err := getGrantByIdentity(ctx, e.db, grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: components[0], + resourceID: components[1], + stripped: components[2] == idFlagStripped, + tail: components[3], + }, + principalTypeID: components[4], + principalID: components[5], + }) if err != nil { if errors.Is(err, pebble.ErrNotFound) { - // Orphan: index entry without a primary. Skip; - // fsck reconciles. continue } - return err - } - r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { return fmt.Errorf("iterate needs_expansion: %w", err) } if !yield(r) { @@ -764,46 +1325,88 @@ func (e *Engine) IterateGrantsByNeedsExpansion(ctx context.Context, yield func(* // success or ("", "", false) if the tail is empty, malformed, or has // fewer than two components. func decodeTwoTupleComponents(key, prefix []byte) (string, string, bool) { - if len(key) <= len(prefix) { - return "", "", false - } - tail := key[len(prefix):] - first, next, ok := codec.DecodeTupleStringAlias(tail, 0) - if !ok || next >= len(tail) { - return "", "", false - } - // next points at the separator byte; skip it. - second, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + components, ok := decodeTupleComponents(key, prefix, 2) if !ok { return "", "", false } - return string(first), string(second), true + return components[0], components[1], true } -// lastTupleComponent returns the decoded string of the LAST tuple -// component in an index key relative to its prefix. Walks separator- -// delimited components and returns whichever one trails. Returns the -// empty string if the key doesn't extend past the prefix or the tail -// is malformed. -func lastTupleComponent(key, prefix []byte) string { - if len(key) <= len(prefix) { - return "" +func decodeGrantIdentityKey(key []byte) (grantIdentity, bool) { + prefix := []byte{versionV3, typeGrant, 0x00} + return decodeGrantIdentityTail(key, prefix) +} + +func decodeTupleComponents(key, prefix []byte, want int) ([]string, bool) { + if want <= 0 || len(key) <= len(prefix) { + return nil, false } tail := key[len(prefix):] - var last []byte + out := make([]string, 0, want) off := 0 - for off <= len(tail) { + for off <= len(tail) && len(out) < want { decoded, next, ok := codec.DecodeTupleStringAlias(tail, off) if !ok { - return "" + return nil, false } - last = decoded + out = append(out, string(decoded)) if next >= len(tail) { break } - // next is the separator byte position; advance past it to - // the next component. off = next + 1 } - return string(last) + if len(out) != want { + return nil, false + } + return out, true +} + +func decodeGrantIdentityTail(key, prefix []byte) (grantIdentity, bool) { + if len(key) <= len(prefix) { + return grantIdentity{}, false + } + tail := key[len(prefix):] + off := 0 + nextString := func() (string, bool) { + decoded, next, ok := codec.DecodeTupleStringAlias(tail, off) + if !ok { + return "", false + } + off = next + 1 + return string(decoded), true + } + entRT, ok := nextString() + if !ok { + return grantIdentity{}, false + } + entRID, ok := nextString() + if !ok { + return grantIdentity{}, false + } + entFlag, ok := nextString() + if !ok { + return grantIdentity{}, false + } + entTail, ok := nextString() + if !ok { + return grantIdentity{}, false + } + principalRT, ok := nextString() + if !ok { + return grantIdentity{}, false + } + principalID, ok := nextString() + if !ok { + return grantIdentity{}, false + } + return grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: entRT, + resourceID: entRID, + stripped: entFlag == idFlagStripped, + tail: entTail, + }, + principalTypeID: principalRT, + principalID: principalID, + }, true } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_for_entitlements.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_for_entitlements.go index 0e5f3d56..90fdfb1d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_for_entitlements.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_for_entitlements.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "hash/crc32" - "sort" "github.com/cockroachdb/pebble/v2" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" @@ -25,10 +24,10 @@ import ( // varint(entitlement_index) || varint(intra_cursor_len) || // intra_cursor_bytes || crc32(list_checksum) // -// list_checksum is crc32 over the sorted entitlement IDs in the -// request — if the caller changes the entitlement list between -// pages we detect the mismatch and restart from the beginning -// rather than silently mis-paginate. +// list_checksum is crc32 over the entitlement IDs in REQUEST +// ORDER — the cursor resumes by positional index, so a reorder is +// as fatal as a drop/add; any change to the list (including order) +// restarts from the beginning rather than silently mis-paginating. func (a *Adapter) ListGrantsForEntitlements( ctx context.Context, req *reader_v2.GrantsReaderServiceListGrantsForEntitlementsRequest, @@ -64,10 +63,16 @@ func (a *Adapter) ListGrantsForEntitlements( EntitlementLoop: for i := startIdx; i < len(ents); i++ { - entID := ents[i].GetId() - if entID == "" { + if ents[i].GetId() == "" { continue } + entID, err := a.entitlementIdentityForRequest(ctx, ents[i]) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue // unknown entitlement → no grants + } + return nil, err + } intraCursor := "" if i == startIdx { intraCursor = startIntra @@ -86,12 +91,11 @@ EntitlementLoop: for _, rec := range records { out = append(out, V3GrantToV2(rec)) if len(out) == limit { - p := rec.GetPrincipal() - lastIntra = encodeCursor(encodeGrantByEntitlementIndexKey( - entID, - p.GetResourceTypeId(), p.GetResourceId(), - rec.GetExternalId(), - )) + id, err := grantIdentityFromRecord(rec) + if err != nil { + return nil, err + } + lastIntra = encodeCursor(encodeGrantIdentityKey(id)) brokeEarly = true break } @@ -121,19 +125,18 @@ EntitlementLoop: }.Build(), nil } -// entitlementListChecksum hashes the sorted entitlement ID set so -// it is stable across client-side reorderings. If the caller drops -// or adds an entitlement between pages the checksum changes and -// decodeBatchCursor rejects the cursor. +// entitlementListChecksum hashes the entitlement ID list IN REQUEST +// ORDER. The cursor resumes by positional index into the list, so a +// reordering is just as fatal to the resume as a drop or an add — a +// sorted (order-insensitive) checksum would bless a reorder while the +// index resumed over a different entitlement, re-returning some and +// silently skipping others. Any change to the list, including order, +// changes the checksum and decodeBatchCursor restarts from the +// beginning. func entitlementListChecksum(ents []*v2.Entitlement) uint32 { - ids := make([]string, 0, len(ents)) - for _, e := range ents { - ids = append(ids, e.GetId()) - } - sort.Strings(ids) h := crc32.NewIEEE() - for _, id := range ids { - _, _ = h.Write([]byte(id)) + for _, e := range ents { + _, _ = h.Write([]byte(e.GetId())) _, _ = h.Write([]byte{0}) } return h.Sum32() @@ -182,7 +185,10 @@ func decodeBatchCursor(token string, currentChecksum uint32) (int, string, error return 0, "", errors.New("ListGrantsForEntitlements: cursor missing intra length") } raw = raw[n:] - if uint64(len(raw)) < lenU+4 { + // Guard each term separately: lenU is attacker-controlled and lenU+4 + // can wrap around uint64, which would pass a combined comparison and + // panic on the slice below. + if lenU > uint64(len(raw)) || uint64(len(raw))-lenU < 4 { return 0, "", errors.New("ListGrantsForEntitlements: cursor truncated") } intra := string(raw[:lenU]) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go new file mode 100644 index 00000000..e63e72a3 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go @@ -0,0 +1,161 @@ +package pebble + +import ( + "sort" + "sync" + + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +// expandedGrantImmutableAnnotations is the shared one-element annotation list +// every synthesized grant record carries. Marshal only reads it, so one slice +// serves all records (the Any inside is already a package singleton). +var expandedGrantImmutableAnnotations = []*anypb.Any{expandedGrantImmutableAnnotationAny} + +// fillSynthGrantRecord populates a REUSED GrantRecord for one synthesized +// grant. The hot write loops marshal ~50M records per whale expansion and the +// marshaler retains nothing, so one record struct per loop replaces one heap +// allocation per row. +// +// INVARIANT: every field the synthesized writers emit must be assigned here +// unconditionally — a conditionally-set field would leak the previous row's +// value through the reused struct. Two fields are intentionally NOT set +// because the writers hand-encode them onto the wire around the base +// marshal: external_id (field 2, prepended by +// appendSynthGrantExternalIDWire) and sources (field 9, appended by +// appendGrantSourcesWire). Expansion/NeedsExpansion stay zero for +// synthesized grants. +// +// SourceScopeHash is cleared unconditionally, for two reasons: synthesized +// (expander-derived) grants are never part of a source-cache scope, and — +// load-bearing for appendGrantSourcesWire — an empty proto3 scalar emits no +// bytes, which keeps sources (field 9) the highest field on the wire so the +// hand encoder's append-after-base-marshal stays canonical. See +// TestGrantSourcesWireSchemaPin. +func fillSynthGrantRecord(r *v3.GrantRecord, rec *synthesizedGrantRecord, now *timestamppb.Timestamp) { + r.SetEntitlement(rec.entitlement) + r.SetPrincipal(rec.principal) + r.SetAnnotations(expandedGrantImmutableAnnotations) + r.SetDiscoveredAt(now) + r.SetSourceScopeHash("") +} + +// grantExternalIDFieldTag is the wire tag for GrantRecord.external_id (2). +var grantExternalIDFieldTag = protowire.EncodeTag(2, protowire.BytesType) + +// appendSynthGrantExternalIDWire appends GrantRecord.external_id (field 2) +// carrying the legacy concat public id — byte-identical to what the SDK's +// NewGrantID emits — built directly on the wire so no per-row string is +// materialized (54M+ concat strings per whale expansion showed up as a +// top-3 allocation site). external_id is the lowest-numbered field the +// synthesized writers emit (sync_id is never set), so emitting it BEFORE +// the base marshal of the remaining fields preserves the canonical +// ascending field order; byte equality with a fresh reflective marshal is +// pinned by TestFillSynthGrantRecordReuse. +func appendSynthGrantExternalIDWire(dst []byte, rec *synthesizedGrantRecord) []byte { + entID := rec.entitlement.GetEntitlementId() + n := len(entID) + 1 + len(rec.id.principalTypeID) + 1 + len(rec.id.principalID) + dst = protowire.AppendVarint(dst, grantExternalIDFieldTag) + dst = protowire.AppendVarint(dst, uint64(n)) // #nosec G115 -- n is a small positive length. + dst = append(dst, entID...) + dst = append(dst, ':') + dst = append(dst, rec.id.principalTypeID...) + dst = append(dst, ':') + dst = append(dst, rec.id.principalID...) + return dst +} + +// Wire tags for the hand-encoded GrantRecord.sources map field. A protobuf +// map field is encoded as a repeated entry message: key is entry field 1, +// value is entry field 2. +var ( + grantSourcesFieldTag = protowire.EncodeTag(9, protowire.BytesType) // GrantRecord.sources + grantSourceKeyTag = protowire.EncodeTag(1, protowire.BytesType) // entry key (string) + grantSourceValTag = protowire.EncodeTag(2, protowire.BytesType) // entry value (GrantSourceRecord) + grantSourceDirectTag = protowire.EncodeTag(4, protowire.VarintType) // GrantSourceRecord.is_direct +) + +// appendGrantSourcesWire appends the GrantRecord.sources map field (field 9) +// to dst, byte-for-byte identical to what proto.MarshalOptions{Deterministic: +// true} produces for the equivalent map[string]*GrantSourceRecord — entries +// sorted by key, key and value fields both emitted unconditionally, and the +// GrantSourceRecord value carrying only is_direct (the synthesized-grant +// writers never set the other fields). Duplicate entitlement IDs collapse +// last-wins, matching the map construction the reflective path used. +// +// This exists because the reflective deterministic map marshal +// (appendMapDeterministic) allocates via reflect.MapKeys and sorts per record; +// at 50M+ synthesized grants per expansion that was a measurable share of both +// CPU and allocations. sources must be the highest-numbered populated field on +// the record (it is: 9) so appending after the base marshal preserves the +// canonical ascending field order. Byte equality with the reflective marshal +// is pinned by TestAppendGrantSourcesWireMatchesDeterministicProto; it matters +// because the codec equivalence harness asserts equal records produce equal +// bytes. +// +// scratch is a reusable buffer for the sorted copy of sources (the caller's +// slice is not mutated); the (possibly grown) scratch is returned for reuse. +func appendGrantSourcesWire(dst []byte, scratch batonGrant.Sources, sources batonGrant.Sources) ([]byte, batonGrant.Sources) { + if len(sources) == 0 { + return dst, scratch + } + scratch = append(scratch[:0], sources...) + sort.SliceStable(scratch, func(i, j int) bool { + return scratch[i].EntitlementID < scratch[j].EntitlementID + }) + for i, src := range scratch { + // Last occurrence of a duplicate key wins (stable sort keeps the + // original order within equal keys). + if i+1 < len(scratch) && scratch[i+1].EntitlementID == src.EntitlementID { + continue + } + valLen := 0 + if src.IsDirect { + valLen = 2 // is_direct tag byte + varint(1) + } + keyLen := len(src.EntitlementID) + entryLen := 1 + protowire.SizeVarint(uint64(keyLen)) + keyLen + + 1 + protowire.SizeVarint(uint64(valLen)) + valLen + dst = protowire.AppendVarint(dst, grantSourcesFieldTag) + dst = protowire.AppendVarint(dst, uint64(entryLen)) // #nosec G115 -- entryLen is a small positive length. + dst = protowire.AppendVarint(dst, grantSourceKeyTag) + dst = protowire.AppendVarint(dst, uint64(keyLen)) + dst = append(dst, src.EntitlementID...) + dst = protowire.AppendVarint(dst, grantSourceValTag) + dst = protowire.AppendVarint(dst, uint64(valLen)) + if src.IsDirect { + dst = protowire.AppendVarint(dst, grantSourceDirectTag) + dst = protowire.AppendVarint(dst, 1) + } + } + return dst, scratch +} + +// synthRecordsPool recycles the per-flush []synthesizedGrantRecord the +// adapter builds for the engine's synthesized-grant write paths. Each whale +// flush is ~250K records (~40MB); without reuse that backing array was the +// single largest flat allocation in the expansion profile. +var synthRecordsPool = sync.Pool{ + New: func() any { return new([]synthesizedGrantRecord) }, +} + +func getSynthRecords() *[]synthesizedGrantRecord { + ptr := synthRecordsPool.Get().(*[]synthesizedGrantRecord) + *ptr = (*ptr)[:0] + return ptr +} + +// putSynthRecords zeroes the used prefix (releasing the proto/message +// pointers records hold) and returns the buffer to the pool. Callers must +// guarantee the engine did not retain the slice — all Put/ingest paths copy +// key/value bytes out before returning. +func putSynthRecords(ptr *[]synthesizedGrantRecord) { + clear(*ptr) + *ptr = (*ptr)[:0] + synthRecordsPool.Put(ptr) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_format.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_format.go new file mode 100644 index 00000000..77f9fbc9 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_format.go @@ -0,0 +1,129 @@ +package pebble + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + + "github.com/cockroachdb/pebble/v2" + c1zv3 "github.com/conductorone/baton-sdk/pb/c1/c1z/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// id-index format stamps. 0 (no stamp) is the legacy external-id key +// layout; 1 was an interim escaped-identity layout that never shipped +// (dev artifacts only) and is rejected loudly; 2 is the current +// raw-component identity layout (flag/tail byte-prefix compression, see +// identity.go). +const ( + idIndexFormatStructuredV1 uint32 = 1 + idIndexFormatRawTupleV2 uint32 = 2 + + idIndexFormatCurrent = idIndexFormatRawTupleV2 +) + +func encodeIDIndexFormatKey() []byte { + buf := make([]byte, 0, 2+len("grant_entitlement_id_format")) + buf = append(buf, versionV3, typeEngineMeta) + return codec.AppendTupleStrings(buf, "grant_entitlement_id_format") +} + +// encodeDeferredIdxPendingKey is the durable marker that grant writes have +// skipped the inline by_principal index and a deferred rebuild is owed. The +// in-memory deferredIdxPending flag alone cannot survive a process restart: +// a sync interrupted after its expansion writes and resumed in a fresh +// process would otherwise write nothing (idempotent re-run), never re-arm +// the flag, and EndSync would skip the rebuild — saving a "finished" c1z +// whose by_principal index misses every deferred-written grant. +func encodeDeferredIdxPendingKey() []byte { + buf := make([]byte, 0, 2+len("deferred_grant_idx_pending")) + buf = append(buf, versionV3, typeEngineMeta) + return codec.AppendTupleStrings(buf, "deferred_grant_idx_pending") +} + +func (e *Engine) readIDIndexFormat() (uint32, error) { + val, closer, err := e.db.Get(encodeIDIndexFormatKey()) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return 0, nil + } + return 0, err + } + defer closer.Close() + if len(val) != 4 { + return 0, fmt.Errorf("malformed id-index-format blob: %d bytes, want 4", len(val)) + } + return binary.BigEndian.Uint32(val), nil +} + +func (e *Engine) writeIDIndexFormat(version uint32) error { + var buf [4]byte + binary.BigEndian.PutUint32(buf[:], version) + return e.db.Set(encodeIDIndexFormatKey(), buf[:], pebble.Sync) +} + +func (e *Engine) verifyOrStampIDIndexFormat(ctx context.Context) error { + version, err := e.readIDIndexFormat() + if err != nil { + return err + } + if version == idIndexFormatCurrent { + return nil + } + if version == idIndexFormatStructuredV1 { + return fmt.Errorf("pebble: unsupported grant/entitlement id-index format v%d (interim pre-release layout); regenerate this c1z", version) + } + if version != 0 { + return fmt.Errorf("pebble: unsupported grant/entitlement id-index format v%d", version) + } + empty, err := e.isDataKeyspaceEmpty() + if err != nil { + return err + } + if e.opts.readOnly { + if empty { + return nil + } + // A read-only engine cannot migrate in place, and current readers + // would silently miss every point lookup against legacy keys. The + // dotc1z store layer avoids this by running a writable migration + // pre-open on its unpacked temp copy; a direct read-only Open of a + // legacy dir must fail loudly instead. + return errors.New("pebble: this file uses the legacy grant/entitlement id layout; a writable open is required to migrate it") + } + if !empty { + return e.migrateIDIndexFormatToStructuredV1(ctx) + } + return e.writeIDIndexFormat(idIndexFormatCurrent) +} + +func (e *Engine) isDataKeyspaceEmpty() (bool, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: []byte{versionV3, typeResourceType}, + UpperBound: []byte{versionV3, typeEngineMeta}, + }) + if err != nil { + return false, err + } + defer iter.Close() + return !iter.First(), iter.Error() +} + +func (e *Engine) manifestIDIndexFormat() c1zv3.PebbleIdIndexFormat { + version, err := e.readIDIndexFormat() + if err != nil { + return c1zv3.PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED + } + switch version { + // The advisory manifest enum value predates the raw-tuple layout; since + // no file with the interim v1 layout ever shipped, STRUCTURED_V1 simply + // means "structured identity keys" (the current raw-tuple layout). + case idIndexFormatCurrent: + return c1zv3.PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_STRUCTURED_V1 + case 0: + return c1zv3.PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_LEGACY_EXTERNAL_ID + default: + return c1zv3.PebbleIdIndexFormat_PEBBLE_ID_INDEX_FORMAT_UNSPECIFIED + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go new file mode 100644 index 00000000..c592f89f --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go @@ -0,0 +1,616 @@ +package pebble + +import ( + "bufio" + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "sort" + "time" + + "github.com/cockroachdb/pebble/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func (e *Engine) migrateIDIndexFormatToStructuredV1(ctx context.Context) error { + start := time.Now() + l := ctxzap.Extract(ctx) + dir, err := os.MkdirTemp("", "pebble-id-index-migration-") + if err != nil { + return fmt.Errorf("id-index migration: mkdir temp: %w", err) + } + defer os.RemoveAll(dir) + + sortSem := make(chan struct{}, 4) + // 128MiB chunks (deferredIndexSpillChunkBytes), not the bulk import's + // 8MiB: the merge holds a 1MiB read buffer per chunk, so chunk size + // bounds the fan-in. At 8MiB a 150M-grant file would merge ~4,000 + // chunks (~4GB of buffers); at 128MiB it stays in the low hundreds. + // The arena freelist recycles the big chunks across all four sorters. + arenaFree := newSpillArenaFreeList(deferredIndexSpillChunkBytes, 6) + grantPrimary := newSpillSorter(dir, "grant-primary", sortSem, deferredIndexSpillChunkBytes) + entitlementPrimary := newSpillSorter(dir, "entitlement-primary", sortSem, deferredIndexSpillChunkBytes) + byPrincipal := newSpillSorter(dir, "idx-grant-by-principal", sortSem, deferredIndexSpillChunkBytes) + byNeedsExpansion := newSpillSorter(dir, "idx-grant-by-needs-expansion", sortSem, deferredIndexSpillChunkBytes) + sorters := []*spillSorter{grantPrimary, entitlementPrimary, byPrincipal, byNeedsExpansion} + for _, s := range sorters { + s.free = arenaFree + } + defer func() { + for _, s := range sorters { + s.abort() + } + }() + + entitlementRows, err := e.emitStructuredEntitlementMigration(ctx, entitlementPrimary) + if err != nil { + return err + } + grantRows, err := e.emitStructuredGrantMigration(ctx, grantPrimary) + if err != nil { + return err + } + scanDone := time.Now() + + type replacement struct { + name string + sorter *spillSorter + lower, upper []byte + } + replacements := []replacement{ + {name: "entitlement-primary", sorter: entitlementPrimary, lower: EntitlementLowerBound(), upper: EntitlementUpperBound()}, + {name: "grant-primary", sorter: grantPrimary, lower: GrantLowerBound(), upper: GrantUpperBound()}, + {name: "idx-grant-by-principal", sorter: byPrincipal, lower: GrantByPrincipalLowerBound(), upper: GrantByPrincipalUpperBound()}, + {name: "idx-grant-by-needs-expansion", sorter: byNeedsExpansion, lower: GrantByNeedsExpansionLowerBound(), upper: GrantByNeedsExpansionUpperBound()}, + } + + for _, r := range replacements { + var path string + var err error + if r.name == "grant-primary" { + path, err = finalizeGrantPrimaryMigrationSorter(ctx, dir, r.name, r.sorter, byPrincipal, byNeedsExpansion) + } else { + path, err = finalizeMigrationSorter(ctx, dir, r.name, r.sorter) + } + if err != nil { + return err + } + if err := e.replaceRangeWithSST(ctx, r.lower, r.upper, path); err != nil { + return fmt.Errorf("id-index migration: replace %s: %w", r.name, err) + } + } + + for _, r := range [][2][]byte{ + {EntitlementByResourceLowerBound(), EntitlementByResourceUpperBound()}, + {GrantByEntitlementLowerBound(), GrantByEntitlementUpperBound()}, + {GrantByPrincipalResourceTypeLowerBound(), GrantByPrincipalResourceTypeUpperBound()}, + {GrantByEntitlementResourceLowerBound(), GrantByEntitlementResourceUpperBound()}, + } { + if err := e.db.DeleteRange(r[0], r[1], pebble.Sync); err != nil { + return fmt.Errorf("id-index migration: delete dropped range: %w", err) + } + } + + if err := e.recomputeStatsAfterIDIndexMigration(ctx); err != nil { + return err + } + e.noteEntitlementKeyspaceWrite() + if err := e.writeIDIndexFormat(idIndexFormatCurrent); err != nil { + return err + } + e.migratedOnOpen = true + l.Info("id-index migration: structured identity re-key complete", + zap.Int64("grants", grantRows), + zap.Int64("entitlements", entitlementRows), + zap.Duration("scan", scanDone.Sub(start)), + zap.Duration("total", time.Since(start)), + ) + return nil +} + +func (e *Engine) recomputeStatsAfterIDIndexMigration(ctx context.Context) error { + var syncID string + err := e.IterateAllSyncRuns(ctx, func(r *v3.SyncRunRecord) bool { + syncID = r.GetSyncId() + return false + }) + if err != nil { + return fmt.Errorf("id-index migration: find sync for stats: %w", err) + } + if syncID == "" { + return nil + } + if err := e.PersistSyncStats(ctx, syncID); err != nil { + return fmt.Errorf("id-index migration: recompute sync stats: %w", err) + } + return nil +} + +// emitStructuredEntitlementMigration re-keys every legacy entitlement row +// under its structural identity, derived from the record's structured +// resource fields plus the byte-prefix compression rule — the same single +// derivation every reader and write path uses, so primary/index divergence +// is impossible by construction. Values are never modified: external ids +// are an external-consumer contract and migrate byte-identical. Rows whose +// resource ref is missing cannot be represented in the structured keyspace +// at all and are dropped with a warning. +func (e *Engine) emitStructuredEntitlementMigration(ctx context.Context, out *spillSorter) (int64, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: EntitlementLowerBound(), + UpperBound: EntitlementUpperBound(), + }) + if err != nil { + return 0, err + } + defer iter.Close() + var rows, skippedMissingResource int64 + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return rows, err + } + rt, rid, externalID, err := scanEntitlementIdentityFieldsRaw(iter.Value()) + if err != nil { + return rows, fmt.Errorf("id-index migration: scan entitlement: %w", err) + } + if rt == "" || rid == "" { + skippedMissingResource++ + continue + } + id := entitlementIdentityFromParts(rt, rid, externalID) + if err := out.add(encodeEntitlementIdentityKey(id), iter.Value()); err != nil { + return rows, err + } + rows++ + } + if skippedMissingResource > 0 { + ctxzap.Extract(ctx).Warn("id-index migration: dropped legacy entitlements with no resource ref; they cannot be keyed in the structured layout", + zap.Int64("dropped", skippedMissingResource), + ) + } + return rows, iter.Error() +} + +// emitStructuredGrantMigration re-keys every legacy grant row under its +// structural identity from the record's ref fields, with the same +// no-value-rewrite contract as emitStructuredEntitlementMigration. Rows +// missing entitlement or principal ref fields cannot be represented in the +// structured keyspace and are dropped with a warning. +func (e *Engine) emitStructuredGrantMigration(ctx context.Context, primary *spillSorter) (int64, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: GrantLowerBound(), + UpperBound: GrantUpperBound(), + }) + if err != nil { + return 0, err + } + defer iter.Close() + // Periodic progress logging: the migration runs inside Open and can + // take minutes on very large files — a silent multi-minute open looks + // hung and invites an operator (or a startup probe) to kill it, + // restarting the migration from scratch. Mirrors the deferred index + // build's 15s cadence. + l := ctxzap.Extract(ctx) + start := time.Now() + lastLog := start + var rows, skippedMissingRefs int64 + for iter.First(); iter.Valid(); iter.Next() { + if rows&0xFFFF == 0 { + if err := ctx.Err(); err != nil { + return rows, err + } + if now := time.Now(); now.Sub(lastLog) >= 15*time.Second { + l.Info("id-index migration: re-keying grants", + zap.Int64("rows", rows), + zap.Duration("elapsed", now.Sub(start)), + ) + lastLog = now + } + } + entRT, entRID, entID, principalRT, principalID, _, _, err := scanGrantIndexFieldsRaw(iter.Value()) + if err != nil { + return rows, fmt.Errorf("id-index migration: scan grant: %w", err) + } + if entRT == "" || entRID == "" || entID == "" || principalRT == "" || principalID == "" { + skippedMissingRefs++ + continue + } + id := grantIdentity{ + entitlement: entitlementIdentityFromParts(entRT, entRID, entID), + principalTypeID: principalRT, + principalID: principalID, + } + if err := primary.add(encodeGrantIdentityKey(id), iter.Value()); err != nil { + return rows, err + } + rows++ + } + if skippedMissingRefs > 0 { + ctxzap.Extract(ctx).Warn("id-index migration: dropped legacy grants with missing entitlement/principal refs; they cannot be keyed in the structured layout", + zap.Int64("dropped", skippedMissingRefs), + ) + } + return rows, iter.Error() +} + +func finalizeMigrationSorter(ctx context.Context, dir, name string, sorter *spillSorter) (string, error) { + chunks, err := sorter.finalize() + if err != nil { + return "", err + } + if len(chunks) == 0 { + return "", nil + } + path := filepath.Join(dir, name+".sst") + if err := mergeSortedSpillChunksToSST(ctx, path, name, chunks); err != nil { + return "", err + } + return path, nil +} + +func (e *Engine) replaceRangeWithSST(ctx context.Context, lower, upper []byte, path string) error { + if path == "" { + return e.db.DeleteRange(lower, upper, pebble.Sync) + } + _, err := e.db.IngestAndExcise(ctx, []string{path}, nil, nil, pebble.KeyRange{Start: lower, End: upper}) + return err +} + +func finalizeGrantPrimaryMigrationSorter(ctx context.Context, dir, name string, sorter, byPrincipal, byNeedsExpansion *spillSorter) (string, error) { + chunks, err := sorter.finalize() + if err != nil { + return "", err + } + if len(chunks) == 0 { + return "", nil + } + path := filepath.Join(dir, name+".sst") + if err := mergeGrantPrimaryMigrationChunksToSST(ctx, path, name, chunks, byPrincipal, byNeedsExpansion); err != nil { + return "", err + } + return path, nil +} + +func mergeGrantPrimaryMigrationChunksToSST(ctx context.Context, sstPath, name string, chunks []string, byPrincipal, byNeedsExpansion *spillSorter) error { + readers := make([]*os.File, 0, len(chunks)) + defer func() { + for _, r := range readers { + _ = r.Close() + } + }() + bufReaders := make([]*bufio.Reader, len(chunks)) + keyBufs := make([][]byte, len(chunks)) + valBufs := make([][]byte, len(chunks)) + h := &spillChunkHeap{} + var lenBuf [4]byte + for i, chunk := range chunks { + f, err := os.Open(chunk) // #nosec G304 - staged under migration temp dir. + if err != nil { + return err + } + readers = append(readers, f) + bufReaders[i] = bufio.NewReaderSize(f, bulkSpillBufferSize) + ok, err := readSpillEntry(bufReaders[i], &keyBufs[i], &valBufs[i], &lenBuf) + if err != nil { + return err + } + if ok { + h.push(spillChunkItem{chunkIdx: i, key: append([]byte(nil), keyBufs[i]...), val: append([]byte(nil), valBufs[i]...)}) + } + } + + w, err := newBulkSSTWriter(filepath.Dir(sstPath), name) + if err != nil { + return err + } + success := false + defer func() { + if !success { + _ = w.finish() + _ = os.Remove(w.path) + } + }() + + var duplicateGroups, duplicateRowsMerged int64 + var idxKeyScratch []byte + var rowsProcessed int64 + for len(*h) > 0 { + rowsProcessed++ + if rowsProcessed&0xFFFF == 0 { + if err := ctx.Err(); err != nil { + return err + } + } + // Heap items are owned copies (pushed via append([]byte(nil), ...) + // in the initial fill and advanceMigrationChunk), so no re-copy is + // needed to hold them across chunk advances. + first := h.pop() + key := first.key + values := [][]byte{first.val} + if err := advanceMigrationChunk(h, bufReaders, keyBufs, valBufs, &lenBuf, first.chunkIdx); err != nil { + return err + } + for len(*h) > 0 && bytes.Equal((*h)[0].key, key) { + item := h.pop() + values = append(values, item.val) + if err := advanceMigrationChunk(h, bufReaders, keyBufs, valBufs, &lenBuf, item.chunkIdx); err != nil { + return err + } + } + if len(values) > 1 { + duplicateGroups++ + duplicateRowsMerged += int64(len(values) - 1) + } + merged, err := mergeDuplicateGrantValues(values) + if err != nil { + return fmt.Errorf("id-index migration: merge duplicate grant key %x: %w", key, err) + } + if err := w.add(key, merged); err != nil { + return err + } + // The identity is already fully encoded in the primary key being + // written, so both index keys derive from it at the byte level — + // no proto unmarshal per row (the dominant per-row cost at whale + // scale and beyond). by_principal is the pinned segment + // permutation; by_needs_expansion shares the primary's exact tail + // (header swap only). Only the flag needs a value read, via a + // single-field shallow scan. + idxKey, ok := appendGrantByPrincipalKeyFromPrimary(idxKeyScratch[:0], key) + idxKeyScratch = idxKey + if !ok { + return fmt.Errorf("id-index migration: grant primary key %x did not decode as a 6-segment identity", key) + } + if err := byPrincipal.add(idxKey, nil); err != nil { + return err + } + needsExpansion, err := scanGrantNeedsExpansionRaw(merged) + if err != nil { + return fmt.Errorf("id-index migration: scan needs_expansion for key %x: %w", key, err) + } + if needsExpansion { + idxKeyScratch = append(idxKeyScratch[:0], versionV3, typeIndex, idxGrantByNeedsExpansion) + // The primary's sep+tail IS the identity index tail, byte-identical + // (pinned by TestNeedsExpansionKeyHeaderSpliceFromPrimary). + idxKeyScratch = append(idxKeyScratch, key[2:]...) + if err := byNeedsExpansion.add(idxKeyScratch, nil); err != nil { + return err + } + } + } + if err := w.finish(); err != nil { + return err + } + if w.path != sstPath { + if err := os.Rename(w.path, sstPath); err != nil { + return err + } + } + if duplicateGroups > 0 { + ctxzap.Extract(ctx).Warn("id-index migration: merged legacy grant rows that share one structural identity", + zap.Int64("identities_with_duplicates", duplicateGroups), + zap.Int64("rows_merged_away", duplicateRowsMerged), + ) + } + success = true + return nil +} + +func advanceMigrationChunk(h *spillChunkHeap, readers []*bufio.Reader, keyBufs, valBufs [][]byte, lenBuf *[4]byte, idx int) error { + ok, err := readSpillEntry(readers[idx], &keyBufs[idx], &valBufs[idx], lenBuf) + if err != nil { + return err + } + if ok { + h.push(spillChunkItem{chunkIdx: idx, key: append([]byte(nil), keyBufs[idx]...), val: append([]byte(nil), valBufs[idx]...)}) + } + return nil +} + +// mergeDuplicateGrantValues folds N marshaled grant rows that share one +// structural identity into a single record. Callers hand values over in +// FOLD ORDER, which is not stable: the in-place migration and the bulk +// import both pop duplicate groups off a k-way heap whose tie-break is +// chunk index — spill-chunk creation order, which varies run to run. +// +// INVARIANT: the merge of every field must therefore be fold-order +// independent, so the merged record's bytes are reproducible regardless of +// which duplicate arrives first. external_id/discovered_at use the +// commutative winner rule (recordIdentityInfoWins), needs_expansion is an +// OR, sources merge by map key, expansion ids union-sort, and annotations +// sort the merged union by (TypeUrl, Value). A new field added here must +// come with an order-independent merge rule. +func mergeDuplicateGrantValues(values [][]byte) ([]byte, error) { + if len(values) == 1 { + return values[0], nil + } + var out *v3.GrantRecord + for _, value := range values { + var rec v3.GrantRecord + if err := unmarshalRecord(value, &rec); err != nil { + return nil, err + } + if out == nil { + cp := proto.Clone(&rec).(*v3.GrantRecord) + out = cp + continue + } + mergeGrantRecordInto(out, &rec) + } + return marshalRecord(out) +} + +func mergeGrantRecordInto(dst, src *v3.GrantRecord) { + if grantRecordIdentityInfoWins(src, dst) { + dst.SetExternalId(src.GetExternalId()) + dst.SetDiscoveredAt(src.GetDiscoveredAt()) + } + dst.SetNeedsExpansion(dst.GetNeedsExpansion() || src.GetNeedsExpansion()) + dst.SetAnnotations(mergeGrantAnnotations(dst.GetAnnotations(), src.GetAnnotations())) + dst.SetSources(mergeGrantSources(dst.GetSources(), src.GetSources())) + dst.SetExpansion(mergeGrantExpansion(dst.GetExpansion(), src.GetExpansion())) +} + +func grantRecordIdentityInfoWins(candidate, incumbent *v3.GrantRecord) bool { + return recordIdentityInfoWins( + candidate.GetDiscoveredAt(), candidate.GetExternalId(), + incumbent.GetDiscoveredAt(), incumbent.GetExternalId(), + ) +} + +// recordIdentityInfoWins is the shared winner rule for duplicate-identity +// rows: earliest discovered_at wins; ties break to the smallest external id. +func recordIdentityInfoWins(candidateDiscovered *timestamppb.Timestamp, candidateExternalID string, incumbentDiscovered *timestamppb.Timestamp, incumbentExternalID string) bool { + switch { + case candidateDiscovered != nil && incumbentDiscovered == nil: + return true + case candidateDiscovered == nil && incumbentDiscovered != nil: + return false + case candidateDiscovered != nil && incumbentDiscovered != nil: + ct, it := candidateDiscovered.AsTime(), incumbentDiscovered.AsTime() + if !ct.Equal(it) { + return ct.Before(it) + } + } + return candidateExternalID < incumbentExternalID +} + +func mergeGrantAnnotations(a, b []*anypb.Any) []*anypb.Any { + if len(a) == 0 { + return b + } + if len(b) == 0 { + return a + } + out := make([]*anypb.Any, 0, len(a)+len(b)) + seen := make(map[string]struct{}, len(a)+len(b)) + add := func(items []*anypb.Any) { + for _, item := range items { + if item == nil { + continue + } + key := item.GetTypeUrl() + "\x00" + string(item.GetValue()) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, item) + } + } + add(a) + add(b) + // Sort the UNION (single-side inputs above return as-is, preserving the + // winner's stored order): every other merged field is fold-order + // independent (recordIdentityInfoWins, map merges, unionSortedStrings), + // but duplicate-identity values arrive in heap order — the bulk import's + // chunkIdx tie-break reflects spill-chunk creation order, which is not + // stable run to run. Without this, the merged artifact bytes would not + // be reproducible. + sort.Slice(out, func(i, j int) bool { + if out[i].GetTypeUrl() != out[j].GetTypeUrl() { + return out[i].GetTypeUrl() < out[j].GetTypeUrl() + } + return string(out[i].GetValue()) < string(out[j].GetValue()) + }) + return out +} + +// mergeGrantSources merges by map key; colliding values fold field-wise +// with commutative+associative rules only (the mergeDuplicateGrantValues +// invariant): IsDirect is an OR, and each ref field resolves to the +// lexicographically smallest non-empty value ("" is the identity). A +// "direct side's fields win" preference would NOT satisfy the invariant: +// IsDirect ORs into the fold accumulator and loses provenance, so a field +// donated by a non-direct value would masquerade as direct in later fold +// steps and different arrival orders could produce different bytes. In +// practice no writer populates the ref fields today (translate_v2 and the +// synth encoder set only IsDirect), so any rule is byte-neutral for real +// inputs — this one stays correct if a writer ever starts setting them. +func mergeGrantSources(a, b map[string]*v3.GrantSourceRecord) map[string]*v3.GrantSourceRecord { + if len(a) == 0 { + return b + } + if len(b) == 0 { + return a + } + out := make(map[string]*v3.GrantSourceRecord, len(a)+len(b)) + for key, value := range a { + out[key] = proto.Clone(value).(*v3.GrantSourceRecord) + } + for key, value := range b { + if value == nil { + continue + } + existing := out[key] + if existing == nil { + out[key] = proto.Clone(value).(*v3.GrantSourceRecord) + continue + } + out[key] = v3.GrantSourceRecord_builder{ + ResourceTypeId: minNonEmptyString(existing.GetResourceTypeId(), value.GetResourceTypeId()), + ResourceId: minNonEmptyString(existing.GetResourceId(), value.GetResourceId()), + EntitlementId: minNonEmptyString(existing.GetEntitlementId(), value.GetEntitlementId()), + IsDirect: existing.GetIsDirect() || value.GetIsDirect(), + }.Build() + } + return out +} + +// minNonEmptyString returns the lexicographically smallest non-empty +// argument, or "" when both are empty. Commutative and associative, with +// "" as the identity — folding it over N values yields the global smallest +// non-empty value regardless of order. +func minNonEmptyString(a, b string) string { + switch { + case a == "": + return b + case b == "": + return a + case a < b: + return a + default: + return b + } +} + +func mergeGrantExpansion(a, b *v3.GrantExpandableRecord) *v3.GrantExpandableRecord { + if a == nil { + return b + } + if b == nil { + return a + } + return v3.GrantExpandableRecord_builder{ + EntitlementIds: unionSortedStrings(a.GetEntitlementIds(), b.GetEntitlementIds()), + ResourceTypeIds: unionSortedStrings(a.GetResourceTypeIds(), b.GetResourceTypeIds()), + Shallow: a.GetShallow() && b.GetShallow(), + }.Build() +} + +func unionSortedStrings(a, b []string) []string { + if len(a) == 0 && len(b) == 0 { + return nil + } + seen := make(map[string]struct{}, len(a)+len(b)) + for _, v := range a { + if v != "" { + seen[v] = struct{}{} + } + } + for _, v := range b { + if v != "" { + seen[v] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for v := range seen { + out = append(out, v) + } + sort.Strings(out) + return out +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/identity.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/identity.go new file mode 100644 index 00000000..6a911cec --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/identity.go @@ -0,0 +1,143 @@ +package pebble + +import ( + "fmt" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" +) + +// Structural identity. +// +// External ids are an external-consumer contract: baton emits them +// byte-identical to what connectors produced and never interprets their +// grammar. Internally, identity is derived ONLY from the structured fields +// records already carry (resource refs, entitlement refs, principal refs) — +// the lossy ":"-joined id string is never parsed to decide identity. +// +// The one place identity touches the id bytes is pure compression, not +// interpretation: when an entitlement's external id literally begins with +// the exact bytes `resource_type_id + ":" + resource_id + ":"` (the shape +// SDK-generated ids always have), the redundant prefix is stripped from the +// stored key tail and a one-byte flag records that it was. The mapping +// (external_id) ↔ (stripped, tail) is bijective given (rt, rid): stripped +// identities reconstruct as `rt:rid:tail`, opaque identities are exactly +// the ids that do NOT start with that prefix, so the two ranges cannot +// collide and reconstruction is byte-exact. + +// entitlementIdentity is the structural identity of an entitlement: +// (resource_type_id, resource_id, flag, tail). See the package comment +// above for the flag/tail compression contract. +type entitlementIdentity struct { + resourceTypeID string + resourceID string + // stripped reports that the external id begins with the exact bytes + // resourceTypeID + ":" + resourceID + ":" and tail holds the remainder; + // otherwise tail holds the entire raw external id. + stripped bool + tail string +} + +// Key-component encodings of the stripped flag. Single printable bytes so +// the tuple codec never needs to escape them. +const ( + idFlagStripped = "1" + idFlagOpaque = "0" +) + +func (id entitlementIdentity) flagComponent() string { + if id.stripped { + return idFlagStripped + } + return idFlagOpaque +} + +// externalID reconstructs the exact raw external id this identity was +// derived from. +func (id entitlementIdentity) externalID() string { + if !id.stripped { + return id.tail + } + return id.resourceTypeID + ":" + id.resourceID + ":" + id.tail +} + +type grantIdentity struct { + entitlement entitlementIdentity + principalTypeID string + principalID string +} + +// externalID reconstructs the legacy public grant id shape +// (entitlement_id + ":" + principal_rt + ":" + principal_id) for this +// identity. This matches what the SDK's NewGrantID emits; connector-custom +// grant ids are stored on the record and take precedence when present. +func (id grantIdentity) externalID() string { + return id.entitlement.externalID() + ":" + id.principalTypeID + ":" + id.principalID +} + +// entitlementIdentityFromParts derives an entitlement identity from the +// structured resource components and the raw external id. Pure byte-prefix +// check — no grammar, no escaping, byte-exact round trip via externalID(). +func entitlementIdentityFromParts(resourceTypeID, resourceID, entitlementID string) entitlementIdentity { + prefixLen := len(resourceTypeID) + len(resourceID) + 2 + if len(entitlementID) >= prefixLen && + entitlementID[len(resourceTypeID)] == ':' && + entitlementID[prefixLen-1] == ':' && + entitlementID[:len(resourceTypeID)] == resourceTypeID && + entitlementID[len(resourceTypeID)+1:prefixLen-1] == resourceID { + return entitlementIdentity{ + resourceTypeID: resourceTypeID, + resourceID: resourceID, + stripped: true, + tail: entitlementID[prefixLen:], + } + } + return entitlementIdentity{ + resourceTypeID: resourceTypeID, + resourceID: resourceID, + tail: entitlementID, + } +} + +func entitlementIdentityFromRecord(r *v3.EntitlementRecord) (entitlementIdentity, error) { + if r == nil { + return entitlementIdentity{}, fmt.Errorf("entitlement identity: nil record") + } + res := r.GetResource() + if res == nil || res.GetResourceTypeId() == "" || res.GetResourceId() == "" { + return entitlementIdentity{}, fmt.Errorf("entitlement identity: missing resource") + } + return entitlementIdentityFromParts(res.GetResourceTypeId(), res.GetResourceId(), r.GetExternalId()), nil +} + +func grantIdentityFromRecord(r *v3.GrantRecord) (grantIdentity, error) { + if r == nil { + return grantIdentity{}, fmt.Errorf("grant identity: nil record") + } + ent := r.GetEntitlement() + if ent == nil || ent.GetResourceTypeId() == "" || ent.GetResourceId() == "" || ent.GetEntitlementId() == "" { + return grantIdentity{}, fmt.Errorf("grant identity: missing entitlement") + } + princ := r.GetPrincipal() + if princ == nil || princ.GetResourceTypeId() == "" || princ.GetResourceId() == "" { + return grantIdentity{}, fmt.Errorf("grant identity: missing principal") + } + return grantIdentity{ + entitlement: entitlementIdentityFromParts(ent.GetResourceTypeId(), ent.GetResourceId(), ent.GetEntitlementId()), + principalTypeID: princ.GetResourceTypeId(), + principalID: princ.GetResourceId(), + }, nil +} + +// publicGrantRecordID returns the id V3GrantToV2 emits for a record: the +// stored external id when the connector provided one, otherwise the +// reconstructed legacy concat (what the SDK would have emitted). +func publicGrantRecordID(r *v3.GrantRecord) string { + if ext := r.GetExternalId(); ext != "" { + return ext + } + id, err := grantIdentityFromRecord(r) + if err != nil { + return "" + } + return id.externalID() +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go index f1f175f7..e166f8dc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go @@ -45,7 +45,11 @@ func (e *Engine) PutGrantRecordsIfNewer(ctx context.Context, records ...*v3.Gran if r == nil { continue } - key := encodeGrantKey(r.GetExternalId()) + id, err := grantIdentityFromRecord(r) + if err != nil { + return err + } + key := encodeGrantIdentityKey(id) oldVal, closer, getErr := e.db.Get(key) switch { case getErr == nil: @@ -161,7 +165,11 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v if r == nil { continue } - key := encodeEntitlementKey(r.GetExternalId()) + id, err := entitlementIdentityFromRecord(r) + if err != nil { + return err + } + key := encodeEntitlementIdentityKey(id) oldVal, closer, getErr := e.db.Get(key) switch { case getErr == nil: @@ -174,11 +182,16 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v closer.Close() continue } - if err := e.deleteEntitlementIndexesRaw(batch, r.GetExternalId(), oldVal); err != nil { - closer.Close() - return err - } + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" && oldScope != r.GetSourceScopeHash() { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } case errors.Is(getErr, pebble.ErrNotFound): default: return fmt.Errorf("PutEntitlementRecordsIfNewer: get: %w", getErr) @@ -190,15 +203,21 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v if err := batch.Set(key, val, nil); err != nil { return err } - if err := e.writeEntitlementIndexes(batch, r); err != nil { - return err + if sh := r.GetSourceScopeHash(); sh != "" { + if err := batch.Set(encodeEntitlementBySourceScopeIndexKey(sh, id), nil, nil); err != nil { + return err + } } written++ } if written == 0 { return nil } - return batch.Commit(writeOpts(e.opts.durability)) + if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { + return err + } + e.noteEntitlementKeyspaceWrite() + return nil }) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go index 8d49e199..a9ca13c6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go @@ -63,7 +63,16 @@ const ( typeIndex byte = 0x07 typeCounter byte = 0x08 typeSession byte = 0x09 - typeEngineMeta byte = 0xFF + // typeSourceCache holds one SourceCacheEntryRecord per + // (row_kind, scope_hash) — the source-cache replay manifest (see + // proto/c1/connector/v2/annotation_source_cache.proto). Placed + // after typeSession so the clone path's counter/session excise + // span ([typeCounter, upperBoundOf(typeSession))) leaves it in the + // clone: a cloned sync artifact stays usable as a replay source. + // ResetForNewSync's [typeResourceType, typeEngineMeta) span wipes + // it with the rest of the sync-scoped data. + typeSourceCache byte = 0x0A + typeEngineMeta byte = 0xFF ) // Index-discriminator bytes (second byte after typeIndex). One byte @@ -72,12 +81,19 @@ const ( // preceded it. const ( idxResourceByParent byte = 0x01 - idxEntitlementByResource byte = 0x02 - idxGrantByEntitlement byte = 0x03 + idxEntitlementByResource byte = 0x02 // retired: served by entitlement primary key prefixes. + idxGrantByEntitlement byte = 0x03 // retired: grant primary keys are entitlement-first. idxGrantByPrincipal byte = 0x04 idxGrantByNeedsExpansion byte = 0x05 - idxGrantByPrincipalResourceType byte = 0x06 - idxGrantByEntitlementResource byte = 0x07 + idxGrantByPrincipalResourceType byte = 0x06 // retired: served by idxGrantByPrincipal prefix scans. + idxGrantByEntitlementResource byte = 0x07 // retired: served by grant primary entitlement-resource prefix scans. + // by_source_scope families: partial indexes over records whose + // source_scope_hash is non-empty (source-cache replay). Tails are + // identity tuples per the keys.go convention, so replay derives + // each primary key from the index key without touching the record. + idxGrantBySourceScope byte = 0x08 + idxEntitlementBySourceScope byte = 0x09 + idxResourceBySourceScope byte = 0x0A ) // --- Grant --- @@ -102,14 +118,33 @@ func appendGrantKey(dst []byte, externalID string) []byte { return codec.AppendTupleStrings(dst, externalID) } +func encodeGrantIdentityKey(id grantIdentity) []byte { + return appendGrantIdentityKey(make([]byte, 0, 128), id) +} + +func appendGrantIdentityKey(dst []byte, id grantIdentity) []byte { + dst = append(dst, versionV3, typeGrant) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings( + dst, + id.entitlement.resourceTypeID, + id.entitlement.resourceID, + id.entitlement.flagComponent(), + id.entitlement.tail, + id.principalTypeID, + id.principalID, + ) +} + // encodeGrantPrefix returns the by-type prefix for iterating all // grants. Paired with encodeGrantKey. func encodeGrantPrefix() []byte { return []byte{versionV3, typeGrant} } -// encodeGrantByEntitlementIndexKey is the by_entitlement secondary -// index on GrantRecord: +// encodeGrantByEntitlementIndexKey is the retired by_entitlement secondary +// index on GrantRecord. New writes use entitlement-first primary grant keys +// instead; this helper remains for old debug/test tooling. // // v3 | typeIndex | idxGrantByEntitlement | 0x00 | // entitlement_id | 0x00 | @@ -129,46 +164,60 @@ func appendGrantByEntitlementIndexKey(dst []byte, entitlementID, principalRT, pr return codec.AppendTupleStrings(dst, entitlementID, principalRT, principalID, externalID) } -// encodeGrantByPrincipalIndexKey: -// -// v3 | typeIndex | idxGrantByPrincipal | 0x00 | -// principal_resource_type | 0x00 | -// principal_resource_id | 0x00 | -// external_id -// -// Paired with encodeGrantByPrincipalPrefix (by-value prefix, with -// trailing sep). -func encodeGrantByPrincipalIndexKey(principalRT, principalID, externalID string) []byte { - return appendGrantByPrincipalIndexKey(make([]byte, 0, 64), principalRT, principalID, externalID) -} - -func appendGrantByPrincipalIndexKey(dst []byte, principalRT, principalID, externalID string) []byte { +// retired: served by entitlement primary key prefixes. +// func encodeGrantByEntitlementIdentityIndexKey(id grantIdentity) []byte { +// return appendGrantByEntitlementIdentityIndexKey(make([]byte, 0, 128), id) +// } + +// retired +// appendGrantByEntitlementIdentityIndexKey encodes the retired by_entitlement +// identity index. Do not use for new writes; primary grant keys are already +// entitlement-first. +// func appendGrantByEntitlementIdentityIndexKey(dst []byte, id grantIdentity) []byte { +// dst = append(dst, versionV3, typeIndex, idxGrantByEntitlement) +// dst = codec.AppendTupleSeparator(dst) +// return codec.AppendTupleStrings( +// dst, +// id.entitlement.resourceTypeID, +// id.entitlement.resourceID, +// id.entitlement.kind, +// id.entitlement.name, +// id.principalTypeID, +// id.principalID, +// ) +// } + +func encodeGrantByPrincipalIdentityIndexKey(id grantIdentity) []byte { + return appendGrantByPrincipalIdentityIndexKey(make([]byte, 0, 128), id) +} + +func appendGrantByPrincipalIdentityIndexKey(dst []byte, id grantIdentity) []byte { dst = append(dst, versionV3, typeIndex, idxGrantByPrincipal) dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleStrings(dst, principalRT, principalID, externalID) -} - -// encodeGrantByEntitlementPrefix is the by-value prefix for "all -// grants with this entitlement_id". Trailing separator is -// load-bearing — see the keys.go convention doc. -func encodeGrantByEntitlementPrefix(entitlementID string) []byte { - buf := make([]byte, 0, 32+len(entitlementID)) - buf = append(buf, versionV3, typeIndex, idxGrantByEntitlement) + return codec.AppendTupleStrings( + dst, + id.principalTypeID, + id.principalID, + id.entitlement.resourceTypeID, + id.entitlement.resourceID, + id.entitlement.flagComponent(), + id.entitlement.tail, + ) +} + +func encodeGrantPrimaryEntitlementPrefix(id entitlementIdentity) []byte { + buf := make([]byte, 0, 128) + buf = append(buf, versionV3, typeGrant) buf = codec.AppendTupleSeparator(buf) - buf = codec.AppendTupleStrings(buf, entitlementID) + buf = codec.AppendTupleStrings(buf, id.resourceTypeID, id.resourceID, id.flagComponent(), id.tail) return codec.AppendTupleSeparator(buf) } -// encodeGrantByEntitlementPrincipalPrefix is the by-value prefix for -// "all grants with this entitlement_id and principal". It reuses the -// existing by_entitlement index tail: -// -// entitlement_id | principal_resource_type | principal_resource_id | external_id -func encodeGrantByEntitlementPrincipalPrefix(entitlementID, principalRT, principalID string) []byte { - buf := make([]byte, 0, 32+len(entitlementID)+len(principalRT)+len(principalID)) - buf = append(buf, versionV3, typeIndex, idxGrantByEntitlement) +func encodeGrantPrimaryEntitlementResourcePrefix(resourceTypeID, resourceID string) []byte { + buf := make([]byte, 0, 64) + buf = append(buf, versionV3, typeGrant) buf = codec.AppendTupleSeparator(buf) - buf = codec.AppendTupleStrings(buf, entitlementID, principalRT, principalID) + buf = codec.AppendTupleStrings(buf, resourceTypeID, resourceID) return codec.AppendTupleSeparator(buf) } @@ -193,42 +242,29 @@ func appendGrantByNeedsExpansionIndexKey(dst []byte, externalID string) []byte { return codec.AppendTupleStrings(dst, externalID) } -// encodeGrantByPrincipalResourceTypeIndexKey: by-principal-RT -// index. Closes the only O(G) full-scan path in the Reader -// (ListGrantsForResourceType, which previously walked the entire -// grant primary range and post-filtered). -// -// v3 | typeIndex | idxGrantByPrincipalResourceType | 0x00 | -// principal_resource_type | 0x00 | -// external_id -// -// Paired with encodeGrantByPrincipalResourceTypePrefix (by-value -// prefix, with trailing sep). -func encodeGrantByPrincipalResourceTypeIndexKey(principalRT, externalID string) []byte { - return appendGrantByPrincipalResourceTypeIndexKey(make([]byte, 0, 64), principalRT, externalID) +func encodeGrantByNeedsExpansionIdentityIndexKey(id grantIdentity) []byte { + return appendGrantByNeedsExpansionIdentityIndexKey(make([]byte, 0, 128), id) } -func appendGrantByPrincipalResourceTypeIndexKey(dst []byte, principalRT, externalID string) []byte { - dst = append(dst, versionV3, typeIndex, idxGrantByPrincipalResourceType) +func appendGrantByNeedsExpansionIdentityIndexKey(dst []byte, id grantIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxGrantByNeedsExpansion) dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleStrings(dst, principalRT, externalID) -} - -// encodeGrantByPrincipalResourceTypePrefix is the by-value prefix -// for "all grants whose principal has the given resource_type". -// Trailing sep is load-bearing — see keys.go convention. -func encodeGrantByPrincipalResourceTypePrefix(principalRT string) []byte { - buf := make([]byte, 0, 32+len(principalRT)) - buf = append(buf, versionV3, typeIndex, idxGrantByPrincipalResourceType) - buf = codec.AppendTupleSeparator(buf) - buf = codec.AppendTupleStrings(buf, principalRT) - return codec.AppendTupleSeparator(buf) + return codec.AppendTupleStrings( + dst, + id.entitlement.resourceTypeID, + id.entitlement.resourceID, + id.entitlement.flagComponent(), + id.entitlement.tail, + id.principalTypeID, + id.principalID, + ) } // encodeGrantByNeedsExpansionPrefix is the by-type prefix for all // grants that still need expansion processing. func encodeGrantByNeedsExpansionPrefix() []byte { - return []byte{versionV3, typeIndex, idxGrantByNeedsExpansion} + buf := []byte{versionV3, typeIndex, idxGrantByNeedsExpansion} + return codec.AppendTupleSeparator(buf) } // encodeGrantByPrincipalPrefix is the by-value prefix for "all @@ -242,47 +278,128 @@ func encodeGrantByPrincipalPrefix(principalRT, principalID string) []byte { return codec.AppendTupleSeparator(buf) } -// encodeGrantByEntitlementResourceIndexKey is the by_entitlement_resource -// secondary index on GrantRecord. Indexes grants by the -// resource side of their entitlement (i.e. the resource the -// entitlement is on — the group/role/app/etc., NOT the principal). +func encodeGrantByPrincipalResourceTypeIdentityPrefix(principalRT string) []byte { + buf := make([]byte, 0, 32+len(principalRT)) + buf = append(buf, versionV3, typeIndex, idxGrantByPrincipal) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, principalRT) + return codec.AppendTupleSeparator(buf) +} + +// --- Source-cache (by_source_scope indexes + entry keyspace) --- + +// encodeGrantBySourceScopeIndexKey is the partial by_source_scope index +// over grants whose SourceScopeHash is non-empty: // -// v3 | typeIndex | idxGrantByEntitlementResource | 0x00 | -// ent_resource_type | 0x00 | -// ent_resource_id | 0x00 | -// external_id (tail element for index-row uniqueness) +// v3 | typeIndex | idxGrantBySourceScope | 0x00 | +// scope_hash | 0x00 | +// ent_rt | 0x00 | ent_rid | 0x00 | ent_flag | 0x00 | ent_tail | 0x00 | +// principal_rt | 0x00 | principal_id // -// Drives Adapter.ListGrants / ListWithAnnotationsForResourcePage when -// req.Resource is set — matches SQLite's `listGrantsGeneric` which -// filters on grants.resource_id / resource_type_id (the entitlement- -// side resource columns). The pre-existing by_principal index served -// the wrong semantic and produced silently-empty reads for callers -// that wanted "grants on this group" rather than "grants where this -// group is a principal". +// The tail after scope_hash is the grant identity tuple — byte-identical +// to the grant primary key's tail — so replay derives the primary key from +// the index key alone. Paired with encodeGrantBySourceScopePrefix +// (by-value prefix, with trailing sep). +func encodeGrantBySourceScopeIndexKey(scopeHash string, id grantIdentity) []byte { + return appendGrantBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) +} + +func appendGrantBySourceScopeIndexKey(dst []byte, scopeHash string, id grantIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxGrantBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings( + dst, + scopeHash, + id.entitlement.resourceTypeID, + id.entitlement.resourceID, + id.entitlement.flagComponent(), + id.entitlement.tail, + id.principalTypeID, + id.principalID, + ) +} + +// encodeGrantBySourceScopePrefix is the by-value prefix for "all grants +// stamped with this scope hash". Trailing sep is load-bearing — see the +// keys.go convention doc. +func encodeGrantBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxGrantBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeEntitlementBySourceScopeIndexKey: // -// Paired with encodeGrantByEntitlementResourcePrefix (by-value prefix, -// with trailing sep). -func encodeGrantByEntitlementResourceIndexKey(entRT, entRID, externalID string) []byte { - return appendGrantByEntitlementResourceIndexKey(make([]byte, 0, 64), entRT, entRID, externalID) +// v3 | typeIndex | idxEntitlementBySourceScope | 0x00 | +// scope_hash | 0x00 | rt | 0x00 | rid | 0x00 | flag | 0x00 | tail +// +// Tail is the entitlement identity tuple, byte-identical to the +// entitlement primary key's tail. Paired with +// encodeEntitlementBySourceScopePrefix. +func encodeEntitlementBySourceScopeIndexKey(scopeHash string, id entitlementIdentity) []byte { + return appendEntitlementBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) } -func appendGrantByEntitlementResourceIndexKey(dst []byte, entRT, entRID, externalID string) []byte { - dst = append(dst, versionV3, typeIndex, idxGrantByEntitlementResource) +func appendEntitlementBySourceScopeIndexKey(dst []byte, scopeHash string, id entitlementIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxEntitlementBySourceScope) dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleStrings(dst, entRT, entRID, externalID) + return codec.AppendTupleStrings(dst, scopeHash, id.resourceTypeID, id.resourceID, id.flagComponent(), id.tail) } -// encodeGrantByEntitlementResourcePrefix is the by-value prefix for -// "all grants whose entitlement is on this resource". Trailing sep is -// load-bearing — see keys.go convention. -func encodeGrantByEntitlementResourcePrefix(entRT, entRID string) []byte { - buf := make([]byte, 0, 32+len(entRT)+len(entRID)) - buf = append(buf, versionV3, typeIndex, idxGrantByEntitlementResource) +func encodeEntitlementBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxEntitlementBySourceScope) buf = codec.AppendTupleSeparator(buf) - buf = codec.AppendTupleStrings(buf, entRT, entRID) + buf = codec.AppendTupleStrings(buf, scopeHash) return codec.AppendTupleSeparator(buf) } +// encodeResourceBySourceScopeIndexKey: +// +// v3 | typeIndex | idxResourceBySourceScope | 0x00 | +// scope_hash | 0x00 | resource_type_id | 0x00 | resource_id +// +// Tail is the resource primary tuple. Paired with +// encodeResourceBySourceScopePrefix. +func encodeResourceBySourceScopeIndexKey(scopeHash, resourceTypeID, resourceID string) []byte { + return appendResourceBySourceScopeIndexKey(make([]byte, 0, 96), scopeHash, resourceTypeID, resourceID) +} + +func appendResourceBySourceScopeIndexKey(dst []byte, scopeHash, resourceTypeID, resourceID string) []byte { + dst = append(dst, versionV3, typeIndex, idxResourceBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, scopeHash, resourceTypeID, resourceID) +} + +func encodeResourceBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxResourceBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeSourceCacheEntryKey is the primary key for one source-cache +// manifest entry: +// +// v3 | typeSourceCache | 0x00 | row_kind | 0x00 | scope_hash +// +// Paired with encodeSourceCachePrefix (by-type prefix). +func encodeSourceCacheEntryKey(rowKind, scopeHash string) []byte { + buf := make([]byte, 0, 32+len(rowKind)+len(scopeHash)) + buf = append(buf, versionV3, typeSourceCache) + buf = codec.AppendTupleSeparator(buf) + return codec.AppendTupleStrings(buf, rowKind, scopeHash) +} + +// encodeSourceCachePrefix is the by-type prefix for all source-cache +// entries. +func encodeSourceCachePrefix() []byte { + return []byte{versionV3, typeSourceCache} +} + // --- ResourceType --- // encodeResourceTypeKey returns the primary key for a resource_type: @@ -359,6 +476,16 @@ func encodeEntitlementKey(externalID string) []byte { return codec.AppendTupleStrings(buf, externalID) } +func encodeEntitlementIdentityKey(id entitlementIdentity) []byte { + return appendEntitlementIdentityKey(make([]byte, 0, 96), id) +} + +func appendEntitlementIdentityKey(dst []byte, id entitlementIdentity) []byte { + dst = append(dst, versionV3, typeEntitlement) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, id.resourceTypeID, id.resourceID, id.flagComponent(), id.tail) +} + // encodeEntitlementPrefix is the by-type prefix for entitlements. func encodeEntitlementPrefix() []byte { return []byte{versionV3, typeEntitlement} @@ -371,19 +498,27 @@ func encodeEntitlementPrefix() []byte { // // Paired with encodeEntitlementByResourcePrefix (by-value prefix, // with trailing sep). -func encodeEntitlementByResourceIndexKey(resourceTypeID, resourceID, externalID string) []byte { - buf := make([]byte, 0, 64) - buf = append(buf, versionV3, typeIndex, idxEntitlementByResource) - buf = codec.AppendTupleSeparator(buf) - return codec.AppendTupleStrings(buf, resourceTypeID, resourceID, externalID) -} +// func encodeEntitlementByResourceIndexKey(resourceTypeID, resourceID, externalID string) []byte { +// buf := make([]byte, 0, 64) +// buf = append(buf, versionV3, typeIndex, idxEntitlementByResource) +// buf = codec.AppendTupleSeparator(buf) +// return codec.AppendTupleStrings(buf, resourceTypeID, resourceID, externalID) +// } // encodeEntitlementByResourcePrefix is the by-value prefix for "all // entitlements on (resource_type_id, resource_id)". Trailing sep is // load-bearing. -func encodeEntitlementByResourcePrefix(resourceTypeID, resourceID string) []byte { - buf := make([]byte, 0, 32+len(resourceTypeID)+len(resourceID)) - buf = append(buf, versionV3, typeIndex, idxEntitlementByResource) +// func encodeEntitlementByResourcePrefix(resourceTypeID, resourceID string) []byte { +// buf := make([]byte, 0, 32+len(resourceTypeID)+len(resourceID)) +// buf = append(buf, versionV3, typeIndex, idxEntitlementByResource) +// buf = codec.AppendTupleSeparator(buf) +// buf = codec.AppendTupleStrings(buf, resourceTypeID, resourceID) +// return codec.AppendTupleSeparator(buf) +// } + +func encodeEntitlementPrimaryResourcePrefix(resourceTypeID, resourceID string) []byte { + buf := make([]byte, 0, 64) + buf = append(buf, versionV3, typeEntitlement) buf = codec.AppendTupleSeparator(buf) buf = codec.AppendTupleStrings(buf, resourceTypeID, resourceID) return codec.AppendTupleSeparator(buf) @@ -458,6 +593,8 @@ func EntitlementByResourceUpperBound() []byte { return upperBoundOf(EntitlementB func GrantLowerBound() []byte { return encodeGrantPrefix() } func GrantUpperBound() []byte { return upperBoundOf(encodeGrantPrefix()) } +// GrantByEntitlementLowerBound returns the retired by_entitlement index range. +// Kept so cleanup/migration can delete old files' index entries. func GrantByEntitlementLowerBound() []byte { return []byte{versionV3, typeIndex, idxGrantByEntitlement} } @@ -473,6 +610,8 @@ func GrantByNeedsExpansionUpperBound() []byte { return upperBoundOf(GrantByNeedsExpansionLowerBound()) } +// GrantByPrincipalResourceTypeLowerBound returns a retired folded index range. +// Principal-resource-type scans are served by idxGrantByPrincipal prefixes. func GrantByPrincipalResourceTypeLowerBound() []byte { return []byte{versionV3, typeIndex, idxGrantByPrincipalResourceType} } @@ -480,6 +619,8 @@ func GrantByPrincipalResourceTypeUpperBound() []byte { return upperBoundOf(GrantByPrincipalResourceTypeLowerBound()) } +// GrantByEntitlementResourceLowerBound returns a retired folded index range. +// Entitlement-resource scans are served by primary grant key prefixes. func GrantByEntitlementResourceLowerBound() []byte { return []byte{versionV3, typeIndex, idxGrantByEntitlementResource} } @@ -487,6 +628,28 @@ func GrantByEntitlementResourceUpperBound() []byte { return upperBoundOf(GrantByEntitlementResourceLowerBound()) } +func GrantBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxGrantBySourceScope} +} +func GrantBySourceScopeUpperBound() []byte { return upperBoundOf(GrantBySourceScopeLowerBound()) } + +func EntitlementBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxEntitlementBySourceScope} +} +func EntitlementBySourceScopeUpperBound() []byte { + return upperBoundOf(EntitlementBySourceScopeLowerBound()) +} + +func ResourceBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxResourceBySourceScope} +} +func ResourceBySourceScopeUpperBound() []byte { + return upperBoundOf(ResourceBySourceScopeLowerBound()) +} + +func SourceCacheEntryLowerBound() []byte { return encodeSourceCachePrefix() } +func SourceCacheEntryUpperBound() []byte { return upperBoundOf(encodeSourceCachePrefix()) } + func AssetLowerBound() []byte { return encodeAssetPrefix() } func AssetUpperBound() []byte { return upperBoundOf(encodeAssetPrefix()) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go new file mode 100644 index 00000000..282c3a9c --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go @@ -0,0 +1,481 @@ +package pebble + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/cockroachdb/pebble/v2" +) + +// Bare-id lookups. +// +// Identity is structural; the public id string is a lossy join kept only as +// an external contract. Readers that receive nothing but an id string +// (Get/Delete by id — CLI and reader-API edges, never the sync hot path) +// recover the structural components here. Parsing is a query plan, not +// identity: the resolution rule is "exactly one row whose public id equals +// the query string wins" — zero matches is NotFound, more than one is an +// explicit ambiguity error, and a guess is never written. +// +// SAFETY CONTRACT — who may call what: +// +// - Sync and grant expansion must never depend on string resolution. +// Their list/scan calls carry structured refs (the expander fetches +// the entitlement record first; expanderStoreAdapter enforces this at +// the boundary), and grant deletes go through DeleteGrantByIdentityRefs. +// The one string the expansion path resolves is a GrantExpandable +// source-entitlement id — connectors hand those over as bare strings — +// and that resolves ONLY via the exact-match record map +// (resolveEntitlementIdentityByExternalID): byte-equality against +// stored external ids, same semantics as SQLite's external_id lookup, +// erroring loudly on the (new-layout-only) ambiguous case. +// - Combinatorial candidate probing — resolveGrantIdentityByExternalID +// and the grant-keyspace split fallback inside +// resolveGrantScanEntitlementIdentity — is reserved for interactive +// edges: reader-API Get/Delete-grant by id (provisioner --revoke-grant, +// the baton explorer) and list requests whose caller supplied only an +// id string (cmd/baton flags). There an ambiguity error is a usable +// answer for a human; in a sync it would be a correctness hazard. +// +// Entitlements resolve through a lazily built in-memory multimap from raw +// external id → identity, rebuilt when the entitlement keyspace changes +// (generation counter). The map is exact string-match semantics: identity +// reconstruction is bijective, so the reconstructed id in the map IS the +// stored external id. Entitlement cardinality is small relative to grants, +// so the map is memory-bounded and the one-time scan amortizes across +// lookups. +// +// Grants resolve by combinatorial candidate splitting of the concat shape +// `entitlement_id + ":" + principal_rt + ":" + principal_id`, probing each +// candidate identity with a point Get. Connector-custom grant ids with no +// concat shape (or whose splits match nothing) fall back to a full +// primary-keyspace scan matching the STORED external id — SQLite keyed +// rows by that id, so string reads must find them for parity. The scan is +// O(all grants) and acceptable ONLY because this path is a CLI/reader-API +// edge (see the safety contract above); sync and expansion never resolve +// grants by string. + +// maxGrantIDCandidates caps the combinatorial split enumeration for one +// grant-id lookup. Ids with enough colons to exceed it are unresolvable by +// string per policy ("if we can't resolve safely, error"). +const maxGrantIDCandidates = 4096 + +// maxBareIDColons caps the colon count a bare-id string may carry before +// the combinatorial machinery refuses outright: the split enumeration is +// O(colons²) (and each entitlement-scan fallback probe opens an iterator), +// so a pathological id — hostile or corrupt — must fail fast instead of +// grinding through billions of loop iterations that no context check can +// interrupt. Real ids carry a handful of colons; 64 is beyond any +// legitimate ARN-ish shape while keeping the worst case trivial. +const maxBareIDColons = 64 + +// noteEntitlementKeyspaceWrite invalidates the lazy entitlement id lookup +// map. Call after ANY mutation of the entitlement primary keyspace (batch +// writes, deletes, ingests, range replacements, resets). +func (e *Engine) noteEntitlementKeyspaceWrite() { + e.entIDLookupGen.Add(1) +} + +// entitlementIdentitiesForExternalID returns every entitlement identity +// whose raw external id equals externalID, via the lazily built map. +func (e *Engine) entitlementIdentitiesForExternalID(ctx context.Context, externalID string) ([]entitlementIdentity, error) { + gen := e.entIDLookupGen.Load() + e.entIDLookupMu.Lock() + defer e.entIDLookupMu.Unlock() + if e.entIDLookup == nil || e.entIDLookupBuiltGen != gen { + m, err := e.buildEntitlementIDLookup(ctx) + if err != nil { + return nil, err + } + e.entIDLookup = m + e.entIDLookupBuiltGen = gen + } + return e.entIDLookup[externalID], nil +} + +// buildEntitlementIDLookup scans the entitlement primary keyspace once and +// groups identities by their reconstructed (== stored) external id. Only +// keys are decoded; values are never touched. +func (e *Engine) buildEntitlementIDLookup(ctx context.Context) (map[string][]entitlementIdentity, error) { + prefix := encodeEntitlementPrefix() + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return nil, err + } + defer iter.Close() + m := map[string][]entitlementIdentity{} + var n int64 + for iter.First(); iter.Valid(); iter.Next() { + n++ + if n&0x3FF == 0 { + if err := ctx.Err(); err != nil { + return nil, err + } + } + id, ok := decodeEntitlementIdentityKey(iter.Key()) + if !ok { + continue + } + ext := id.externalID() + m[ext] = append(m[ext], id) + } + if err := iter.Error(); err != nil { + return nil, err + } + return m, nil +} + +// decodeEntitlementIdentityKey decodes an entitlement primary key back into +// its identity. Returns ok=false for keys that do not have the expected +// four-component shape. +func decodeEntitlementIdentityKey(key []byte) (entitlementIdentity, bool) { + prefix := []byte{versionV3, typeEntitlement, 0} + components, ok := decodeTupleComponents(key, prefix, 4) + if !ok { + return entitlementIdentity{}, false + } + return entitlementIdentity{ + resourceTypeID: components[0], + resourceID: components[1], + stripped: components[2] == idFlagStripped, + tail: components[3], + }, true +} + +// resolveEntitlementIdentityByExternalID applies the exactly-one rule to +// entitlementIdentitiesForExternalID: one match wins, zero is +// pebble.ErrNotFound, several is ErrAmbiguousExternalID. +func (e *Engine) resolveEntitlementIdentityByExternalID(ctx context.Context, externalID string) (entitlementIdentity, error) { + matches, err := e.entitlementIdentitiesForExternalID(ctx, externalID) + if err != nil { + return entitlementIdentity{}, err + } + switch len(matches) { + case 0: + return entitlementIdentity{}, pebble.ErrNotFound + case 1: + return matches[0], nil + default: + return entitlementIdentity{}, fmt.Errorf("%w: entitlement id %q matches %d records on different resources", + ErrAmbiguousExternalID, externalID, len(matches)) + } +} + +// resolveGrantScanEntitlementIdentity resolves an entitlement id string for +// GRANT-scoped scans (list/iterate grants of an entitlement). It first +// resolves through the entitlement record map; when the id matches no +// record (grants can exist for entitlements the store never saw), it falls +// back to direct byte-split candidates of the prefix shape, keeping a +// candidate only when the grant primary keyspace actually has rows under +// it. Exactly-one rule throughout. +func (e *Engine) resolveGrantScanEntitlementIdentity(ctx context.Context, entitlementID string) (entitlementIdentity, error) { + matches, err := e.entitlementIdentitiesForExternalID(ctx, entitlementID) + if err != nil { + return entitlementIdentity{}, err + } + switch { + case len(matches) == 1: + return matches[0], nil + case len(matches) > 1: + return entitlementIdentity{}, fmt.Errorf("%w: entitlement id %q matches %d records on different resources", + ErrAmbiguousExternalID, entitlementID, len(matches)) + } + // No entitlement record: probe direct (rt | rid | tail) splits against + // the grant primary keyspace. Each probe opens a (bounded, single-seek) + // iterator; the colon cap bounds the enumeration at C(maxBareIDColons, 2) + // = ~2k probes worst case, so no maxGrantIDCandidates-style cap is + // needed here — O(colons²) can never exceed it, unlike the grant + // path's O(colons⁴) direct splits. + if n := strings.Count(entitlementID, ":"); n > maxBareIDColons { + return entitlementIdentity{}, fmt.Errorf("%w: entitlement id has %d colons; too complex to resolve safely by string", ErrAmbiguousExternalID, n) + } + var hits []entitlementIdentity + for k := 0; k < len(entitlementID); k++ { + if entitlementID[k] != ':' { + continue + } + if err := ctx.Err(); err != nil { + return entitlementIdentity{}, err + } + for l := k + 1; l < len(entitlementID); l++ { + if entitlementID[l] != ':' { + continue + } + cand := entitlementIdentity{ + resourceTypeID: entitlementID[:k], + resourceID: entitlementID[k+1 : l], + stripped: true, + tail: entitlementID[l+1:], + } + nonEmpty, err := e.grantPrimaryPrefixNonEmpty(encodeGrantPrimaryEntitlementPrefix(cand)) + if err != nil { + return entitlementIdentity{}, err + } + if nonEmpty { + hits = append(hits, cand) + } + } + } + switch len(hits) { + case 0: + return entitlementIdentity{}, pebble.ErrNotFound + case 1: + return hits[0], nil + default: + return entitlementIdentity{}, fmt.Errorf("%w: entitlement id %q matches grants under %d distinct identities", + ErrAmbiguousExternalID, entitlementID, len(hits)) + } +} + +func (e *Engine) grantPrimaryPrefixNonEmpty(prefix []byte) (bool, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return false, err + } + defer iter.Close() + ok := iter.First() + if err := iter.Error(); err != nil { + return false, err + } + return ok, nil +} + +// resolveGrantIdentityByExternalID resolves a public grant id string to the +// single grant row whose public id equals it, by enumerating candidate +// splits of the concat shape and probing each candidate identity: +// +// - direct splits: boundaries for (rt, rid, tail, principal_rt, +// principal_id) — the stripped-entitlement shape, needing no +// entitlement record; +// - entitlement-resolved splits: boundaries for (entitlement_id, +// principal_rt, principal_id), with the entitlement id resolved through +// the lazy map — this covers opaque entitlement ids. +// +// A probe hit is exact: the candidate's reconstructed public id equals the +// query by construction, and the hit is counted only when the row's stored +// public id also equals the query (a connector-custom stored id addresses +// the row instead of the concat). Exactly one hit wins; zero is +// pebble.ErrNotFound; several is ErrAmbiguousExternalID. +func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID string) (grantIdentity, error) { + id, err := e.resolveGrantIdentityByCandidates(ctx, grantID) + if err == nil || !errors.Is(err, errNoGrantCandidateHits) { + return id, err + } + // Candidate probing proved nothing either way: the id may still be a + // connector-custom STORED external id (with or without colons), which + // only the O(all grants) scan can find. Interactive edges pay it; + // bounded callers (DeleteGrantRecordBounded) stop before this line. + return e.scanGrantIdentityByStoredExternalID(ctx, grantID) +} + +// errNoGrantCandidateHits reports that combinatorial candidate probing +// completed without a hit — distinct from pebble.ErrNotFound, which the +// full resolution reserves for "provably absent after the scan of last +// resort". Internal to the resolution layer. +var errNoGrantCandidateHits = errors.New("pebble: no grant candidate parse hit") + +// resolveGrantIdentityByCandidates is the bounded stage of grant-id +// resolution: combinatorial concat splits probed with point Gets, no +// keyspace scan. Returns errNoGrantCandidateHits when the shape yields no +// candidates (fewer than two colons) or every candidate missed. +func (e *Engine) resolveGrantIdentityByCandidates(ctx context.Context, grantID string) (grantIdentity, error) { + var colons []int + for i := 0; i < len(grantID); i++ { + if grantID[i] == ':' { + colons = append(colons, i) + } + } + if len(colons) < 2 { + // No concat shape to split: findable only by STORED external id. + return grantIdentity{}, errNoGrantCandidateHits + } + if len(colons) > maxBareIDColons { + return grantIdentity{}, fmt.Errorf("%w: grant id has %d colons; too complex to resolve safely by string", ErrAmbiguousExternalID, len(colons)) + } + + var candidates []grantIdentity + addCandidate := func(id grantIdentity) error { + if len(candidates) >= maxGrantIDCandidates { + return fmt.Errorf("%w: grant id %q has too many candidate parses", ErrAmbiguousExternalID, grantID) + } + candidates = append(candidates, id) + return nil + } + + // Entitlement-resolved splits: (entID | prt | pid). The entitlement + // lookup depends only on the first boundary, so it is hoisted out of + // the inner loop — one map probe per candidate entitlement id, not one + // per (i, j) pair. + for ii := 0; ii < len(colons); ii++ { + if err := ctx.Err(); err != nil { + return grantIdentity{}, err + } + i := colons[ii] + entMatches, err := e.entitlementIdentitiesForExternalID(ctx, grantID[:i]) + if err != nil { + return grantIdentity{}, err + } + if len(entMatches) == 0 { + continue + } + for jj := ii + 1; jj < len(colons); jj++ { + j := colons[jj] + for _, ent := range entMatches { + if err := addCandidate(grantIdentity{ + entitlement: ent, + principalTypeID: grantID[i+1 : j], + principalID: grantID[j+1:], + }); err != nil { + return grantIdentity{}, err + } + } + } + } + // Direct stripped splits: (rt | rid | tail | prt | pid). These need no + // entitlement record, covering grants whose entitlement row is absent. + // Bounded by maxBareIDColons⁴ in the worst case, which the colon cap + // above keeps trivial; the ctx check makes even that interruptible. + for kk := 0; kk < len(colons); kk++ { + if err := ctx.Err(); err != nil { + return grantIdentity{}, err + } + for ll := kk + 1; ll < len(colons); ll++ { + for ii := ll + 1; ii < len(colons); ii++ { + for jj := ii + 1; jj < len(colons); jj++ { + k, l, i, j := colons[kk], colons[ll], colons[ii], colons[jj] + if err := addCandidate(grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: grantID[:k], + resourceID: grantID[k+1 : l], + stripped: true, + tail: grantID[l+1 : i], + }, + principalTypeID: grantID[i+1 : j], + principalID: grantID[j+1:], + }); err != nil { + return grantIdentity{}, err + } + } + } + } + } + + seen := map[string]struct{}{} + var hits []grantIdentity + for _, cand := range candidates { + key := encodeGrantIdentityKey(cand) + if _, dup := seen[string(key)]; dup { + continue + } + seen[string(key)] = struct{}{} + val, closer, err := e.db.Get(key) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue + } + return grantIdentity{}, err + } + // Count the hit only when the row's PUBLIC id equals the query: a + // row with a connector-custom stored external id is addressed by + // that id, not by its concat reconstruction. + ext, serr := scanGrantExternalIDRaw(val) + closer.Close() + if serr != nil { + return grantIdentity{}, serr + } + if ext != "" && ext != grantID { + continue + } + hits = append(hits, cand) + } + switch len(hits) { + case 0: + return grantIdentity{}, errNoGrantCandidateHits + case 1: + return hits[0], nil + default: + return grantIdentity{}, fmt.Errorf("%w: grant id %q matches %d records", ErrAmbiguousExternalID, grantID, len(hits)) + } +} + +// scanGrantIdentityByStoredExternalID finds the grant whose STORED +// external id equals grantID by scanning the grant primary keyspace with +// a shallow per-value field read. This is the resolution of last resort +// for connector-custom ids that carry no reconstructible concat shape — +// SQLite keyed grant rows by exactly this id, so string reads must find +// them for reader parity. +// +// COST: O(all grants) per call, hit or miss. Every NotFound outcome ends +// here (absence is unprovable without the scan), and a successful custom-id +// hit scans to EOF anyway — the exactly-one rule keeps looking for a second +// match (early exit only on ambiguity). Only the external_id field is +// decoded per row. This is acceptable ONLY because every caller is an +// interactive one-shot: +// +// - reachable from the CLI/reader-API edge (provisioner --revoke-grant, +// explorer detail views via the storecache, which memoizes per id); +// the combinatorial prober above answers SDK-shaped ids without ever +// getting here; +// - sync and expansion never reach it: the syncer's only +// delete-grants-by-id loop prefers the store's DeleteGrantByRefs +// (grantByRefsDeleter in pkg/sync), which Pebble implements, so the +// string path there runs only on SQLite where external_id is indexed. +// +// If a real workload ever resolves many DISTINCT custom ids (each one a +// full scan — quadratic in aggregate), the escape hatch is a skinny +// by-custom-external-id index written at put time only for rows whose +// stored id differs from their concat reconstruction; SDK-shaped ids need +// no index entry, so the added write weight would be near zero. +// +// Exactly-one rule: zero matches is pebble.ErrNotFound, several is +// ErrAmbiguousExternalID. +func (e *Engine) scanGrantIdentityByStoredExternalID(ctx context.Context, grantID string) (grantIdentity, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: GrantLowerBound(), + UpperBound: GrantUpperBound(), + }) + if err != nil { + return grantIdentity{}, err + } + defer iter.Close() + var hits []grantIdentity + var scanned int64 + for iter.First(); iter.Valid(); iter.Next() { + scanned++ + if scanned&0x3FFF == 0 { + if err := ctx.Err(); err != nil { + return grantIdentity{}, err + } + } + ext, serr := scanGrantExternalIDRaw(iter.Value()) + if serr != nil { + return grantIdentity{}, serr + } + if ext != grantID { + continue + } + id, ok := decodeGrantIdentityKey(iter.Key()) + if !ok { + continue + } + hits = append(hits, id) + if len(hits) > 1 { + return grantIdentity{}, fmt.Errorf("%w: grant id %q matches %d records", ErrAmbiguousExternalID, grantID, len(hits)) + } + } + if err := iter.Error(); err != nil { + return grantIdentity{}, err + } + if len(hits) == 0 { + return grantIdentity{}, pebble.ErrNotFound + } + return hits[0], nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/manifest.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/manifest.go index a792ab77..9a7cdabd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/manifest.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/manifest.go @@ -22,9 +22,10 @@ func BuildManifest(encoding c1zstore.PayloadEncoding) (*c1zv3.C1ZManifestV3, err return nil, err } return c1zv3.C1ZManifestV3_builder{ - // PebbleManifestEngine ("pebble2"), not EnginePebble: pre-single-sync - // readers dispatch on this name and must reject the file instead of - // reading the sync_id-less keyspace as empty. See its doc comment. + // PebbleManifestEngine ("pebble3"), not EnginePebble: readers + // dispatch on this name, and SDKs that predate the structural- + // identity keyspace must reject the file loudly instead of + // scanning retired index families as empty. See its doc comment. Engine: c1zstore.PebbleManifestEngine, EngineSchemaVersion: uint32(SDKPebbleFormat), PayloadEncoding: payloadEncodingToProto(encoding), @@ -48,6 +49,7 @@ func BuildManifestWithSyncRuns(ctx context.Context, e *Engine, encoding c1zstore return nil, err } m.SetSyncRuns(syncRuns) + m.SetPebbleIdIndexFormat(e.manifestIDIndexFormat()) return m, nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/merge_accessor.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/merge_accessor.go index 5107887f..4049b807 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/merge_accessor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/merge_accessor.go @@ -4,6 +4,9 @@ import ( "context" "errors" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" @@ -41,6 +44,28 @@ func AsEngine(w connectorstore.Writer) (*Engine, bool) { return nil, false } +// LogCompactionMetrics logs a snapshot of the LSM's compaction and ingest +// counters, labeled with the caller's phase name. Diagnostic only: used to +// attribute wall time lost to background compactions triggered by large +// ingests (e.g. the synthesized-grant layer segments) during benchmark runs. +func (e *Engine) LogCompactionMetrics(ctx context.Context, phase string) { + if e == nil || e.db == nil { + return + } + m := e.db.Metrics() + ctxzap.Extract(ctx).Info("pebble compaction metrics", + zap.String("phase", phase), + zap.Int64("compact_count", m.Compact.Count), + zap.Duration("compact_duration_total", m.Compact.Duration), + zap.Uint64("compact_estimated_debt_bytes", m.Compact.EstimatedDebt), + zap.Int64("compact_in_progress", m.Compact.NumInProgress), + zap.Int64("compact_in_progress_bytes", m.Compact.InProgressBytes), + zap.Uint64("ingest_count", m.Ingest.Count), + zap.Int64("flush_count", m.Flush.Count), + zap.Int("read_amp", m.ReadAmp()), + ) +} + // ReadSyncStatsRecord returns the raw stats sidecar record for a sync, // or (nil, nil) when no sidecar exists. Exposed for synccompactor // tests that compare merge-accumulated stats against a recompute. @@ -181,6 +206,22 @@ func GrantRecordKey(externalID string) []byte { return encodeGrantKey(externalID) } +func EntitlementRecordIdentityKey(r *v3.EntitlementRecord) string { + id, err := entitlementIdentityFromRecord(r) + if err != nil { + return r.GetExternalId() + } + return string(encodeEntitlementIdentityKey(id)) +} + +func GrantRecordIdentityKey(r *v3.GrantRecord) string { + id, err := grantIdentityFromRecord(r) + if err != nil { + return r.GetExternalId() + } + return string(encodeGrantIdentityKey(id)) +} + func ResourceIndexKeys(r *v3.ResourceRecord) [][]byte { parent := r.GetParent() if parent == nil || parent.GetResourceId() == "" { @@ -221,36 +262,15 @@ func AppendResourceIndexKeyRawBytes(dst []byte, parentRT []byte, parentID []byte } func EntitlementIndexKeys(r *v3.EntitlementRecord) [][]byte { - res := r.GetResource() - if res == nil || res.GetResourceId() == "" { - return nil - } - return [][]byte{encodeEntitlementByResourceIndexKey(res.GetResourceTypeId(), res.GetResourceId(), r.GetExternalId())} + return nil } func ForEachEntitlementIndexKey(r *v3.EntitlementRecord, yield func([]byte) error) error { - res := r.GetResource() - if res == nil || res.GetResourceId() == "" { - return nil - } - return yield(encodeEntitlementByResourceIndexKey(res.GetResourceTypeId(), res.GetResourceId(), r.GetExternalId())) + return nil } func ForEachEntitlementIndexKeyRaw(resourceRT string, resourceID string, externalID string, yield func([]byte) error) error { - if resourceID == "" { - return nil - } - return yield(encodeEntitlementByResourceIndexKey(resourceRT, resourceID, externalID)) -} - -func AppendEntitlementIndexKeyRawBytes(dst []byte, resourceRT []byte, resourceID []byte, externalID []byte) []byte { - dst = append(dst, versionV3, typeIndex, idxEntitlementByResource) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, resourceRT) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, resourceID) - dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleBytes(dst, externalID) + return nil } func GrantIndexKeys(r *v3.GrantRecord) [][]byte { @@ -258,29 +278,8 @@ func GrantIndexKeys(r *v3.GrantRecord) [][]byte { } func ForEachGrantIndexKey(r *v3.GrantRecord, yield func([]byte) error) error { - ent := r.GetEntitlement() - princ := r.GetPrincipal() - ext := r.GetExternalId() - if ent != nil && princ != nil { - if err := yield(encodeGrantByEntitlementIndexKey(ent.GetEntitlementId(), princ.GetResourceTypeId(), princ.GetResourceId(), ext)); err != nil { - return err - } - } - if ent != nil && ent.GetResourceId() != "" { - if err := yield(encodeGrantByEntitlementResourceIndexKey(ent.GetResourceTypeId(), ent.GetResourceId(), ext)); err != nil { - return err - } - } - if princ != nil { - if err := yield(encodeGrantByPrincipalIndexKey(princ.GetResourceTypeId(), princ.GetResourceId(), ext)); err != nil { - return err - } - if err := yield(encodeGrantByPrincipalResourceTypeIndexKey(princ.GetResourceTypeId(), ext)); err != nil { - return err - } - } - if r.GetNeedsExpansion() { - if err := yield(encodeGrantByNeedsExpansionIndexKey(ext)); err != nil { + for _, key := range grantIndexKeys(r) { + if err := yield(key); err != nil { return err } } @@ -297,74 +296,21 @@ func ForEachGrantIndexKeyRaw( needsExpansion bool, yield func([]byte) error, ) error { - if entitlementID != "" && principalRT != "" && principalID != "" { - if err := yield(encodeGrantByEntitlementIndexKey(entitlementID, principalRT, principalID, externalID)); err != nil { - return err - } + if entitlementRT == "" || entitlementResourceID == "" || entitlementID == "" || principalRT == "" || principalID == "" { + return nil } - if entitlementResourceID != "" { - if err := yield(encodeGrantByEntitlementResourceIndexKey(entitlementRT, entitlementResourceID, externalID)); err != nil { - return err - } + id := grantIdentity{ + entitlement: entitlementIdentityFromParts(entitlementRT, entitlementResourceID, entitlementID), + principalTypeID: principalRT, + principalID: principalID, } - if principalRT != "" && principalID != "" { - if err := yield(encodeGrantByPrincipalIndexKey(principalRT, principalID, externalID)); err != nil { - return err - } - if err := yield(encodeGrantByPrincipalResourceTypeIndexKey(principalRT, externalID)); err != nil { - return err - } + if err := yield(encodeGrantByPrincipalIdentityIndexKey(id)); err != nil { + return err } if needsExpansion { - if err := yield(encodeGrantByNeedsExpansionIndexKey(externalID)); err != nil { + if err := yield(encodeGrantByNeedsExpansionIdentityIndexKey(id)); err != nil { return err } } return nil } - -func AppendGrantByEntitlementIndexKeyRawBytes(dst []byte, entitlementID []byte, principalRT []byte, principalID []byte, externalID []byte) []byte { - dst = append(dst, versionV3, typeIndex, idxGrantByEntitlement) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, entitlementID) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, principalRT) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, principalID) - dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleBytes(dst, externalID) -} - -func AppendGrantByEntitlementResourceIndexKeyRawBytes(dst []byte, entitlementRT []byte, entitlementResourceID []byte, externalID []byte) []byte { - dst = append(dst, versionV3, typeIndex, idxGrantByEntitlementResource) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, entitlementRT) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, entitlementResourceID) - dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleBytes(dst, externalID) -} - -func AppendGrantByPrincipalIndexKeyRawBytes(dst []byte, principalRT []byte, principalID []byte, externalID []byte) []byte { - dst = append(dst, versionV3, typeIndex, idxGrantByPrincipal) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, principalRT) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, principalID) - dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleBytes(dst, externalID) -} - -func AppendGrantByPrincipalResourceTypeIndexKeyRawBytes(dst []byte, principalRT []byte, externalID []byte) []byte { - dst = append(dst, versionV3, typeIndex, idxGrantByPrincipalResourceType) - dst = codec.AppendTupleSeparator(dst) - dst = codec.AppendTupleBytes(dst, principalRT) - dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleBytes(dst, externalID) -} - -func AppendGrantByNeedsExpansionIndexKeyRawBytes(dst []byte, externalID []byte) []byte { - dst = append(dst, versionV3, typeIndex, idxGrantByNeedsExpansion) - dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleBytes(dst, externalID) -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/options.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/options.go index daaa6355..343a30bb 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/options.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/options.go @@ -109,6 +109,11 @@ func newPebbleOptions(o *Options) *pebble.Options { ReadOnly: o.readOnly, Logger: discardPebbleLogger{}, } + // Pausable variant of pebble's default ConcurrencyLimitScheduler so the + // engine can suppress automatic compactions during the EndSync-to-close + // window, where their output never survives to the saved artifact (see + // pausableCompactionScheduler). + opts.Experimental.CompactionScheduler = newPausableCompactionScheduler() // L0 BlockSize tuning lands per-level below; bloom-filter wiring can // be added after benchmark data justifies it. opts.Levels[0].BlockSize = 32 << 10 diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/paginate.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/paginate.go index 4bb80996..349d6355 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/paginate.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/paginate.go @@ -1,6 +1,7 @@ package pebble import ( + "bytes" "context" "encoding/base64" "errors" @@ -59,7 +60,12 @@ func decodeCursor(s string) ([]byte, error) { // rangeAfter returns the half-open Pebble range [start, upper) where // `start` is positioned to be strictly greater than `cursor`. If -// `cursor` is empty, returns the prefix itself. +// `cursor` is empty, returns the prefix itself. A cursor that does not +// carry the prefix is rejected with ErrInvalidPageToken: cursors are +// raw keys we minted for THIS keyspace, so a foreign prefix means a +// corrupted or hostile token — accepting one whose bytes sort below +// the prefix would start the scan in a different record type's +// keyspace and serve its values as this type's records. // // The trick: pebble.IterOptions.LowerBound is inclusive, so we append // 0x00 to the cursor to make the iteration start at any key @@ -67,15 +73,18 @@ func decodeCursor(s string) ([]byte, error) { // equal to `cursor + 0x00` for our tuple-encoded keyspace because // tuple-encoded strings always end at the separator (0x00) or // end-of-prefix, never with a bare 0x00 byte at the tail. -func rangeAfter(prefix, cursor []byte) ([]byte, []byte) { +func rangeAfter(prefix, cursor []byte) ([]byte, []byte, error) { upper := upperBoundOf(prefix) if len(cursor) == 0 { - return prefix, upper + return prefix, upper, nil + } + if !bytes.HasPrefix(cursor, prefix) { + return nil, nil, fmt.Errorf("%w: cursor does not belong to this keyspace", ErrInvalidPageToken) } lower := make([]byte, 0, len(cursor)+1) lower = append(lower, cursor...) lower = append(lower, 0x00) - return lower, upper + return lower, upper, nil } // iteratePrimaryPageWithKey iterates over a [prefix, upperBoundOf(prefix)) @@ -97,7 +106,10 @@ func iteratePrimaryPageWithKey[T proto.Message]( if limit <= 0 { limit = DefaultPageSize } - lower, upper := rangeAfter(prefix, cursor) + lower, upper, rangeErr := rangeAfter(prefix, cursor) + if rangeErr != nil { + return nil, "", rangeErr + } iter, err := db.NewIter(&pebble.IterOptions{ LowerBound: lower, UpperBound: upper, @@ -134,6 +146,71 @@ func iteratePrimaryPageWithKey[T proto.Message]( return out, nextCursor, nil } +func iterateGrantPrimaryPage( + ctx context.Context, + db *pebble.DB, + prefix, cursor []byte, + limit int, +) ([]*v3.GrantRecord, string, error) { + if limit <= 0 { + limit = DefaultPageSize + } + lower, upper, rangeErr := rangeAfter(prefix, cursor) + if rangeErr != nil { + return nil, "", rangeErr + } + iter, err := db.NewIter(&pebble.IterOptions{ + LowerBound: lower, + UpperBound: upper, + }) + if err != nil { + return nil, "", fmt.Errorf("page iter: %w", err) + } + defer iter.Close() + out := make([]*v3.GrantRecord, 0, limit) + var lastReturnedKey []byte + hasMore := false + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return nil, "", err + } + if len(out) == limit { + hasMore = true + break + } + r := &v3.GrantRecord{} + if err := unmarshalRecord(iter.Value(), r); err != nil { + return nil, "", fmt.Errorf("page unmarshal: %w", err) + } + lastReturnedKey = append(lastReturnedKey[:0], iter.Key()...) + out = append(out, r) + } + if err := iter.Error(); err != nil { + return nil, "", err + } + var nextCursor string + if hasMore { + nextCursor = encodeCursor(lastReturnedKey) + } + return out, nextCursor, nil +} + +func getGrantByIdentity(ctx context.Context, db *pebble.DB, id grantIdentity) (*v3.GrantRecord, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + val, closer, err := db.Get(encodeGrantIdentityKey(id)) + if err != nil { + return nil, err + } + defer closer.Close() + r := &v3.GrantRecord{} + if err := unmarshalRecord(val, r); err != nil { + return nil, err + } + return r, nil +} + // === Paginated grant variants === // PaginateGrantsBySync returns up to `limit` grants from the @@ -174,79 +251,26 @@ func (e *Engine) PaginateGrants( return records, next, nil } -// PaginateGrantsByEntitlement uses the by_entitlement index. The -// index keys carry the principal-rt/principal-id/external-id tail; -// each match triggers a secondary primary-key Get to materialize the -// full grant. Cursor is the index key, not the primary key. +// PaginateGrantsByEntitlement scans the primary grant keyspace under the +// entitlement identity prefix. Cursor is the primary key. Callers resolve +// the identity from structured refs (or the bare-id lookup) — the engine +// never parses id strings here. func (e *Engine) PaginateGrantsByEntitlement( - ctx context.Context, entitlementID, cursor string, limit int, + ctx context.Context, entID entitlementIdentity, cursor string, limit int, ) ([]*v3.GrantRecord, string, error) { cursorBytes, err := decodeCursor(cursor) if err != nil { return nil, "", err } - if limit <= 0 { - limit = DefaultPageSize - } - indexPrefix := encodeGrantByEntitlementPrefix(entitlementID) - lower, upper := rangeAfter(indexPrefix, cursorBytes) - iter, err := e.db.NewIter(&pebble.IterOptions{ - LowerBound: lower, - UpperBound: upper, - }) - if err != nil { - return nil, "", fmt.Errorf("page iter: %w", err) - } - defer iter.Close() - out := make([]*v3.GrantRecord, 0, limit) - var lastReturnedKey []byte - hasMore := false - for iter.First(); iter.Valid(); iter.Next() { - if err := ctx.Err(); err != nil { - return nil, "", err - } - if len(out) == limit { - hasMore = true - break - } - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { - continue - } - val, closer, getErr := e.db.Get(encodeGrantKey(externalID)) - if getErr != nil { - if errors.Is(getErr, pebble.ErrNotFound) { - // Orphan index entry — primary deleted before the - // index. Reconcile via fsck; keep iterating. - continue - } - return nil, "", fmt.Errorf("paginate: get primary: %w", getErr) - } - r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { - return nil, "", err - } - lastReturnedKey = append(lastReturnedKey[:0], iter.Key()...) - out = append(out, r) - } - if err := iter.Error(); err != nil { - return nil, "", err - } - var nextCursor string - if hasMore { - nextCursor = encodeCursor(lastReturnedKey) - } - return out, nextCursor, nil + return iterateGrantPrimaryPage(ctx, e.db, encodeGrantPrimaryEntitlementPrefix(entID), cursorBytes, limit) } -// PaginateGrantPrincipalKeysByEntitlement scans the existing by_entitlement -// index and returns only principal identity keys for each matching grant. The -// key format is principal_resource_type + "\x00" + principal_resource_id, -// matching pkg/sync/expand's descendantGrantKey. +// PaginateGrantPrincipalKeysByEntitlement scans the primary grant keyspace under +// the entitlement identity prefix and returns only principal identity keys for +// each matching grant. The key format is principal_resource_type + "\x00" + +// principal_resource_id, matching pkg/sync/expand's descendantGrantKey. func (e *Engine) PaginateGrantPrincipalKeysByEntitlement( - ctx context.Context, entitlementID, cursor string, limit int, + ctx context.Context, entID entitlementIdentity, cursor string, limit int, ) ([]string, string, error) { cursorBytes, err := decodeCursor(cursor) if err != nil { @@ -255,8 +279,11 @@ func (e *Engine) PaginateGrantPrincipalKeysByEntitlement( if limit <= 0 { limit = DefaultPageSize } - indexPrefix := encodeGrantByEntitlementPrefix(entitlementID) - lower, upper := rangeAfter(indexPrefix, cursorBytes) + primaryPrefix := encodeGrantPrimaryEntitlementPrefix(entID) + lower, upper, err := rangeAfter(primaryPrefix, cursorBytes) + if err != nil { + return nil, "", err + } iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: lower, UpperBound: upper, @@ -276,7 +303,7 @@ func (e *Engine) PaginateGrantPrincipalKeysByEntitlement( hasMore = true break } - principalRT, principalID, ok := decodeTwoTupleComponents(iter.Key(), indexPrefix) + principalRT, principalID, ok := decodeTwoTupleComponents(iter.Key(), primaryPrefix) if !ok { continue } @@ -293,73 +320,33 @@ func (e *Engine) PaginateGrantPrincipalKeysByEntitlement( return out, nextCursor, nil } -// PaginateGrantsByEntitlementPrincipal uses the by_entitlement index narrowed -// to the entitlement_id + principal tuple. This is the hot path for grant +// PaginateGrantsByEntitlementPrincipal uses the structured primary key for a +// point lookup by entitlement + principal. This is the hot path for grant // expansion, where callers repeatedly ask whether a single principal already // has a grant on a descendant entitlement. func (e *Engine) PaginateGrantsByEntitlementPrincipal( - ctx context.Context, entitlementID, principalRT, principalID, cursor string, limit int, + ctx context.Context, entID entitlementIdentity, principalRT, principalID, cursor string, limit int, ) ([]*v3.GrantRecord, string, error) { cursorBytes, err := decodeCursor(cursor) if err != nil { return nil, "", err } - if limit <= 0 { - limit = DefaultPageSize + if len(cursorBytes) != 0 { + return nil, "", nil } - outCap := limit - if outCap > 16 { - outCap = 16 + id := grantIdentity{ + entitlement: entID, + principalTypeID: principalRT, + principalID: principalID, } - indexPrefix := encodeGrantByEntitlementPrincipalPrefix(entitlementID, principalRT, principalID) - lower, upper := rangeAfter(indexPrefix, cursorBytes) - iter, err := e.db.NewIter(&pebble.IterOptions{ - LowerBound: lower, - UpperBound: upper, - }) + r, err := getGrantByIdentity(ctx, e.db, id) if err != nil { - return nil, "", fmt.Errorf("page iter: %w", err) - } - defer iter.Close() - out := make([]*v3.GrantRecord, 0, outCap) - var lastReturnedKey []byte - hasMore := false - for iter.First(); iter.Valid(); iter.Next() { - if err := ctx.Err(); err != nil { - return nil, "", err - } - if len(out) == limit { - hasMore = true - break - } - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { - continue - } - val, closer, getErr := e.db.Get(encodeGrantKey(externalID)) - if getErr != nil { - if errors.Is(getErr, pebble.ErrNotFound) { - continue - } - return nil, "", fmt.Errorf("paginate: get primary: %w", getErr) - } - r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { - return nil, "", err + if errors.Is(err, pebble.ErrNotFound) { + return nil, "", nil } - lastReturnedKey = append(lastReturnedKey[:0], iter.Key()...) - out = append(out, r) - } - if err := iter.Error(); err != nil { return nil, "", err } - var nextCursor string - if hasMore { - nextCursor = encodeCursor(lastReturnedKey) - } - return out, nextCursor, nil + return []*v3.GrantRecord{r}, "", nil } // PaginateGrantsByPrincipal uses the by_principal index. Same shape @@ -375,7 +362,10 @@ func (e *Engine) PaginateGrantsByPrincipal( limit = DefaultPageSize } indexPrefix := encodeGrantByPrincipalPrefix(principalRT, principalID) - lower, upper := rangeAfter(indexPrefix, cursorBytes) + lower, upper, err := rangeAfter(indexPrefix, cursorBytes) + if err != nil { + return nil, "", err + } iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: lower, UpperBound: upper, @@ -395,25 +385,27 @@ func (e *Engine) PaginateGrantsByPrincipal( hasMore = true break } - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { + components, ok := decodeTupleComponents(iter.Key(), indexPrefix, 4) + if !ok { continue } - val, closer, getErr := e.db.Get(encodeGrantKey(externalID)) + id := grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: components[0], + resourceID: components[1], + stripped: components[2] == idFlagStripped, + tail: components[3], + }, + principalTypeID: principalRT, + principalID: principalID, + } + r, getErr := getGrantByIdentity(ctx, e.db, id) if getErr != nil { if errors.Is(getErr, pebble.ErrNotFound) { - // Orphan index entry — primary deleted before the - // index. Reconcile via fsck; keep iterating. continue } return nil, "", fmt.Errorf("paginate: get primary: %w", getErr) } - r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { - return nil, "", err - } lastReturnedKey = append(lastReturnedKey[:0], iter.Key()...) out = append(out, r) } @@ -427,16 +419,15 @@ func (e *Engine) PaginateGrantsByPrincipal( return out, nextCursor, nil } -// PaginateGrantsByEntitlementResource walks the by_entitlement_resource -// index — all grants in `syncID` whose entitlement's resource is -// (entRT, entRID). Cursor is the index key. +// PaginateGrantsByEntitlementResource walks the primary grant keyspace for all +// grants whose entitlement's resource is (entRT, entRID). Cursor is the primary +// key. // -// Drives Adapter.ListGrants and ListWithAnnotationsForResourcePage -// when req.Resource is set, matching SQLite's `listGrantsGeneric` -// filter on grants.resource_id / resource_type_id (the entitlement- -// side resource columns). The pre-existing Pebble path used -// PaginateGrantsByPrincipal here, which returned empty for the -// common "grants on this group" semantic. +// Drives Adapter.ListGrants and ListWithAnnotationsForResourcePage when +// req.Resource is set, matching SQLite's `listGrantsGeneric` filter on +// grants.resource_id / resource_type_id (the entitlement-side resource columns). +// The pre-existing Pebble path used PaginateGrantsByPrincipal here, which +// returned empty for the common "grants on this group" semantic. func (e *Engine) PaginateGrantsByEntitlementResource( ctx context.Context, entRT, entRID, cursor string, limit int, ) ([]*v3.GrantRecord, string, error) { @@ -447,55 +438,7 @@ func (e *Engine) PaginateGrantsByEntitlementResource( if limit <= 0 { limit = DefaultPageSize } - indexPrefix := encodeGrantByEntitlementResourcePrefix(entRT, entRID) - lower, upper := rangeAfter(indexPrefix, cursorBytes) - iter, err := e.db.NewIter(&pebble.IterOptions{ - LowerBound: lower, - UpperBound: upper, - }) - if err != nil { - return nil, "", fmt.Errorf("page iter: %w", err) - } - defer iter.Close() - out := make([]*v3.GrantRecord, 0, limit) - var lastReturnedKey []byte - hasMore := false - for iter.First(); iter.Valid(); iter.Next() { - if err := ctx.Err(); err != nil { - return nil, "", err - } - if len(out) == limit { - hasMore = true - break - } - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { - continue - } - val, closer, getErr := e.db.Get(encodeGrantKey(externalID)) - if getErr != nil { - if errors.Is(getErr, pebble.ErrNotFound) { - continue - } - return nil, "", fmt.Errorf("paginate: get primary: %w", getErr) - } - r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { - return nil, "", err - } - lastReturnedKey = append(lastReturnedKey[:0], iter.Key()...) - out = append(out, r) - } - if err := iter.Error(); err != nil { - return nil, "", err - } - var nextCursor string - if hasMore { - nextCursor = encodeCursor(lastReturnedKey) - } - return out, nextCursor, nil + return iterateGrantPrimaryPage(ctx, e.db, encodeGrantPrimaryEntitlementResourcePrefix(entRT, entRID), cursorBytes, limit) } // PaginateGrantsByPrincipalResourceType walks the by-principal-RT @@ -511,8 +454,11 @@ func (e *Engine) PaginateGrantsByPrincipalResourceType( if limit <= 0 { limit = DefaultPageSize } - indexPrefix := encodeGrantByPrincipalResourceTypePrefix(principalRT) - lower, upper := rangeAfter(indexPrefix, cursorBytes) + indexPrefix := encodeGrantByPrincipalResourceTypeIdentityPrefix(principalRT) + lower, upper, err := rangeAfter(indexPrefix, cursorBytes) + if err != nil { + return nil, "", err + } iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: lower, UpperBound: upper, @@ -532,23 +478,27 @@ func (e *Engine) PaginateGrantsByPrincipalResourceType( hasMore = true break } - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { + components, ok := decodeTupleComponents(iter.Key(), indexPrefix, 5) + if !ok { continue } - val, closer, getErr := e.db.Get(encodeGrantKey(externalID)) + id := grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: components[1], + resourceID: components[2], + stripped: components[3] == idFlagStripped, + tail: components[4], + }, + principalTypeID: principalRT, + principalID: components[0], + } + r, getErr := getGrantByIdentity(ctx, e.db, id) if getErr != nil { if errors.Is(getErr, pebble.ErrNotFound) { continue } return nil, "", fmt.Errorf("paginate: get primary: %w", getErr) } - r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { - return nil, "", err - } lastReturnedKey = append(lastReturnedKey[:0], iter.Key()...) out = append(out, r) } @@ -579,7 +529,10 @@ func (e *Engine) PaginateGrantsByNeedsExpansion( limit = DefaultPageSize } indexPrefix := encodeGrantByNeedsExpansionPrefix() - lower, upper := rangeAfter(indexPrefix, cursorBytes) + lower, upper, err := rangeAfter(indexPrefix, cursorBytes) + if err != nil { + return nil, "", err + } iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: lower, UpperBound: upper, @@ -599,24 +552,27 @@ func (e *Engine) PaginateGrantsByNeedsExpansion( hasMore = true break } - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { + components, ok := decodeTupleComponents(iter.Key(), indexPrefix, 6) + if !ok { continue } - val, closer, getErr := e.db.Get(encodeGrantKey(externalID)) + id := grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: components[0], + resourceID: components[1], + stripped: components[2] == idFlagStripped, + tail: components[3], + }, + principalTypeID: components[4], + principalID: components[5], + } + r, getErr := getGrantByIdentity(ctx, e.db, id) if getErr != nil { if errors.Is(getErr, pebble.ErrNotFound) { - // Orphan index entry; skip. continue } return nil, "", fmt.Errorf("paginate: get primary: %w", getErr) } - r := &v3.GrantRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { - return nil, "", err - } lastReturnedKey = append(lastReturnedKey[:0], iter.Key()...) out = append(out, r) } @@ -659,7 +615,10 @@ func (e *Engine) PaginateResourcesByParent( limit = DefaultPageSize } indexPrefix := encodeResourceByParentPrefix(parentRT, parentID) - lower, upper := rangeAfter(indexPrefix, cursorBytes) + lower, upper, err := rangeAfter(indexPrefix, cursorBytes) + if err != nil { + return nil, "", err + } iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: lower, UpperBound: upper, @@ -738,7 +697,7 @@ func (e *Engine) PaginateEntitlements( }) } -// PaginateEntitlementsByResource uses the by_resource index. +// PaginateEntitlementsByResource uses the entitlement primary key prefix. func (e *Engine) PaginateEntitlementsByResource( ctx context.Context, resourceTypeID, resourceID, cursor string, limit int, ) ([]*v3.EntitlementRecord, string, error) { @@ -749,8 +708,11 @@ func (e *Engine) PaginateEntitlementsByResource( if limit <= 0 { limit = DefaultPageSize } - indexPrefix := encodeEntitlementByResourcePrefix(resourceTypeID, resourceID) - lower, upper := rangeAfter(indexPrefix, cursorBytes) + indexPrefix := encodeEntitlementPrimaryResourcePrefix(resourceTypeID, resourceID) + lower, upper, err := rangeAfter(indexPrefix, cursorBytes) + if err != nil { + return nil, "", err + } iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: lower, UpperBound: upper, @@ -770,21 +732,8 @@ func (e *Engine) PaginateEntitlementsByResource( hasMore = true break } - externalID := lastTupleComponent(iter.Key(), indexPrefix) - if externalID == "" { - continue - } - val, closer, getErr := e.db.Get(encodeEntitlementKey(externalID)) - if getErr != nil { - if errors.Is(getErr, pebble.ErrNotFound) { - continue - } - return nil, "", fmt.Errorf("paginate: get primary: %w", getErr) - } r := &v3.EntitlementRecord{} - err = unmarshalRecord(val, r) - closer.Close() - if err != nil { + if err := unmarshalRecord(iter.Value(), r); err != nil { return nil, "", err } lastReturnedKey = append(lastReturnedKey[:0], iter.Key()...) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go index 295b44f9..40640954 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go @@ -110,51 +110,78 @@ func rawTimestampNanos(value []byte) (int64, error) { } func (e *Engine) deleteResourceIndexesRaw(batch *pebble.Batch, resourceTypeID string, resourceID string, value []byte) error { - parentRT, parentID, err := scanResourceParentRaw(value) + parentRT, parentID, sourceScopeHash, err := scanResourceIndexFieldsRaw(value) if err != nil { return err } - if parentID == "" { - return nil + if parentID != "" { + if err := batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil); err != nil { + return err + } } - return batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil) + if sourceScopeHash != "" { + if err := batch.Delete(encodeResourceBySourceScopeIndexKey(sourceScopeHash, resourceTypeID, resourceID), nil); err != nil { + return err + } + } + return nil } -func (e *Engine) deleteEntitlementIndexesRaw(batch *pebble.Batch, externalID string, value []byte) error { - resourceRT, resourceID, err := scanEntitlementResourceRaw(value) +func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, value []byte) error { + entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) if err != nil { return err } - if resourceID == "" { + if entID == "" || entRT == "" || entRID == "" || principalRT == "" || principalID == "" { return nil } - return batch.Delete(encodeEntitlementByResourceIndexKey(resourceRT, resourceID, externalID), nil) -} - -func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, value []byte) error { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) - if err != nil { + id := grantIdentity{ + entitlement: entitlementIdentityFromParts(entRT, entRID, entID), + principalTypeID: principalRT, + principalID: principalID, + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { return err } - if entID != "" && principalRT != "" && principalID != "" { - if err := batch.Delete(encodeGrantByEntitlementIndexKey(entID, principalRT, principalID, externalID), nil); err != nil { + if sourceScopeHash != "" { + if err := batch.Delete(encodeGrantBySourceScopeIndexKey(sourceScopeHash, id), nil); err != nil { return err } } - if entRID != "" { - if err := batch.Delete(encodeGrantByEntitlementResourceIndexKey(entRT, entRID, externalID), nil); err != nil { - return err + return batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil) +} + +// scanGrantExternalIDRaw extracts only the stored external_id (field 2) +// from a marshaled GrantRecord. Used by the bare-id grant lookup to check +// a probe hit's public id without a full unmarshal. Last occurrence wins, +// matching the full scanners. +func scanGrantExternalIDRaw(value []byte) (string, error) { + var externalID string + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return "", protowire.ParseError(n) } - } - if principalRT != "" && principalID != "" { - if err := batch.Delete(encodeGrantByPrincipalIndexKey(principalRT, principalID, externalID), nil); err != nil { - return err + value = value[n:] + if num != 2 { + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return "", protowire.ParseError(n) + } + value = value[n:] + continue } - if err := batch.Delete(encodeGrantByPrincipalResourceTypeIndexKey(principalRT, externalID), nil); err != nil { - return err + if typ != protowire.BytesType { + return "", fmt.Errorf("raw record: grant external_id has wire type %v", typ) } + v, n := protowire.ConsumeString(value) + if n < 0 { + return "", protowire.ParseError(n) + } + externalID = v + value = value[n:] } - return batch.Delete(encodeGrantByNeedsExpansionIndexKey(externalID), nil) + return externalID, nil } // scanGrantEntitlementResourceTypeRaw extracts only the entitlement's @@ -242,37 +269,83 @@ func scanEntitlementResourceTypeRaw(value []byte) ([]byte, error) { // occurrence of the target field, matching scanGrantIndexFieldsRaw and // approximating proto merge semantics. Values written by this SDK carry // at most one occurrence, so this only matters for foreign writers. -func scanResourceParentRaw(value []byte) (string, string, error) { - var rt, id string +// scanResourceIndexFieldsRaw extracts the parent ref (field 6) and +// source_scope_hash (field 9) from a marshaled ResourceRecord — the +// fields that key resource secondary indexes. +func scanResourceIndexFieldsRaw(value []byte) (string, string, string, error) { + var rt, id, sourceScopeHash string for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + switch num { + case 6: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + } + msg, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + var err error + rt, id, err = scanResourceRefRaw(msg) + if err != nil { + return "", "", "", err + } + value = value[n:] + case 9: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource source_scope_hash has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + sourceScopeHash = string(s) + value = value[n:] + default: + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + } + } + return rt, id, sourceScopeHash, nil +} + +// scanEntitlementSourceScopeRaw extracts source_scope_hash (field 11) +// from a marshaled EntitlementRecord. Keys the entitlement +// by_source_scope index — the only entitlement secondary index. +func scanEntitlementSourceScopeRaw(value []byte) (string, error) { + var sourceScopeHash string + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return "", protowire.ParseError(n) } value = value[n:] - if num != 6 { + if num != 11 { n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", protowire.ParseError(n) } value = value[n:] continue } if typ != protowire.BytesType { - return "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + return "", fmt.Errorf("raw record: entitlement source_scope_hash has wire type %v", typ) } - msg, n := protowire.ConsumeBytes(value) + s, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", protowire.ParseError(n) - } - var err error - rt, id, err = scanResourceRefRaw(msg) - if err != nil { - return "", "", err + return "", protowire.ParseError(n) } + sourceScopeHash = string(s) value = value[n:] } - return rt, id, nil + return sourceScopeHash, nil } func scanEntitlementResourceRaw(value []byte) (string, string, error) { @@ -308,63 +381,149 @@ func scanEntitlementResourceRaw(value []byte) (string, string, error) { return rt, id, nil } -func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, error) { - var entRT, entRID, entID, principalRT, principalID string +func scanEntitlementIdentityFieldsRaw(value []byte) (string, string, string, error) { + var externalID, rt, id string + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + switch num { + case 2: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: entitlement external_id has wire type %v", typ) + } + v, n := protowire.ConsumeString(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + externalID = v + value = value[n:] + case 3: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: entitlement resource has wire type %v", typ) + } + msg, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + var err error + rt, id, err = scanResourceRefRaw(msg) + if err != nil { + return "", "", "", err + } + value = value[n:] + default: + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + } + } + return rt, id, externalID, nil +} + +func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, string, error) { + var entRT, entRID, entID, principalRT, principalID, sourceScopeHash string var needsExpansion bool for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } value = value[n:] switch num { case 3: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant entitlement has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant entitlement has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error entRT, entRID, entID, err = scanEntitlementRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 4: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant principal has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant principal has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error principalRT, principalID, err = scanPrincipalRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 7: if typ != protowire.VarintType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) } v, n := protowire.ConsumeVarint(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } needsExpansion = v != 0 value = value[n:] + case 10: + if typ != protowire.BytesType { + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant source_scope_hash has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", "", "", false, "", protowire.ParseError(n) + } + sourceScopeHash = string(s) + value = value[n:] default: n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) + } + value = value[n:] + } + } + return entRT, entRID, entID, principalRT, principalID, needsExpansion, sourceScopeHash, nil +} + +// scanGrantNeedsExpansionRaw extracts only the needs_expansion flag +// (GrantRecord field 7) with a shallow wire scan — for callers that already +// carry the identity in the key and need nothing else from the value. +func scanGrantNeedsExpansionRaw(value []byte) (bool, error) { + var needsExpansion bool + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + if num == 7 { + if typ != protowire.VarintType { + return false, fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) + } + v, n := protowire.ConsumeVarint(value) + if n < 0 { + return false, protowire.ParseError(n) } + needsExpansion = v != 0 value = value[n:] + continue + } + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return false, protowire.ParseError(n) } + value = value[n:] } - return entRT, entRID, entID, principalRT, principalID, needsExpansion, nil + return needsExpansion, nil } func scanResourceRefRaw(value []byte) (string, string, error) { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go index 679f0511..0f0d820c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go @@ -99,10 +99,14 @@ func (e *Engine) PutResourceRecords(ctx context.Context, records ...*v3.Resource if fresh { opts = pebble.NoSync } - if err := priBatch.Commit(opts); err != nil { + // One atomic commit: folding the index batch into the primary batch + // closes the durability gap where the primary commit landed but the + // index commit failed — a divergence that would persist in the saved + // artifact if the caller shipped it anyway. + if err := priBatch.Apply(idxBatch, nil); err != nil { return err } - return idxBatch.Commit(opts) + return priBatch.Commit(opts) }) } @@ -147,15 +151,22 @@ func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resou } func (e *Engine) writeResourceIndexes(batch *pebble.Batch, r *v3.ResourceRecord) error { - parent := r.GetParent() - if parent == nil || parent.GetResourceId() == "" { - return nil + if parent := r.GetParent(); parent != nil && parent.GetResourceId() != "" { + k := encodeResourceByParentIndexKey( + parent.GetResourceTypeId(), parent.GetResourceId(), + r.GetResourceTypeId(), r.GetResourceId(), + ) + if err := batch.Set(k, nil, nil); err != nil { + return err + } + } + if sh := r.GetSourceScopeHash(); sh != "" { + k := encodeResourceBySourceScopeIndexKey(sh, r.GetResourceTypeId(), r.GetResourceId()) + if err := batch.Set(k, nil, nil); err != nil { + return err + } } - k := encodeResourceByParentIndexKey( - parent.GetResourceTypeId(), parent.GetResourceId(), - r.GetResourceTypeId(), r.GetResourceId(), - ) - return batch.Set(k, nil, nil) + return nil } func (e *Engine) IterateResources(ctx context.Context, yield func(*v3.ResourceRecord) bool) error { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/session_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/session_store.go index 3a78df78..c03767d9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/session_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/session_store.go @@ -271,7 +271,19 @@ func (e *Engine) SessionSet(ctx context.Context, key string, value []byte, opt . return fmt.Errorf("error marshalling session record: %w", err) } - return e.db.Set(keyBytes, val, writeOpts(e.opts.durability)) + // Under the write barrier like every other record write: a bare Set + // would race Close's teardown (no writeWG coverage) and could land + // inside CheckpointTo's Flush→Checkpoint window as a WAL-only record + // the truncate silently drops from the saved snapshot. + // + // AllowSealed, like the sync-run metadata writers: sessions are + // connector scratch state, not sync records — the seal exists to keep + // record data immutable after EndSync, and the production lifecycle + // WRITES to the session keyspace after EndSync (connector Cleanup + // clears sessions post-sync so they don't ship in the saved c1z). + return e.withWriteAllowSealed(func() error { + return e.db.Set(keyBytes, val, writeOpts(e.opts.durability)) + }) } func (e *Engine) SessionSetMany(ctx context.Context, values map[string][]byte, opt ...sessions.SessionStoreOption) error { @@ -280,30 +292,32 @@ func (e *Engine) SessionSetMany(ctx context.Context, values map[string][]byte, o return fmt.Errorf("error applying session option: %w", err) } - batch := e.db.NewBatch() - defer batch.Close() - - for key, value := range values { - prefixedKey := bag.Prefix + key - keyBytes := encodeSessionKey(bag.SyncID, prefixedKey) - val, err := marshalRecord(&v3.SessionRecord{ - SyncId: bag.SyncID, - Key: prefixedKey, - Value: value, - }) - if err != nil { - return fmt.Errorf("error marshalling session record: %w", err) - } - if err := batch.Set(keyBytes, val, nil); err != nil { - return fmt.Errorf("error setting session record: %w", err) + // See SessionSet for why the barrier is required and why AllowSealed. + return e.withWriteAllowSealed(func() error { + batch := e.db.NewBatch() + defer batch.Close() + + for key, value := range values { + prefixedKey := bag.Prefix + key + keyBytes := encodeSessionKey(bag.SyncID, prefixedKey) + val, err := marshalRecord(&v3.SessionRecord{ + SyncId: bag.SyncID, + Key: prefixedKey, + Value: value, + }) + if err != nil { + return fmt.Errorf("error marshalling session record: %w", err) + } + if err := batch.Set(keyBytes, val, nil); err != nil { + return fmt.Errorf("error setting session record: %w", err) + } } - } - err = batch.Commit(writeOpts(e.opts.durability)) - if err != nil { - return fmt.Errorf("error committing session batch: %w", err) - } - return nil + if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { + return fmt.Errorf("error committing session batch: %w", err) + } + return nil + }) } func (e *Engine) SessionDelete(ctx context.Context, key string, opt ...sessions.SessionStoreOption) error { @@ -312,7 +326,10 @@ func (e *Engine) SessionDelete(ctx context.Context, key string, opt ...sessions. return fmt.Errorf("error applying session option: %w", err) } keyBytes := encodeSessionKey(bag.SyncID, bag.Prefix+key) - return e.db.Delete(keyBytes, writeOpts(e.opts.durability)) + // See SessionSet for why the barrier is required and why AllowSealed. + return e.withWriteAllowSealed(func() error { + return e.db.Delete(keyBytes, writeOpts(e.opts.durability)) + }) } func (e *Engine) SessionClear(ctx context.Context, opt ...sessions.SessionStoreOption) error { @@ -322,41 +339,47 @@ func (e *Engine) SessionClear(ctx context.Context, opt ...sessions.SessionStoreO } syncPrefix := encodeSessionBySyncPrefix(bag.SyncID) if bag.Prefix == "" { - return e.db.DeleteRange(syncPrefix, upperBoundOf(syncPrefix), writeOpts(e.opts.durability)) - } - - lower := encodeSessionKey(bag.SyncID, bag.Prefix) - iter, err := e.db.NewIter(&pebble.IterOptions{ - LowerBound: lower, - UpperBound: upperBoundOf(syncPrefix), - }) - if err != nil { - return err + // See SessionSet for why the barrier is required and why AllowSealed. + return e.withWriteAllowSealed(func() error { + return e.db.DeleteRange(syncPrefix, upperBoundOf(syncPrefix), writeOpts(e.opts.durability)) + }) } - defer iter.Close() - batch := e.db.NewBatch() - defer batch.Close() - for iter.First(); iter.Valid(); iter.Next() { - if err := ctx.Err(); err != nil { + // See SessionSet for why the barrier is required and why AllowSealed. + return e.withWriteAllowSealed(func() error { + lower := encodeSessionKey(bag.SyncID, bag.Prefix) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: lower, + UpperBound: upperBoundOf(syncPrefix), + }) + if err != nil { return err } - r := &v3.SessionRecord{} - if err := unmarshalRecord(iter.Value(), r); err != nil { - return fmt.Errorf("session-clear: unmarshal: %w", err) - } - if r.GetSyncId() != bag.SyncID { - continue - } - if !strings.HasPrefix(r.GetKey(), bag.Prefix) { - break + defer iter.Close() + + batch := e.db.NewBatch() + defer batch.Close() + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + r := &v3.SessionRecord{} + if err := unmarshalRecord(iter.Value(), r); err != nil { + return fmt.Errorf("session-clear: unmarshal: %w", err) + } + if r.GetSyncId() != bag.SyncID { + continue + } + if !strings.HasPrefix(r.GetKey(), bag.Prefix) { + break + } + if err := batch.Delete(iter.Key(), nil); err != nil { + return err + } } - if err := batch.Delete(iter.Key(), nil); err != nil { - return err + if err := iter.Error(); err != nil { + return fmt.Errorf("session-clear: iter: %w", err) } - } - if err := iter.Error(); err != nil { - return fmt.Errorf("session-clear: iter: %w", err) - } - return batch.Commit(writeOpts(e.opts.durability)) + return batch.Commit(writeOpts(e.opts.durability)) + }) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go new file mode 100644 index 00000000..dfb238c4 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go @@ -0,0 +1,835 @@ +package pebble + +import ( + "context" + "errors" + "fmt" + + "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/types/known/timestamppb" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// Source-cache replay, engine side. +// +// The typeSourceCache keyspace holds one SourceCacheEntryRecord per +// (row_kind, scope_hash): the opaque upstream validator (etag / delta +// token) the sync recorded for that scope. Rows produced under a scope +// are stamped with source_scope_hash and indexed under the +// by_source_scope families, whose tails are identity tuples — so a +// replay derives every primary key from the index key alone and copies +// raw values across files without a proto unmarshal. +// +// The previous sync lives in a separate read-only engine (a Pebble c1z +// holds exactly one sync); replay copies from prev into the receiver. + +// replayBatchRows bounds how many rows accumulate in one pebble.Batch +// before an intermediate commit. Replay of a delta-query collection can +// be the whole previous row set, so the batch must not grow unbounded. +const replayBatchRows = 10_000 + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult struct { + Rows int64 + // NeedsExpansion is true when at least one copied grant row carried + // needs_expansion. The syncer must arm grant expansion in this case: + // replayed pages never pass GrantExpandable-annotated rows through + // the syncer's connector-response path, which is otherwise the only + // thing that enables the expansion phase. + NeedsExpansion bool +} + +// PutSourceCacheEntry writes the manifest entry for (rowKind, scopeHash). +// Zero-row scopes still get entries — the validator must survive to the +// next sync even when the scope produced no rows. +func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeHash, etag string) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + rec := &v3.SourceCacheEntryRecord{} + rec.SetRowKind(rowKind) + rec.SetScopeHash(scopeHash) + rec.SetEtag(etag) + rec.SetDiscoveredAt(timestamppb.Now()) + val, err := marshalRecord(rec) + if err != nil { + return err + } + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + return e.db.Set(encodeSourceCacheEntryKey(rowKind, scopeHash), val, opts) + }) +} + +// GetSourceCacheEntry returns the manifest entry for (rowKind, scopeHash), +// or pebble.ErrNotFound. +func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeHash string) (*v3.SourceCacheEntryRecord, error) { + val, closer, err := e.db.Get(encodeSourceCacheEntryKey(rowKind, scopeHash)) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("GetSourceCacheEntry: unmarshal: %w", err) + } + return rec, nil +} + +// DeleteGrantRecordBounded deletes a grant by canonical public id WITHOUT +// the O(all grants) stored-external-id scan fallback that the interactive +// DeleteGrantRecord path is allowed to take. Used by the source-cache +// tombstone path, where a mass-removal round would otherwise pay a full +// keyspace scan PER already-absent id. +// +// Consequence, by design: a grant stored under a connector-CUSTOM id (one +// that isn't the SDK concat shape) is unreachable here and the delete +// no-ops. Connectors with custom grant ids must use principal-scoped +// tombstones (SourceCacheScope.deleted_principal_ids) instead — documented +// in the annotation proto. +func (e *Engine) DeleteGrantRecordBounded(ctx context.Context, externalID string) error { + return e.withWrite(func() error { + id, err := e.resolveGrantIdentityByCandidates(ctx, externalID) + if err != nil { + if errors.Is(err, errNoGrantCandidateHits) || errors.Is(err, pebble.ErrNotFound) { + return nil // absent (or custom-id) — tombstone no-op + } + return err + } + return e.deleteGrantByIdentityLocked(id) + }) +} + +// DeleteGrantsByPrincipalsInScope deletes every grant row in the CURRENT +// store stamped with scopeHash whose principal id is in principalIDs — +// the engine side of principal-scoped delta tombstones +// (SourceCacheScope.deleted_principal_ids). +// +// One prefix scan of the scope's by_source_scope index resolves +// everything: the index tail IS the grant identity, so the primary key +// and every secondary index key for a match are constructible from the +// index key alone — no value reads, no string resolution, no guessing. +// A principal with no rows in the scope is a no-op (providers tombstone +// objects the client never synced). Deleting a missing secondary index +// entry is a pebble no-op, which covers the mixed inline/deferred +// by_principal state mid-sync. +// +// Complexity: O(scope size) tuple-walks per call regardless of tombstone +// count — callers batch a page's tombstones into one call. +func (e *Engine) DeleteGrantsByPrincipalsInScope(ctx context.Context, scopeHash string, principalIDs map[string]struct{}) (int64, error) { + if len(principalIDs) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: ent_rt | ent_rid | flag | ent_tail | prin_rt | prin_id + // (identical to the grant primary key tail; decoder shared with + // the primary-prefix scan paths in grants.go). + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + if _, hit := principalIDs[id.principalID]; !hit { + continue + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + // The scope index entry itself (the key under the iterator — + // safe: the iterator reads a snapshot). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteGrantsByExternalIDsInScope deletes every grant row in the CURRENT +// store stamped with scopeHash whose STORED grant id (external id, which +// may be a connector-custom shape) is in ids. One scan of the scope's +// index, loading each candidate's primary row to compare the stored id — +// bounded by the scope's row count, never the whole keyspace. This is the +// tombstone path for connectors with custom grant ids whose scopes span +// multiple resources (so principal-scoped deletes would over-delete). +func (e *Engine) DeleteGrantsByExternalIDsInScope(ctx context.Context, scopeHash string, ids map[string]struct{}) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + + val, closer, err := e.db.Get(priKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue // index ahead of primary — defensive skip + } + return err + } + rec := &v3.GrantRecord{} + uerr := unmarshalRecord(val, rec) + _ = closer.Close() + if uerr != nil { + return fmt.Errorf("DeleteGrantsByExternalIDsInScope: unmarshal: %w", uerr) + } + if _, hit := ids[rec.GetExternalId()]; !hit { + continue + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteResourcesByIDsInScope deletes every resource row in the CURRENT +// store stamped with scopeHash whose resource id is in resourceIDs (any +// resource type) — principal-scoped tombstones for RowKindResources. +func (e *Engine) DeleteResourcesByIDsInScope(ctx context.Context, scopeHash string, resourceIDs map[string]struct{}) (int64, error) { + if len(resourceIDs) == 0 { + return 0, nil + } + prefix := encodeResourceBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: resource_type_id | resource_id. + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + continue + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + continue + } + if _, hit := resourceIDs[string(ridBytes)]; !hit { + continue + } + rt, rid := string(rtBytes), string(ridBytes) + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeResource) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + // by_parent cleanup needs the parent ref from the value. + if val, closer, getErr := e.db.Get(priKey); getErr == nil { + err := e.deleteResourceIndexesRaw(batch, rt, rid, val) + closer.Close() + if err != nil { + return err + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + // deleteResourceIndexesRaw already covered the scope entry + // (source_scope_hash is in the value), but delete the iterated + // key too in case the value read missed (orphan entry). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// grantValueHasSourcesRaw reports whether a marshaled GrantRecord carries +// at least one sources entry (field 9), without unmarshaling. +func grantValueHasSourcesRaw(value []byte) (bool, error) { + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + if num == 9 { + return true, nil + } + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + } + return false, nil +} + +// stripExpanderSourcesRaw clears a replayed grant's Sources map when it is +// expander-written, so the current sync's expansion recomputes it from +// true state instead of inheriting contributions that may have been +// removed upstream. Classification mirrors RollbackExpansion: a Sources +// map containing a self-source entry (keyed by the grant's own entitlement +// id) was written by the expander; one without a self-source is +// connector-set public data and is preserved. Returns (newValue, true) when +// the record was rewritten, (nil, false) when the original bytes should be +// copied verbatim. +func stripExpanderSourcesRaw(value []byte, ownEntitlementID string) ([]byte, bool, error) { + r := &v3.GrantRecord{} + if err := unmarshalRecord(value, r); err != nil { + return nil, false, fmt.Errorf("source cache replay: unmarshal grant for sources strip: %w", err) + } + sources := r.GetSources() + if len(sources) == 0 { + return nil, false, nil + } + if _, hasSelf := sources[ownEntitlementID]; !hasSelf { + // No self-source: connector-set Sources. Preserve verbatim. + return nil, false, nil + } + r.SetSources(nil) + stripped, err := marshalRecord(r) + if err != nil { + return nil, false, fmt.Errorf("source cache replay: re-marshal grant after sources strip: %w", err) + } + return stripped, true, nil +} + +// decodeResourcePrimaryTail decodes (resource_type_id, resource_id) +// from a resource primary key (v3 | typeResource | 0x00 | rt | 0x00 | rid). +func decodeResourcePrimaryTail(priKey []byte) (string, string, error) { + const headerLen = 3 // versionV3, typeResource, separator + if len(priKey) <= headerLen { + return "", "", fmt.Errorf("source cache replay: malformed resource primary key %x", priKey) + } + tail := priKey[headerLen:] + rtBytes, next, err := codec.DecodeTupleStringTo(nil, tail, 0) + if err != nil { + return "", "", err + } + if next >= len(tail) { + return "", "", fmt.Errorf("source cache replay: resource primary key missing resource_id: %x", priKey) + } + ridBytes, _, err := codec.DecodeTupleStringTo(nil, tail, next+1) + if err != nil { + return "", "", err + } + return string(rtBytes), string(ridBytes), nil +} + +// replayPrimaryFromIndexKey derives a record's primary key from its +// by_source_scope index key. The index prefix (header|0x00|scope|0x00) +// is followed by exactly the identity tuple that forms the primary +// key's tail, so the primary is header' + 0x00-separated remainder. +func replayPrimaryFromIndexKey(indexKey, indexPrefix []byte, primaryHeader [2]byte) ([]byte, error) { + if len(indexKey) <= len(indexPrefix) { + return nil, fmt.Errorf("source cache replay: malformed index key %x", indexKey) + } + tail := indexKey[len(indexPrefix):] + key := make([]byte, 0, 3+len(tail)) + key = append(key, primaryHeader[0], primaryHeader[1], 0x00) + return append(key, tail...), nil +} + +// ReplaySourceCacheGrants copies every grant stamped with scopeHash from +// prev into the receiver: raw primary copy plus index synthesis from the +// raw value (principal, needs_expansion, source-scope families). Mirrors +// PutGrantRecords' read-before-write index cleanup when the receiver +// already holds a record at the same identity. +func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeGrantBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeGrant} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + // Orphan index entry in the previous file — skip, + // matching the defensive-skip semantic of the other + // index read paths. + continue + } + return fmt.Errorf("source cache replay: get prev grant: %w", getErr) + } + + entRT, entRID, entID, principalRT, principalID, needsExpansion, srcScope, scanErr := scanGrantIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope. An index entry pointing at a row stamped + // differently (or not at all) is left over from a path that + // replaced the row without cleaning the index — e.g. a fold + // compaction predating the source-cache bucket plans, or an + // in-sync same-identity rewrite under a different scope. Copying + // it would inject rows upstream never returned for this scope. + if srcScope != scopeHash { + closer.Close() + continue + } + + // Clean up index entries for any record the current sync + // already wrote at this identity (same discipline as + // PutGrantRecords' read-before-write). + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteGrantIndexesRaw(batch, "", oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current grant: %w", oldErr) + } + + // Replay-equivalence: a cached sync must reproduce what a full + // resync would produce. The one field where a verbatim copy + // diverges is expander-written Sources — the previous sync's + // expansion baked contributions into direct grants, and + // re-expansion only ADDS, so a contribution removed this sync + // (via a delta tombstone or a refetched page) would survive + // forever. Strip expander-written Sources so the current sync's + // expansion recomputes them from true state; connector-set + // Sources (no self-source entry — same classification as + // RollbackExpansion) are connector data and are preserved + // verbatim. The probe is a cheap protowire scan; the vast + // majority of rows carry no Sources and stay on the raw-copy + // path. + writeVal := val + if hasSources, probeErr := grantValueHasSourcesRaw(val); probeErr != nil { + closer.Close() + return probeErr + } else if hasSources { + stripped, strippedOK, stripErr := stripExpanderSourcesRaw(val, entID) + if stripErr != nil { + closer.Close() + return stripErr + } + if strippedOK { + writeVal = stripped + } + } + if err := batch.Set(priKey, writeVal, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if entID != "" && entRT != "" && entRID != "" && principalRT != "" && principalID != "" { + id := grantIdentity{ + entitlement: entitlementIdentityFromParts(entRT, entRID, entID), + principalTypeID: principalRT, + principalID: principalID, + } + if _, err := e.writeGrantIndexesForIdentityScratch(batch, id, needsExpansion, srcScope, nil); err != nil { + return err + } + } + if needsExpansion { + res.NeedsExpansion = true + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// ReplaySourceCacheEntitlements copies every entitlement stamped with +// scopeHash from prev into the receiver. +func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeEntitlementBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeEntitlement} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + continue + } + return fmt.Errorf("source cache replay: get prev entitlement: %w", getErr) + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + prevScope, prevScanErr := scanEntitlementSourceScopeRaw(val) + if prevScanErr != nil { + closer.Close() + return prevScanErr + } + if prevScope != scopeHash { + closer.Close() + continue + } + // The replayed row's source-scope index key is byte-identical + // to prev's — the only entitlement secondary index. Clean up a + // differing stamp on any record the current sync already wrote + // at this identity. + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + oldCloser.Close() + if scanErr != nil { + closer.Close() + return scanErr + } + if oldScope != "" && oldScope != scopeHash { + // The old index key's tail equals the primary key's + // tail (identity tuple), so rebuild it byte-wise. + oldIdxKey := make([]byte, 0, 8+len(oldScope)+len(priKey)) + oldIdxKey = append(oldIdxKey, versionV3, typeIndex, idxEntitlementBySourceScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = codec.AppendTupleStrings(oldIdxKey, oldScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = append(oldIdxKey, priKey[3:]...) + if err := batch.Delete(oldIdxKey, nil); err != nil { + closer.Close() + return err + } + } + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current entitlement: %w", oldErr) + } + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + if err := batch.Set(iter.Key(), nil, nil); err != nil { + return err + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// ReplaySourceCacheResources copies every resource stamped with scopeHash +// from prev into the receiver, synthesizing by_parent and by_source_scope +// index entries from the raw value. +func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeResourceBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeResource} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + continue + } + return fmt.Errorf("source cache replay: get prev resource: %w", getErr) + } + + rt, rid, decodeErr := decodeResourcePrimaryTail(priKey) + if decodeErr != nil { + closer.Close() + return decodeErr + } + parentRT, parentID, srcScope, scanErr := scanResourceIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + if srcScope != scopeHash { + closer.Close() + continue + } + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteResourceIndexesRaw(batch, rt, rid, oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current resource: %w", oldErr) + } + + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if parentID != "" { + if err := batch.Set(encodeResourceByParentIndexKey(parentRT, parentID, rt, rid), nil, nil); err != nil { + return err + } + } + if srcScope != "" { + if err := batch.Set(encodeResourceBySourceScopeIndexKey(srcScope, rt, rid), nil, nil); err != nil { + return err + } + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go index 60bdd703..b3d2b267 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go @@ -29,7 +29,11 @@ func (e *Engine) PutSyncRunRecord(ctx context.Context, r *v3.SyncRunRecord) erro if r.GetSyncId() == "" { return errors.New("PutSyncRunRecord: empty sync_id") } - return e.withWrite(func() error { + // AllowSealed: sync-run metadata is legitimately stamped on a finished + // sync — ToPebble preserves the source ended_at, the sanitizer applies + // diff links / supports_diff, and the compactor renames the folded sync + // — all after EndSync sealed the engine. + return e.withWriteAllowSealed(func() error { val, err := marshalRecord(r) if err != nil { return err @@ -63,7 +67,9 @@ func (e *Engine) GetSyncRunRecord(ctx context.Context, syncID string) (*v3.SyncR } func (e *Engine) DeleteSyncRunRecord(ctx context.Context, syncID string) error { - return e.withWrite(func() error { + // AllowSealed: sync-run pruning is metadata maintenance on finished + // syncs, same class as PutSyncRunRecord above. + return e.withWriteAllowSealed(func() error { if syncID == "" { return errors.New("DeleteSyncRunRecord: empty sync_id") } @@ -136,7 +142,7 @@ func (e *Engine) IterateAllSyncRuns(ctx context.Context, yield func(*v3.SyncRunR // This is the single source of truth for "pick the latest finished // sync" on the Pebble engine; all three external entry points // (connectorstore.LatestFinishedSyncIDFetcher, -// dotc1z.SyncMeta.LatestFullSync / LatestFinishedSyncOfAnyType, and +// c1zstore.SyncMeta.LatestFullSync / LatestFinishedSyncOfAnyType, and // reader_v2.SyncsReaderService.GetLatestFinishedSync) call here so // the tiebreaker and predicate semantics stay consistent. // diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go index 808ef887..040cfedd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go @@ -84,7 +84,15 @@ func (e *Engine) writeSyncStats(ctx context.Context, rec *v3.SyncStatsRecord) er if err != nil { return err } - return e.db.Set(encodeSyncStatsKey(), val, pebble.Sync) + // AllowSealed under the write barrier: callers span EndSync's sealed + // finalize window and the compactor's bound-sync flow. The barrier + // (rather than a bare Set) gives the write the closing check, writeWG + // coverage against Close's teardown, and exclusion from CheckpointTo's + // Flush→Checkpoint window (a WAL-only record landing mid-window would + // be truncated out of the saved snapshot). + return e.withWriteAllowSealed(func() error { + return e.db.Set(encodeSyncStatsKey(), val, pebble.Sync) + }) } // computeSyncStats does one full pass over the per-record-type @@ -176,26 +184,38 @@ func (e *Engine) computeSyncStats(ctx context.Context, syncID string) (*v3.SyncS // are NOT contiguous. map[string]*int64 keeps the per-row map read // allocation-free and only materializes a key string the first time // each resource type appears. - grantsByEntRTPtr := map[string]*int64{} - grants, err := countKeyRange(ctx, e.db, GrantLowerBound(), GrantUpperBound(), func(_ []byte, value []byte) error { - entRT, err := scanGrantEntitlementResourceTypeRaw(value) + // + // When BuildDeferredGrantIndexes already accumulated these numbers + // during its own full grant scan (EndSync with a deferred principal + // index), consume the stash instead of scanning tens of millions of + // grant rows a second time. + var grants int64 + var grantsByEntitlementRT map[string]int64 + if ds := e.takeDeferredGrantStats(syncID); ds != nil { + grants = ds.grants + grantsByEntitlementRT = ds.grantsByEntitlementRT + } else { + grantsByEntRTPtr := map[string]*int64{} + grants, err = countKeyRange(ctx, e.db, GrantLowerBound(), GrantUpperBound(), func(_ []byte, value []byte) error { + entRT, err := scanGrantEntitlementResourceTypeRaw(value) + if err != nil { + return err + } + if p, ok := grantsByEntRTPtr[string(entRT)]; ok { + *p++ + return nil + } + n := int64(1) + grantsByEntRTPtr[string(entRT)] = &n + return nil + }) if err != nil { - return err + return nil, fmt.Errorf("computeSyncStats: grants: %w", err) } - if p, ok := grantsByEntRTPtr[string(entRT)]; ok { - *p++ - return nil + grantsByEntitlementRT = make(map[string]int64, len(grantsByEntRTPtr)) + for rt, p := range grantsByEntRTPtr { + grantsByEntitlementRT[rt] = *p } - n := int64(1) - grantsByEntRTPtr[string(entRT)] = &n - return nil - }) - if err != nil { - return nil, fmt.Errorf("computeSyncStats: grants: %w", err) - } - grantsByEntitlementRT := make(map[string]int64, len(grantsByEntRTPtr)) - for rt, p := range grantsByEntRTPtr { - grantsByEntitlementRT[rt] = *p } rec.SetGrants(grants) @@ -211,6 +231,29 @@ func (e *Engine) computeSyncStats(ctx context.Context, syncID string) (*v3.SyncS return rec, nil } +// stashDeferredGrantStats records the grant counts BuildDeferredGrantIndexes +// accumulated so the next computeSyncStats call for the same sync can skip +// its own grant scan. Overwrites any prior stash. +func (e *Engine) stashDeferredGrantStats(ds *deferredGrantStats) { + e.deferredGrantStatsMu.Lock() + e.deferredGrantStats = ds + e.deferredGrantStatsMu.Unlock() +} + +// takeDeferredGrantStats pops the stashed grant stats if they belong to +// syncID, or returns nil. Consume-once: a later RecalculateStats falls back +// to the full scan. +func (e *Engine) takeDeferredGrantStats(syncID string) *deferredGrantStats { + e.deferredGrantStatsMu.Lock() + defer e.deferredGrantStatsMu.Unlock() + ds := e.deferredGrantStats + if ds == nil || ds.syncID != syncID { + return nil + } + e.deferredGrantStats = nil + return ds +} + // PersistSyncStats computes and writes the stats sidecar for // syncID. Exposed for the EndFreshSync caller and the on-Open // migration backfill. If a caller-computed record was stashed for diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/translate_v2.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/translate_v2.go index 0846a091..fac6d9c6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/translate_v2.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/translate_v2.go @@ -123,7 +123,7 @@ func V3GrantToV2(r *v3.GrantRecord) *v2.Grant { } } return v2.Grant_builder{ - Id: r.GetExternalId(), + Id: publicGrantRecordID(r), Entitlement: entitlementRefToStub(r.GetEntitlement()), Principal: principalRefToStubResource(r.GetPrincipal()), Annotations: anns, @@ -267,6 +267,8 @@ func entitlementRefToStub(ref *v3.EntitlementRef) *v2.Entitlement { return nil } return v2.Entitlement_builder{ + // The ref carries the raw connector id; emit it verbatim — external + // ids are an external-consumer contract and are never re-encoded. Id: ref.GetEntitlementId(), Resource: v2.Resource_builder{ Id: v2.ResourceId_builder{ @@ -501,6 +503,7 @@ func V3EntitlementToV2(r *v3.EntitlementRecord) *v2.Entitlement { } } return v2.Entitlement_builder{ + // Stored raw external id, emitted verbatim. Id: r.GetExternalId(), Resource: resource, DisplayName: r.GetDisplayName(), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go index d1a4f4c7..403f1683 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go @@ -10,6 +10,7 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" ) @@ -31,13 +32,13 @@ type StoreOptions struct { SyncLimit int SkipCleanup bool V2GrantsWriter bool - Engine Engine + Engine c1zstore.Engine // PayloadEncoding selects the v3 envelope payload framing for // engines that produce a v3 envelope (currently Pebble). Zero // value means "engine default" (PayloadEncodingIndexedZstd for // Pebble). - PayloadEncoding PayloadEncoding + PayloadEncoding c1zstore.PayloadEncoding // DecoderPool optionally scopes v3 payload-decoder reuse to the // caller's operation (see WithDecoderPool). Nil means a one-shot @@ -84,20 +85,20 @@ func WithDecoderPool(p *EnvelopeDecoderPool) C1ZOption { // and Pebble drivers are both registered statically by this package; // RegisterEngine exists for additional engines. type EngineDriver interface { - Engine() Engine + Engine() c1zstore.Engine Format() C1ZFormat - OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (C1ZStore, error) + OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (c1zstore.Store, error) } type engineRegistry struct { mu sync.RWMutex - byEngine map[Engine]EngineDriver + byEngine map[c1zstore.Engine]EngineDriver } var defaultEngineRegistry = &engineRegistry{ - byEngine: map[Engine]EngineDriver{ - EngineSQLite: sqliteDriver{}, - EnginePebble: pebbleDriver{}, + byEngine: map[c1zstore.Engine]EngineDriver{ + c1zstore.EngineSQLite: sqliteDriver{}, + c1zstore.EnginePebble: pebbleDriver{}, }, } @@ -108,7 +109,7 @@ func RegisterEngine(driver EngineDriver) error { } // EngineDriverFor returns the registered driver for engine. -func EngineDriverFor(engine Engine) (EngineDriver, bool) { +func EngineDriverFor(engine c1zstore.Engine) (EngineDriver, bool) { return defaultEngineRegistry.driverForEngine(engine) } @@ -134,7 +135,7 @@ func (r *engineRegistry) register(driver EngineDriver) error { return nil } -func (r *engineRegistry) driverForEngine(engine Engine) (EngineDriver, bool) { +func (r *engineRegistry) driverForEngine(engine c1zstore.Engine) (EngineDriver, bool) { r.mu.RLock() defer r.mu.RUnlock() driver, ok := r.byEngine[engine] @@ -145,7 +146,7 @@ func (r *engineRegistry) driverForEngine(engine Engine) (EngineDriver, bool) { // the engine-neutral constructor for callers that may opt into non-default // engines. NewC1ZFile remains the concrete SQLite constructor for legacy // callers that need *C1File. -func NewStore(ctx context.Context, outputFilePath string, opts ...C1ZOption) (C1ZStore, error) { +func NewStore(ctx context.Context, outputFilePath string, opts ...C1ZOption) (c1zstore.Store, error) { options, err := buildC1ZOptions(opts...) if err != nil { return nil, err @@ -193,7 +194,7 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions { MaxDecoderMemoryBytes: maxDecoderMemoryBytes, } if out.Engine == "" { - out.Engine = EngineSQLite + out.Engine = c1zstore.EngineSQLite } out.Pragmas = make([]StorePragma, 0, len(options.pragmas)) for _, p := range options.pragmas { @@ -224,10 +225,10 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO l := ctxzap.Extract(ctx) requested := options.engine if requested == "" { - requested = EngineSQLite + requested = c1zstore.EngineSQLite } - stat, err := os.Stat(outputFilePath) + stat, err := os.Stat(outputFilePath) // #nosec G703 -- c1z path is caller-controlled by API design. switch { case errors.Is(err, os.ErrNotExist): return requireEngineDriver(requested) @@ -237,7 +238,7 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return requireEngineDriver(requested) } - f, err := os.Open(outputFilePath) + f, err := os.Open(outputFilePath) // #nosec G703 -- c1z path is caller-controlled by API design. if err != nil { return nil, err } @@ -252,11 +253,11 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return nil, err } - var fileEngine Engine + var fileEngine c1zstore.Engine switch format { case C1ZFormatV1: // Maybe error if the file is read-only? - if requested == EnginePebble && !options.readOnly { + if requested == c1zstore.EnginePebble && !options.readOnly { // Close our header-read handle before converting: the conversion // renames a temp file over outputFilePath, which fails on Windows // if any handle to the destination is still open. Nil out f so @@ -271,9 +272,9 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return nil, fmt.Errorf("select-store-driver: convert existing v1 c1z to pebble: %w", err) } l.Debug("converted existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) - return requireEngineDriver(EnginePebble) + return requireEngineDriver(c1zstore.EnginePebble) } - fileEngine = EngineSQLite + fileEngine = c1zstore.EngineSQLite case C1ZFormatV3: if _, err := f.Seek(0, 0); err != nil { return nil, err @@ -288,9 +289,13 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO if err != nil { return nil, err } - fileEngine = Engine(m.GetEngine()) - if fileEngine == PebbleManifestEngine { // single-sync manifest name; same driver - fileEngine = EnginePebble + fileEngine = c1zstore.Engine(m.GetEngine()) + // Current and legacy pebble manifest names all dispatch to the same + // driver; legacy interiors are re-keyed by the on-open id-index + // migration. Unknown (newer) names fall through and fail loudly in + // requireEngineDriver. + if fileEngine == c1zstore.PebbleManifestEngine || fileEngine == c1zstore.PebbleManifestEngineV2 { + fileEngine = c1zstore.EnginePebble } default: return nil, ErrInvalidFile @@ -307,7 +312,7 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return requireEngineDriver(fileEngine) } -func requireEngineDriver(engine Engine) (EngineDriver, error) { +func requireEngineDriver(engine c1zstore.Engine) (EngineDriver, error) { driver, ok := EngineDriverFor(engine) if !ok { return nil, fmt.Errorf("require-engine-driver: %w: %s", ErrEngineNotAvailable, engine) @@ -317,12 +322,12 @@ func requireEngineDriver(engine Engine) (EngineDriver, error) { type sqliteDriver struct{} -func (sqliteDriver) Engine() Engine { return EngineSQLite } -func (sqliteDriver) Format() C1ZFormat { return C1ZFormatV1 } +func (sqliteDriver) Engine() c1zstore.Engine { return c1zstore.EngineSQLite } +func (sqliteDriver) Format() C1ZFormat { return C1ZFormatV1 } -func (sqliteDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (C1ZStore, error) { +func (sqliteDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (c1zstore.Store, error) { c1zOpts := []C1ZOption{ - WithEngine(EngineSQLite), + WithEngine(c1zstore.EngineSQLite), WithEncoderConcurrency(opts.EncoderConcurrency), } if opts.TmpDir != "" { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go deleted file mode 100644 index d3210586..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go +++ /dev/null @@ -1,25 +0,0 @@ -package dotc1z - -import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" - -// The file-operations contract lives in pkg/dotc1z/c1zstore so storage -// engines can implement it without importing this package. These aliases -// preserve the historical dotc1z names. - -// FileOps is the file-level operations sub-store of C1ZStore. See -// c1zstore.FileOps for the full contract. -type FileOps = c1zstore.FileOps - -// CloneSyncOption configures a FileOps.CloneSync call. See -// c1zstore.CloneSyncOption. -type CloneSyncOption = c1zstore.CloneSyncOption - -// CloneSyncOptions carries the engine-neutral knobs for FileOps.CloneSync. -// See c1zstore.CloneSyncOptions. -type CloneSyncOptions = c1zstore.CloneSyncOptions - -// WithCloneTmpDir sets the temporary directory used while assembling the -// cloned c1z. Replaces WithC1FTmpDir at FileOps.CloneSync call sites. -func WithCloneTmpDir(dir string) CloneSyncOption { - return c1zstore.WithCloneTmpDir(dir) -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go index 91cec9e9..bda658a8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go @@ -4,8 +4,6 @@ import ( "bytes" "fmt" "io" - - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" ) // C1ZFormat identifies the on-disk format of a .c1z file. The format byte @@ -42,63 +40,10 @@ func (f C1ZFormat) String() string { // C1Z3FileHeader is the magic byte sequence for v3 files. var C1Z3FileHeader = []byte("C1Z3\x00") -// Engine identifies a storage engine implementation. The engine is -// chosen by callers via WithEngine(...) on write; on read, the engine -// is dictated by the file's magic byte and (for v3) the manifest's -// engine field. The type lives in pkg/dotc1z/c1zstore so engine -// packages can name it without importing dotc1z. -type Engine = c1zstore.Engine - -const ( - // EngineSQLite is the default engine: the v1 .c1z format backed by - // a zstd-compressed SQLite database. Connectors use this; backend - // infra can opt out. - EngineSQLite = c1zstore.EngineSQLite - - // EnginePebble is the v3 engine: a Pebble LSM wrapped in the v3 - // envelope. - EnginePebble = c1zstore.EnginePebble - - // PebbleManifestEngine is the engine name recorded in a single-sync - // Pebble v3 manifest. It deliberately differs from EnginePebble so - // pre-single-sync readers reject the file at dispatch instead of - // reading its keys as empty. See c1zstore.PebbleManifestEngine. - PebbleManifestEngine = c1zstore.PebbleManifestEngine -) - // ErrEngineNotAvailable is returned when a caller requests an engine // that the binary does not support. var ErrEngineNotAvailable = fmt.Errorf("dotc1z: engine not available") -// PayloadEncoding selects the v3 envelope payload framing. Only the -// Pebble engine consults this; SQLite engines ignore it. See -// c1zstore.PayloadEncoding. -type PayloadEncoding = c1zstore.PayloadEncoding - -const ( - // PayloadEncodingUnspecified is the zero value. Means "use the - // engine's default" — IndexedZstd for Pebble. - PayloadEncodingUnspecified = c1zstore.PayloadEncodingUnspecified - - // PayloadEncodingTarZstd is the default Pebble v3 envelope - // encoding: tar of the Pebble directory, compressed with zstd. - PayloadEncodingTarZstd = c1zstore.PayloadEncodingTarZstd - - // PayloadEncodingTar is uncompressed tar. Useful when Pebble's - // L5/L6 SSTs are already zstd-compressed at the engine layer - // (avoids double-compression CPU), or when the storage target - // compresses in transit. - PayloadEncodingTar = c1zstore.PayloadEncodingTar - - // PayloadEncodingIndexedZstd stores each payload file as an - // independent zstd frame with a self-describing header. Opens - // decode frames in parallel, and rewrites of a store opened from - // an indexed file splice unchanged frames verbatim instead of - // re-compressing them (incremental fold compaction relies on - // this). Readers older than this encoding reject the file. - PayloadEncodingIndexedZstd = c1zstore.PayloadEncodingIndexedZstd -) - // ReadHeaderFormat reads the first 5 bytes of reader and returns the // detected format. On return, the reader is positioned immediately // after the header bytes. If reader is also an io.Seeker, it is diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go index 8db6a0e5..042577c5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path/filepath" "strconv" @@ -40,7 +41,6 @@ var ErrEnvelopeTruncated = errors.New("c1z v3: envelope truncated") // usage and protects against a malicious file claiming a billion-byte // manifest length. const maxManifestBytes = 16 << 20 -const maxTarEntryBytes int64 = 4 << 30 // Tar entries larger than this are streamed straight to disk on the // reader goroutine instead of being buffered in memory for the writer @@ -69,19 +69,63 @@ var fcsFailFastDisabled = os.Getenv(fcsFailFastDisableEnvVar) == "1" // bombs in untrusted v3 c1z files. var ErrMaxSizeExceeded = fmt.Errorf("c1z v3: max decoded payload size exceeded, increase via the %s environment variable", maxDecodedSizeEnvVar) -// envSizeBytes reads an env var holding a size in MiB and converts it -// to bytes, falling back to def when unset, unparsable, zero, or large -// enough to overflow the MiB→bytes conversion. -func envSizeBytes(envVar string, def uint64) uint64 { +// envSizeBytesExplicit reads an env var holding a size in MiB and +// converts it to bytes. ok is false when the var is unset, unparsable, +// zero, or large enough to overflow the MiB→bytes conversion. +func envSizeBytesExplicit(envVar string) (uint64, bool) { v := os.Getenv(envVar) if v == "" { - return def + return 0, false } mb, err := strconv.ParseUint(v, 10, 64) if err != nil || mb == 0 || mb > (1<<63)>>20 { - return def + return 0, false } - return mb << 20 + return mb << 20, true +} + +// envSizeBytes is envSizeBytesExplicit with a fallback default. +func envSizeBytes(envVar string, def uint64) uint64 { + if n, ok := envSizeBytesExplicit(envVar); ok { + return n + } + return def +} + +// maxPayloadCompressionRatio scales the automatic decoded-payload +// budget with the size of the envelope file itself. The budget's job +// is to bound how much disk a hostile envelope can consume at extract +// relative to what was actually stored; a fixed cap can't do that job +// without also refusing legitimate large files (a whale c1z's payload +// decodes to well past any constant that is still meaningful against +// bombs). Real Pebble payloads compress ~2-5x under zstd, so 100x is +// an order of magnitude of headroom while still capping a bomb at +// 100 bytes of output per byte of input. +const maxPayloadCompressionRatio = 100 + +// payloadBudgetForFileSize resolves the decoded-payload budget for an +// envelope of fileSize bytes when the caller configured nothing +// explicit: the env var when set, otherwise the LARGER of the flat +// default and fileSize × maxPayloadCompressionRatio. This is what +// keeps a legitimately huge envelope openable with default settings — +// the flat default alone would reject any file whose raw payload +// exceeds it, even though the file was written by us moments earlier. +func payloadBudgetForFileSize(fileSize int64) uint64 { + if n, ok := envSizeBytesExplicit(maxDecodedSizeEnvVar); ok { + return n + } + budget := defaultMaxDecodedPayloadBytes + if fileSize > 0 { + scaled := uint64(fileSize) + if scaled > math.MaxUint64/maxPayloadCompressionRatio { + return math.MaxUint64 + } + scaled *= maxPayloadCompressionRatio + if scaled > budget { + budget = scaled + } + } + return budget } func maxDecodedPayloadBytes() uint64 { @@ -94,9 +138,14 @@ func decoderMaxMemoryBytes() uint64 { type payloadOptions struct { maxDecodedPayloadBytes uint64 - maxDecoderMemoryBytes uint64 - disableSizeFailFast bool - pool *DecoderPool + // budgetExplicit is true when the decoded-payload budget came from + // the caller (WithMaxDecodedPayloadBytes) rather than defaults; + // only non-explicit budgets are rescaled by the envelope file size + // (see payloadBudgetForFileSize). + budgetExplicit bool + maxDecoderMemoryBytes uint64 + disableSizeFailFast bool + pool *DecoderPool } type PayloadOption func(*payloadOptions) @@ -107,6 +156,7 @@ type PayloadOption func(*payloadOptions) func WithMaxDecodedPayloadBytes(n uint64) PayloadOption { return func(o *payloadOptions) { o.maxDecodedPayloadBytes = n + o.budgetExplicit = n > 0 } } @@ -493,6 +543,15 @@ func readEnvelope(r io.Reader, headerOnly bool, pool *DecoderPool) (*Envelope, e return nil, err } // 4. Payload. The reader is positioned at the first payload byte. + // When the reader is a real file, scale the decoded-byte budget + // with its size (see payloadBudgetForFileSize); a non-stat-able + // stream falls back to the flat default. + budget := maxDecodedPayloadBytes() + if st, ok := r.(interface{ Stat() (os.FileInfo, error) }); ok { + if fi, err := st.Stat(); err == nil { + budget = payloadBudgetForFileSize(fi.Size()) + } + } env := &Envelope{Manifest: m} switch m.GetPayloadEncoding() { case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR_ZSTD: @@ -502,9 +561,9 @@ func readEnvelope(r io.Reader, headerOnly bool, pool *DecoderPool) (*Envelope, e } env.zstdReader = zr env.pool = pool - env.PayloadReader = &limitedPayloadReader{r: zr, limit: maxDecodedPayloadBytes()} + env.PayloadReader = &limitedPayloadReader{r: zr, limit: budget} case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR: - env.PayloadReader = &limitedPayloadReader{r: r, limit: maxDecodedPayloadBytes()} + env.PayloadReader = &limitedPayloadReader{r: r, limit: budget} case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_INDEXED_ZSTD: // Indexed payloads are not a tar stream; extraction goes // through ExtractEnvelopePayload (random access over the @@ -592,6 +651,19 @@ func unmarshalManifestHeader(b []byte) (*c1zv3.C1ZManifestV3, error) { } out.SetFoldDeadBytes(int64(v)) //nolint:gosec // proto int64 varint round-trips through uint64 by definition. b = b[n:] + case 42: + if typ != protowire.VarintType { + return nil, fmt.Errorf("c1z v3: manifest pebble_id_index_format has wire type %v", typ) + } + v, n := protowire.ConsumeVarint(b) + if n < 0 { + return nil, protowire.ParseError(n) + } + if v > uint64(1<<31-1) { + return nil, fmt.Errorf("c1z v3: manifest pebble_id_index_format overflow: %d", v) + } + out.SetPebbleIdIndexFormat(c1zv3.PebbleIdIndexFormat(v)) + b = b[n:] default: n := protowire.ConsumeFieldValue(num, typ, b) if n < 0 { @@ -736,7 +808,7 @@ func writeTar(w io.Writer, dir string) error { // writer worker pool; workers perform the per-file open/write/close // syscalls in parallel. Larger entries are streamed straight to disk // on this goroutine so a hostile archive full of multi-GiB entries -// can't drive memory to extractWorkerCount × maxTarEntryBytes. Memory +// can't drive memory up with the worker fan-out. Memory // peak is bounded by (extractWorkerCount + channel buffer) × // inlineCopyThresholdBytes; at Pebble's typical 2 MiB FlushSplitBytes // nearly every entry takes the parallel path — the per-entry @@ -820,8 +892,13 @@ entryLoop: break entryLoop } case tar.TypeReg: - if hdr.Size < 0 || hdr.Size > maxTarEntryBytes { - readErr = fmt.Errorf("c1z v3: tar entry %q size %d exceeds cap %d", hdr.Name, hdr.Size, maxTarEntryBytes) + // No per-entry size cap: a single Pebble SST can legitimately + // exceed any fixed bound. Aggregate extraction is bounded by + // the caller's decoded-byte budget (limitedPayloadReader), and + // memory by inlineCopyThresholdBytes — larger entries stream + // straight to disk. + if hdr.Size < 0 { + readErr = fmt.Errorf("c1z v3: tar entry %q has negative size %d", hdr.Name, hdr.Size) break entryLoop } if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go index 6063c38e..1d051df4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go @@ -464,12 +464,27 @@ func readIndexedTrailer(f *os.File, payloadStart int64) (*c1zv3.IndexedFrameInde // positive: WithZeroFrames guarantees even an empty file encodes // to a complete zstd frame, so a zero-length frame range can // only come from a corrupt or hand-mangled index. + // + // There is deliberately NO upper bound on either size: a single + // Pebble SST can legitimately exceed any fixed cap (whale-scale + // grants buckets are written as one whole-bucket SST), and the + // bomb protections live elsewhere — compSize is bounds-checked + // against the real file layout just below, and total decoded + // output is enforced by the extraction budget + // (BATON_DECODER_MAX_DECODED_SIZE_MB) regardless of what + // raw_size claims. rawSize, compSize := e.GetRawSize(), e.GetCompressedSize() - if rawSize < 0 || rawSize > maxTarEntryBytes || compSize <= 0 || compSize > maxTarEntryBytes { + if rawSize < 0 || compSize <= 0 { return nil, nil, fmt.Errorf("c1z v3: trailer index entry %q sizes out of range (raw=%d comp=%d)", name, rawSize, compSize) } + // Overflow-safe form of off+compSize > indexOff: with no upper + // bound on compSize, the addition could wrap negative for a + // hostile compressed_size near MaxInt64 and slip past the + // comparison. Subtraction can't wrap here (off >= payloadStart + // >= 0 and indexOff fits the file), and when off > indexOff the + // negative difference rejects too, as it must. off := e.GetFrameOffset() - if off < payloadStart || off+compSize > indexOff { + if off < payloadStart || compSize > indexOff-off { return nil, nil, fmt.Errorf("c1z v3: trailer index entry %q frame range out of bounds (off=%d comp=%d)", name, off, compSize) } if len(e.GetRawSha256()) != sha256.Size { @@ -665,6 +680,17 @@ func extractOneFrame(f *os.File, e *ReuseEntry, dec *zstd.Decoder, budget *decod // owned by the caller. func ExtractEnvelopePayload(f *os.File, destDir string, opts ...PayloadOption) (*c1zv3.C1ZManifestV3, *PayloadReuse, error) { cfg := resolvePayloadOptions(opts...) + // With no explicit budget from the caller, scale the decoded-byte + // budget with the envelope's own size: the flat default would + // refuse any legitimately large file (one WE wrote), while the + // scaled budget still bounds a hostile envelope's disk consumption + // proportionally to its actual size. The env var, when set, wins + // inside payloadBudgetForFileSize. + if !cfg.budgetExplicit { + if st, err := f.Stat(); err == nil { + cfg.maxDecodedPayloadBytes = payloadBudgetForFileSize(st.Size()) + } + } if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, nil, err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go deleted file mode 100644 index 9aad68b8..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go +++ /dev/null @@ -1,19 +0,0 @@ -package dotc1z - -import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" - -// The grant-store contract lives in pkg/dotc1z/c1zstore so storage engines -// can implement it without importing this package. These aliases preserve -// the historical dotc1z names. - -// GrantStore is the grant-specific slice of C1ZStore. See -// c1zstore.GrantStore for the full contract. -type GrantStore = c1zstore.GrantStore - -// PendingExpansion is a lightweight row yielded by -// GrantStore.PendingExpansion. See c1zstore.PendingExpansion. -type PendingExpansion = c1zstore.PendingExpansion - -// GrantAnnotation is a row yielded by GrantStore.ListWithAnnotations. See -// c1zstore.GrantAnnotation. -type GrantAnnotation = c1zstore.GrantAnnotation diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants.go index c76bb6d7..505a7a9c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants.go @@ -414,7 +414,8 @@ func (c *C1File) resolveSyncIDForGrantGet(ctx context.Context, explicit string) // GrantsForEntitlementPrincipalSorted reports false: the SQLite engine orders // ListGrantsForEntitlement results by grant id, not by principal. The grant // expander uses this to fall back to its buffering/sorting path; only the -// Pebble engine (whose by_entitlement index is principal-keyed) reports true. +// Pebble engine (whose primary grant key is entitlement/principal-keyed) reports +// true. func (c *C1File) GrantsForEntitlementPrincipalSorted() bool { return false } func (c *C1File) ListGrantsForEntitlement( diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants_for_entitlements.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants_for_entitlements.go index ac42da4c..bd00f314 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants_for_entitlements.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants_for_entitlements.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "hash/crc32" - "sort" "strconv" "time" @@ -212,17 +211,16 @@ func findEntitlementIndex(ents []*v2.Entitlement, id string) int { return -1 } -// entitlementListChecksum hashes the sorted entitlement ID set so -// reorderings don't bust the cursor. Drop/add does. +// entitlementListChecksum hashes the entitlement ID list IN REQUEST +// ORDER. The cursor resumes positionally (ents[startIdx] provides the +// resume entitlement id), so a reordered same-set list must restart the +// scan — a sorted (order-insensitive) checksum would bless the stale +// token while startIdx selected a different entitlement, silently +// skipping and re-returning grants. func entitlementListChecksum(ents []*v2.Entitlement) uint32 { - ids := make([]string, 0, len(ents)) - for _, e := range ents { - ids = append(ids, e.GetId()) - } - sort.Strings(ids) h := crc32.NewIEEE() - for _, id := range ids { - _, _ = h.Write([]byte(id)) + for _, e := range ents { + _, _ = h.Write([]byte(e.GetId())) _, _ = h.Write([]byte{0}) } return h.Sum32() diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go index 9a3cf839..f30f25b0 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go @@ -12,18 +12,22 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "google.golang.org/protobuf/types/known/anypb" c1zv3 "github.com/conductorone/baton-sdk/pb/c1/c1z/v3" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" ) // pebbleDriver is the EngineDriver for the Pebble v3 engine. type pebbleDriver struct{} -var _ C1ZStore = (*pebbleStore)(nil) +var _ c1zstore.Store = (*pebbleStore)(nil) var _ connectorstore.Writer = (*pebbleStore)(nil) // Local mirrors of the optional capabilities the c1z sanitizer probes on @@ -38,7 +42,7 @@ type sanitizeSupportsDiffWriter interface { SetSupportsDiff(ctx context.Context, syncID string) error } type sanitizeSyncRunMetadataReader interface { - ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*SyncRun, string, error) + ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*c1zstore.SyncRun, string, error) } var ( @@ -50,10 +54,10 @@ var ( _ sanitizeSyncRunMetadataReader = (*C1File)(nil) ) -func (pebbleDriver) Engine() Engine { return EnginePebble } -func (pebbleDriver) Format() C1ZFormat { return C1ZFormatV3 } +func (pebbleDriver) Engine() c1zstore.Engine { return c1zstore.EnginePebble } +func (pebbleDriver) Format() C1ZFormat { return C1ZFormatV3 } -func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (C1ZStore, error) { +func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (c1zstore.Store, error) { tmpDir, err := os.MkdirTemp(opts.TmpDir, "c1z-pebble") if err != nil { return nil, err @@ -71,18 +75,47 @@ func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts S return nil, cleanupOnError(err) } + if opts.ReadOnly { + // A read-only open of a missing or empty c1z must fail loudly (as it + // does on main via pebble's ErrDBDoesNotExist), not silently create + // an empty DB in the temp dir: the writable migration pre-open below + // would otherwise mint a fresh store and mask a mistyped path as + // "file has no syncs". unpackExistingPebbleC1Z creates dbDir exactly + // when it actually unpacked a payload. + if _, statErr := os.Stat(dbDir); statErr != nil { + return nil, cleanupOnError(fmt.Errorf("pebble: read-only open: %s does not exist or is empty", outputFilePath)) + } + // Match SQLite read-only semantics: the source c1z is immutable, but the + // unpacked temp DB may be migrated so current read paths see the latest + // layout. Reopen read-only afterwards so callers that reach the engine + // directly still get read-only write barriers. + migratingEngine, err := pebble.Open(ctx, dbDir) + if err != nil { + return nil, cleanupOnError(err) + } + if err := migratingEngine.Close(); err != nil { + return nil, cleanupOnError(err) + } + } + e, err := pebble.Open(ctx, dbDir, pebble.WithReadOnly(opts.ReadOnly)) if err != nil { return nil, cleanupOnError(err) } encoding := opts.PayloadEncoding - if encoding == PayloadEncodingUnspecified { + if encoding == c1zstore.PayloadEncodingUnspecified { encoding = fileEncoding } adapter := pebble.NewAdapter(e) err = adapter.InitCurrentSync(ctx) if err != nil { + // Close the engine before removing its directory: a live pebble DB + // holds open fds and background goroutines that would otherwise + // leak for the life of the process. + if closeErr := e.Close(); closeErr != nil { + err = errors.Join(err, closeErr) + } return nil, cleanupOnError(err) } @@ -97,6 +130,11 @@ func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts S foldDeadBytes: foldDeadBytes, syncLimit: opts.SyncLimit, skipCleanup: opts.SkipCleanup, + // A writable open that ran the in-place id-index migration must + // save the migrated layout back into the c1z even if the caller + // never writes, or every subsequent open re-pays the O(rows) + // migration. + dirty: !opts.ReadOnly && e.MigratedOnOpen(), }, nil } @@ -106,32 +144,32 @@ func unpackExistingPebbleC1Z( maxDecodedPayloadBytes uint64, maxDecoderMemoryBytes uint64, pool *EnvelopeDecoderPool, -) (*formatv3.PayloadReuse, PayloadEncoding, int64, error) { +) (*formatv3.PayloadReuse, c1zstore.PayloadEncoding, int64, error) { stat, err := os.Stat(outputFilePath) switch { case errors.Is(err, os.ErrNotExist): - return nil, PayloadEncodingUnspecified, 0, nil + return nil, c1zstore.PayloadEncodingUnspecified, 0, nil case err != nil: - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err case stat.Size() == 0: - return nil, PayloadEncodingUnspecified, 0, nil + return nil, c1zstore.PayloadEncodingUnspecified, 0, nil } f, err := os.Open(outputFilePath) if err != nil { - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err } defer f.Close() header, err := formatv3.ReadManifestHeader(f) if err != nil { - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err } - if e := Engine(header.GetEngine()); e != EnginePebble && e != PebbleManifestEngine { - return nil, PayloadEncodingUnspecified, 0, fmt.Errorf("%w: %s", pebble.ErrUnknownEngine, header.GetEngine()) + if e := c1zstore.Engine(header.GetEngine()); e != c1zstore.EnginePebble && e != c1zstore.PebbleManifestEngine && e != c1zstore.PebbleManifestEngineV2 { + return nil, c1zstore.PayloadEncodingUnspecified, 0, fmt.Errorf("%w: %s", pebble.ErrUnknownEngine, header.GetEngine()) } if err := os.MkdirAll(dbDir, 0o755); err != nil { - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err } manifest, reuse, err := formatv3.ExtractEnvelopePayload(f, dbDir, formatv3.WithMaxDecodedPayloadBytes(maxDecodedPayloadBytes), @@ -139,7 +177,7 @@ func unpackExistingPebbleC1Z( formatv3.WithPayloadDecoderPool(pool), ) if err != nil { - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err } // fold_dead_bytes is inherited from the source file so the waste // accounting survives arbitrary open/save cycles, not just fold @@ -147,18 +185,18 @@ func unpackExistingPebbleC1Z( return reuse, payloadEncodingFromProto(manifest.GetPayloadEncoding()), header.GetFoldDeadBytes(), nil } -func payloadEncodingFromProto(enc c1zv3.PayloadEncoding) PayloadEncoding { +func payloadEncodingFromProto(enc c1zv3.PayloadEncoding) c1zstore.PayloadEncoding { switch enc { case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR: - return PayloadEncodingTar + return c1zstore.PayloadEncodingTar case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_INDEXED_ZSTD: - return PayloadEncodingIndexedZstd + return c1zstore.PayloadEncodingIndexedZstd case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR_ZSTD: - return PayloadEncodingTarZstd + return c1zstore.PayloadEncodingTarZstd case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_UNSPECIFIED: - return PayloadEncodingUnspecified + return c1zstore.PayloadEncodingUnspecified default: - return PayloadEncodingUnspecified + return c1zstore.PayloadEncodingUnspecified } } @@ -168,7 +206,7 @@ type pebbleStore struct { outputFilePath string tmpDir string readOnly bool - payloadEncoding PayloadEncoding + payloadEncoding c1zstore.PayloadEncoding payloadReuse *formatv3.PayloadReuse // foldDeadBytes is the cumulative fold-waste counter carried in // the envelope manifest (C1ZManifestV3.fold_dead_bytes): seeded @@ -196,7 +234,7 @@ type pebbleStore struct { // Close(ctx) signature. Lets callers route Pebble stores through // pkg/sync.NewSyncer's WithConnectorStore option the same way they // route SQLite *C1File handles today. -var _ C1ZStore = (*pebbleStore)(nil) +var _ c1zstore.Store = (*pebbleStore)(nil) // FileOps overrides the Adapter-level FileOps for two reasons: // @@ -207,7 +245,7 @@ var _ C1ZStore = (*pebbleStore)(nil) // flip the dirty bit — without it, Close would skip the envelope // save and the diff sync would exist only in the discarded temp // directory. -func (s *pebbleStore) FileOps() FileOps { +func (s *pebbleStore) FileOps() c1zstore.FileOps { return pebbleStoreFileOps{inner: s.FileOpsWithEncoding(s.payloadEncoding), store: s} } @@ -216,15 +254,15 @@ func (s *pebbleStore) FileOps() FileOps { // dirty-marking path. CloneSync writes a separate file and passes // through unchanged. type pebbleStoreFileOps struct { - inner FileOps + inner c1zstore.FileOps store *pebbleStore } -func (f pebbleStoreFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { +func (f pebbleStoreFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { return f.inner.CloneSync(ctx, outPath, syncID, opts...) } -func (f pebbleStoreFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { +func (f pebbleStoreFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { return f.inner.CopyIsolateSync(ctx, outPath, syncID, opts...) } @@ -247,8 +285,8 @@ func (f pebbleStoreFileOps) GenerateSyncDiff(ctx context.Context, baseSyncID, ap func (s *pebbleStore) Metadata() connectorstore.StoreMetadata { md := s.Adapter.Metadata() enc := s.payloadEncoding - if enc == PayloadEncodingUnspecified { - enc = PayloadEncodingIndexedZstd + if enc == c1zstore.PayloadEncodingUnspecified { + enc = c1zstore.PayloadEncodingIndexedZstd } md.PayloadEncoding = enc.String() return md @@ -458,11 +496,18 @@ func (s *pebbleStore) DeleteGrant(ctx context.Context, grantID string) error { return s.markDirty(s.Adapter.DeleteGrant(ctx, grantID)) } +// DeleteGrantByRefs is the exact grant delete for callers holding the full +// grant: identity derives from the structured refs, never the lossy id +// string. The syncer prefers this when available. +func (s *pebbleStore) DeleteGrantByRefs(ctx context.Context, grant *v2.Grant) error { + return s.markDirty(s.Adapter.DeleteGrantByRefs(ctx, grant)) +} + // Grants overrides Adapter.Grants() so the returned GrantStore // routes StoreExpandedGrants through the pebbleStore's dirty-marking // path. The Adapter-level wrapper calls Adapter.PutGrants directly, // which skips the dirty flag. -func (s *pebbleStore) Grants() GrantStore { +func (s *pebbleStore) Grants() c1zstore.GrantStore { return pebbleStoreGrants{inner: s.Adapter.Grants(), store: s} } @@ -470,33 +515,152 @@ func (s *pebbleStore) Grants() GrantStore { // only StoreExpandedGrants (the lone mutating method) to flip the // dirty bit. Read-only methods pass through. type pebbleStoreGrants struct { - inner GrantStore + inner c1zstore.GrantStore store *pebbleStore } +var pebbleStoreExpandedGrantImmutableAnnotationAny = func() *anypb.Any { + a, err := anypb.New(&v2.GrantImmutable{}) + if err != nil { + panic(fmt.Errorf("dotc1z: marshal GrantImmutable annotation: %w", err)) + } + return a +}() + func (g pebbleStoreGrants) StoreExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { return g.store.markDirty(g.inner.StoreExpandedGrants(ctx, grants...)) } -func (g pebbleStoreGrants) PendingExpansionPage(ctx context.Context, pageToken string) ([]PendingExpansion, string, error) { +func (g pebbleStoreGrants) StoreNewExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { + if fast, ok := g.inner.(interface { + StoreNewExpandedGrants(context.Context, ...*v2.Grant) error + }); ok { + return g.store.markDirty(fast.StoreNewExpandedGrants(ctx, grants...)) + } + return g.store.markDirty(g.inner.StoreExpandedGrants(ctx, grants...)) +} + +func (g pebbleStoreGrants) StoreNewExpandedGrantContributions(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error { + if fast, ok := g.inner.(interface { + StoreNewExpandedGrantContributions(context.Context, *v2.Entitlement, []*v3.PrincipalRef, []batonGrant.Sources) error + }); ok { + return g.store.markDirty(fast.StoreNewExpandedGrantContributions(ctx, dest, principals, sources)) + } + grants := make([]*v2.Grant, 0, len(principals)) + for i, principalRef := range principals { + principal := newPebbleStorePrincipalResource(principalRef) + grant, err := newPebbleStoreExpandedGrant(dest, principal, sources[i]) + if err != nil { + return err + } + grants = append(grants, grant) + } + return g.store.markDirty(g.inner.StoreExpandedGrants(ctx, grants...)) +} + +// pebbleStoreGrantLayerStorer is the layer-scoped layer session surface the +// engine-level grant store may implement (see the pebble adapter). Kept as a +// local interface so the wrapper can pass sessions through with the dirty bit. +type pebbleStoreGrantLayerStorer interface { + BeginExpandedGrantLayer(ctx context.Context) (bool, error) + AddExpandedGrantLayerContributions(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error + FinishExpandedGrantLayer(ctx context.Context) error + AbortExpandedGrantLayer(ctx context.Context) error +} + +func (g pebbleStoreGrants) BeginExpandedGrantLayer(ctx context.Context) (bool, error) { + if fast, ok := g.inner.(pebbleStoreGrantLayerStorer); ok { + return fast.BeginExpandedGrantLayer(ctx) + } + return false, nil +} + +func (g pebbleStoreGrants) AddExpandedGrantLayerContributions(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error { + fast, ok := g.inner.(pebbleStoreGrantLayerStorer) + if !ok { + return fmt.Errorf("expanded grant layer: store does not support layer sessions") + } + return fast.AddExpandedGrantLayerContributions(ctx, dest, principals, sources) +} + +func (g pebbleStoreGrants) FinishExpandedGrantLayer(ctx context.Context) error { + fast, ok := g.inner.(pebbleStoreGrantLayerStorer) + if !ok { + return fmt.Errorf("expanded grant layer: store does not support layer sessions") + } + return g.store.markDirty(fast.FinishExpandedGrantLayer(ctx)) +} + +func (g pebbleStoreGrants) AbortExpandedGrantLayer(ctx context.Context) error { + fast, ok := g.inner.(pebbleStoreGrantLayerStorer) + if !ok { + return nil + } + return fast.AbortExpandedGrantLayer(ctx) +} + +func newPebbleStorePrincipalResource(ref *v3.PrincipalRef) *v2.Resource { + if ref == nil { + return nil + } + var parent *v2.ResourceId + if ref.GetParentResourceId() != "" { + parent = v2.ResourceId_builder{ + ResourceType: ref.GetParentResourceTypeId(), + Resource: ref.GetParentResourceId(), + }.Build() + } + return v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: ref.GetResourceTypeId(), + Resource: ref.GetResourceId(), + }.Build(), + ParentResourceId: parent, + }.Build() +} + +func newPebbleStoreExpandedGrant(dest *v2.Entitlement, principal *v2.Resource, sources batonGrant.Sources) (*v2.Grant, error) { + if len(sources) == 0 { + return nil, fmt.Errorf("new expanded grant: empty sources") + } + if dest == nil || dest.GetResource() == nil { + return nil, fmt.Errorf("new expanded grant: entitlement has no resource") + } + if principal == nil { + return nil, fmt.Errorf("new expanded grant: principal is nil") + } + sourceMap := make(map[string]*v2.GrantSources_GrantSource, len(sources)) + for _, src := range sources { + sourceMap[src.EntitlementID] = &v2.GrantSources_GrantSource{IsDirect: src.IsDirect} + } + return v2.Grant_builder{ + Id: batonGrant.NewGrantID(principal, dest), + Entitlement: dest, + Principal: principal, + Sources: v2.GrantSources_builder{Sources: sourceMap}.Build(), + Annotations: []*anypb.Any{pebbleStoreExpandedGrantImmutableAnnotationAny}, + }.Build(), nil +} + +func (g pebbleStoreGrants) PendingExpansionPage(ctx context.Context, pageToken string) ([]c1zstore.PendingExpansion, string, error) { return g.inner.PendingExpansionPage(ctx, pageToken) } -func (g pebbleStoreGrants) PendingExpansion(ctx context.Context) iter.Seq2[PendingExpansion, error] { +func (g pebbleStoreGrants) PendingExpansion(ctx context.Context) iter.Seq2[c1zstore.PendingExpansion, error] { return g.inner.PendingExpansion(ctx) } -func (g pebbleStoreGrants) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]GrantAnnotation, string, error) { +func (g pebbleStoreGrants) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]c1zstore.GrantAnnotation, string, error) { return g.inner.ListWithAnnotationsPage(ctx, pageToken) } func (g pebbleStoreGrants) ListWithAnnotationsForResourcePage( ctx context.Context, resource *v2.Resource, syncID string, pageToken string, pageSize uint32, -) ([]GrantAnnotation, string, error) { +) ([]c1zstore.GrantAnnotation, string, error) { return g.inner.ListWithAnnotationsForResourcePage(ctx, resource, syncID, pageToken, pageSize) } -func (g pebbleStoreGrants) ListWithAnnotations(ctx context.Context) iter.Seq2[GrantAnnotation, error] { +func (g pebbleStoreGrants) ListWithAnnotations(ctx context.Context) iter.Seq2[c1zstore.GrantAnnotation, error] { return g.inner.ListWithAnnotations(ctx) } @@ -506,6 +670,20 @@ func (s *pebbleStore) Close(ctx context.Context) (retErr error) { if s.closed { return nil } + + if !s.readOnly && s.dirty { + if err := s.save(ctx); err != nil { + // Tear NOTHING down: the unpacked DB under tmpDir is the only + // copy of the synced data, and save failures are frequently + // transient (target-path permissions, disk space for the + // envelope). Leave the store open so the caller can fix the + // condition and Close again; if the process exits instead, the + // temp dir survives on disk for manual recovery rather than + // being deleted out from under a failed save. + return fmt.Errorf("pebble store close: save failed, store left open and unsaved data preserved under %s: %w", s.tmpDir, err) + } + s.dirty = false + } s.closed = true defer func() { @@ -514,11 +692,6 @@ func (s *pebbleStore) Close(ctx context.Context) (retErr error) { } }() - if !s.readOnly && s.dirty { - if err := s.save(ctx); err != nil { - retErr = errors.Join(retErr, err) - } - } if err := s.engine.Close(); err != nil { retErr = errors.Join(retErr, err) } @@ -531,6 +704,13 @@ func (s *pebbleStore) save(ctx context.Context) error { } saveStart := time.Now() checkpointDir := filepath.Join(s.tmpDir, "checkpoint") + // A previous failed save can leave this dir behind, and pebble's + // Checkpoint refuses an existing destination — without this, the + // "fix the condition and Close again" recovery path advertised by + // Close would fail forever with ErrExist. + if err := os.RemoveAll(checkpointDir); err != nil { + return fmt.Errorf("pebble save: clear stale checkpoint dir: %w", err) + } if err := s.engine.CheckpointTo(ctx, checkpointDir); err != nil { return err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go new file mode 100644 index 00000000..80171981 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go @@ -0,0 +1,213 @@ +package dotc1z + +import ( + "context" + "errors" + "fmt" + + cdbpebble "github.com/cockroachdb/pebble/v2" + + "github.com/conductorone/baton-sdk/pkg/bid" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult = pebble.SourceCacheReplayResult + +// SourceCacheStore is the optional store capability backing source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). It is +// implemented ONLY by the Pebble engine; the syncer type-asserts for it and +// treats a store without it as "source cache unsupported" (no-op lookup, +// no replay). It is deliberately NOT part of c1zstore.Store. +type SourceCacheStore interface { + // LookupSourceCacheEntry returns this store's manifest entry for + // (kind, scopeHash). Backs the connector-facing lookup when this + // store is the previous sync. + LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) + + // PutSourceCacheEntry writes the current sync's manifest entry for + // (kind, scopeHash). Zero-row scopes still get entries. + PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error + + // ReplaySourceCache copies every row stamped with scopeHash from prev + // (the previous sync's store, opened read-only) into this store. prev + // must be a Pebble store. Does NOT write the manifest entry — the + // caller writes it after the scope's overlay/deletes complete, so a + // failed replay can't leave a phantom hit for the next sync. + ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) + + // DeleteSourceCacheRows removes rows by public canonical ID from the + // current sync, after replay + overlay (delta-query tombstones). + // ID formats per kind: grants and entitlements use their canonical + // IDs; resources use Baton resource BIDs ("bid:r:..."). + // + // Grant resolution is BOUNDED: candidate probing only, never the + // O(all grants) stored-external-id scan. Grants stored under + // connector-custom ids are unreachable here (delete no-ops); such + // connectors use DeleteSourceCacheRowsInScope instead. + DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error + + // DeleteSourceCacheRowsInScope removes rows stamped with scopeHash by + // bare object id — grants by principal id (no principal type, no + // canonical-id reconstruction), resources by resource id (any type). + // One index scan of the scope per call; a page's tombstones are + // batched into one call. Ids with no matching rows are no-ops. + // Not supported for entitlements. + DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) + + // DeleteSourceCacheGrantsByIDInScope removes grant rows stamped with + // scopeHash whose STORED grant id is in ids — works for + // connector-custom grant-id shapes that the global bounded delete + // cannot resolve, and stays bounded by the scope's row count. Ids with + // no matching rows are no-ops. + DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) +} + +var _ SourceCacheStore = (*pebbleStore)(nil) + +// sourceCacheEngine recovers the Pebble engine from an arbitrary store, +// nil-safe. Mirrors pebble.AsEngine but accepts any value so the syncer +// can probe its previous-sync reader without caring about its static type. +func sourceCacheEngine(store any) (*pebble.Engine, bool) { + a, ok := store.(interface{ PebbleEngine() *pebble.Engine }) + if !ok { + return nil, false + } + e := a.PebbleEngine() + return e, e != nil +} + +func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return sourcecache.Entry{}, false, err + } + rec, err := s.engine.GetSourceCacheEntry(ctx, string(kind), scopeHash) + if err != nil { + if errors.Is(err, cdbpebble.ErrNotFound) { + return sourcecache.Entry{}, false, nil + } + return sourcecache.Entry{}, false, err + } + return sourcecache.Entry{ + ETag: rec.GetEtag(), + DiscoveredAt: rec.GetDiscoveredAt().AsTime(), + }, true, nil +} + +func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return err + } + return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeHash, etag)) +} + +func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) { + prevEngine, ok := sourceCacheEngine(prev) + if !ok { + return SourceCacheReplayResult{}, errors.New("source cache replay: previous sync store is not a pebble store") + } + var res SourceCacheReplayResult + var err error + switch kind { + case sourcecache.RowKindResources: + res, err = s.engine.ReplaySourceCacheResources(ctx, prevEngine, scopeHash) + case sourcecache.RowKindEntitlements: + res, err = s.engine.ReplaySourceCacheEntitlements(ctx, prevEngine, scopeHash) + case sourcecache.RowKindGrants: + res, err = s.engine.ReplaySourceCacheGrants(ctx, prevEngine, scopeHash) + default: + return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: invalid row kind %q", kind) + } + if err != nil { + return SourceCacheReplayResult{}, err + } + if res.Rows > 0 { + s.MarkDirty() + } + return res, nil +} + +// DeleteSourceCacheRows deletes delta tombstones by public id string. +// +// NOTE on the bare-id lookup safety contract (engine/pebble/lookup.go): +// sync paths normally must not resolve grants by string. This path is a +// deliberate, narrow exception: tombstone ids are strings the connector +// itself emitted for these rows, volumes are delta-sized (not O(rows)), +// and resolution keeps the exactly-one rule — an ambiguous id fails the +// sync loudly rather than guessing a delete, which matches the +// source-cache replay-phase error policy. +func (s *pebbleStore) DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error { + for _, id := range ids { + switch kind { + case sourcecache.RowKindGrants: + if err := s.markDirty(s.engine.DeleteGrantRecordBounded(ctx, id)); err != nil { + return fmt.Errorf("source cache delete grant %q: %w", id, err) + } + case sourcecache.RowKindEntitlements: + if err := s.markDirty(s.engine.DeleteEntitlementRecord(ctx, id)); err != nil { + return fmt.Errorf("source cache delete entitlement %q: %w", id, err) + } + case sourcecache.RowKindResources: + r, err := bid.ParseResourceBid(id) + if err != nil { + return fmt.Errorf("source cache delete resource: invalid resource bid %q: %w", id, err) + } + rid := r.GetId() + if err := s.markDirty(s.engine.DeleteResourceRecord(ctx, rid.GetResourceType(), rid.GetResource())); err != nil { + return fmt.Errorf("source cache delete resource %q: %w", id, err) + } + default: + return fmt.Errorf("source cache delete: invalid row kind %q", kind) + } + } + return nil +} + +func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + deleted, err := s.engine.DeleteGrantsByExternalIDsInScope(ctx, scopeHash, idSet) + if err != nil { + return 0, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeHash, err) + } + if deleted > 0 { + s.MarkDirty() + } + return deleted, nil +} + +func (s *pebbleStore) DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + var deleted int64 + var err error + switch kind { + case sourcecache.RowKindGrants: + deleted, err = s.engine.DeleteGrantsByPrincipalsInScope(ctx, scopeHash, idSet) + case sourcecache.RowKindResources: + deleted, err = s.engine.DeleteResourcesByIDsInScope(ctx, scopeHash, idSet) + case sourcecache.RowKindEntitlements: + return 0, fmt.Errorf("source cache scoped delete: not supported for entitlements") + default: + return 0, fmt.Errorf("source cache scoped delete: invalid row kind %q", kind) + } + if err != nil { + return 0, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeHash, err) + } + if deleted > 0 { + s.MarkDirty() + } + return deleted, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go index 301983e3..b4da1e81 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go @@ -574,7 +574,7 @@ func (c *C1File) getResourceObject(ctx context.Context, resourceID *v2.ResourceI case c.viewSyncID != "": q = q.Where(goqu.C("sync_id").Eq(c.viewSyncID)) default: - var latestSyncRun *SyncRun + var latestSyncRun *c1zstore.SyncRun var err error latestSyncRun, err = c.getFinishedSync(ctx, 0, connectorstore.SyncTypeFull) if err != nil { @@ -634,7 +634,7 @@ func (c *C1File) getConnectorObject(ctx context.Context, tableName string, id st case c.viewSyncID != "": q = q.Where(goqu.C("sync_id").Eq(c.viewSyncID)) default: - var latestSyncRun *SyncRun + var latestSyncRun *c1zstore.SyncRun var err error latestSyncRun, err = c.getFinishedSync(ctx, 0, connectorstore.SyncTypeAny) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go index 7d12cb5e..1d7ee54d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go @@ -4,33 +4,16 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" ) -// C1ZStore is the internal contract used by the sync pipeline, compactor, and -// related infrastructure to read and write a .c1z file. The interface lives -// in pkg/dotc1z/c1zstore (as c1zstore.Store) so storage engines can -// implement it without importing this package; this alias preserves the -// historical dotc1z name. -// -// Implementations: -// -// - *C1File — the original SQLite-backed implementation -// (pkg/dotc1z/c1file.go). -// - *pebbleStore — the Pebble v3 engine implementation opened via -// NewStore(WithEngine(EnginePebble)) (pkg/dotc1z/pebble_store.go). -// -// Both engines are registered statically; no extra imports are needed to -// open either format. -type C1ZStore = c1zstore.Store - -// AsSQLiteStore type-asserts a C1ZStore to the concrete *C1File. It is an +// AsSQLiteStore type-asserts a c1zstore.Store to the concrete *C1File. It is an // escape hatch for callers that legitimately need SQLite-specific primitives // (today: the attached compactor in pkg/synccompactor/attached, which uses // SQL ATTACH for cross-file merge). Returns (nil, false) when the store is // not backed by *C1File OR when the underlying *C1File is nil. // // Avoid using this outside pkg/synccompactor. If you find yourself reaching -// for it, prefer adding a named method to C1ZStore that expresses what you +// for it, prefer adding a named method to c1zstore.Store that expresses what you // need; sqlite-specific leak-through is a smell. See RFC 0002 §4.4. -func AsSQLiteStore(s C1ZStore) (*C1File, bool) { +func AsSQLiteStore(s c1zstore.Store) (*C1File, bool) { cf, ok := s.(*C1File) if !ok || cf == nil { return nil, false diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go deleted file mode 100644 index bea6f88a..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go +++ /dev/null @@ -1,14 +0,0 @@ -package dotc1z - -import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" - -// The sync-metadata contract lives in pkg/dotc1z/c1zstore so storage -// engines can implement it without importing this package. These aliases -// preserve the historical dotc1z names. - -// SyncMeta is the sync-run-metadata sub-store of C1ZStore. See -// c1zstore.SyncMeta for the full contract. -type SyncMeta = c1zstore.SyncMeta - -// SyncRun is the exported shape of a sync run. See c1zstore.SyncRun. -type SyncRun = c1zstore.SyncRun diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go index afdbbac1..9171ac44 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go @@ -154,7 +154,7 @@ func (r *syncRunsTable) Migrations(ctx context.Context, db *goqu.Database) (bool // getCachedViewSyncRun returns the cached sync run for read operations. // This avoids N+1 queries when paginating through listConnectorObjects. // The cache is invalidated when a sync starts or ends. -func (c *C1File) getCachedViewSyncRun(ctx context.Context) (*SyncRun, error) { +func (c *C1File) getCachedViewSyncRun(ctx context.Context) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getCachedViewSyncRun") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -188,7 +188,7 @@ func (c *C1File) invalidateCachedViewSyncRun() { c.cachedViewSyncErr = nil } -func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connectorstore.SyncType) (*SyncRun, error) { +func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connectorstore.SyncType) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getLatestUnfinishedSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -200,7 +200,7 @@ func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connector // Don't resume syncs that started over a week ago oneWeekAgo := time.Now().AddDate(0, 0, -7) - ret := &SyncRun{} + ret := &c1zstore.SyncRun{} q := c.db.From(syncRuns.Name()) q = q.Select("sync_id", "started_at", "ended_at", "sync_token", "sync_type", "parent_sync_id", "linked_sync_id", "supports_diff", "stats") q = q.Where(goqu.C("ended_at").IsNull()) @@ -231,7 +231,7 @@ func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connector return ret, nil } -func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType connectorstore.SyncType) (*SyncRun, error) { +func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType connectorstore.SyncType) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getFinishedSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -246,7 +246,7 @@ func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType conn return nil, status.Errorf(codes.InvalidArgument, "invalid sync type: %s", syncType) } - ret := &SyncRun{} + ret := &c1zstore.SyncRun{} q := c.db.From(syncRuns.Name()) q = q.Select("sync_id", "started_at", "ended_at", "sync_token", "sync_type", "parent_sync_id", "linked_sync_id", "supports_diff", "stats") q = q.Where(goqu.C("ended_at").IsNotNull()) @@ -301,7 +301,7 @@ func parseStats(ctx context.Context, statsBytes *[]byte) *reader_v2.SyncStats { return ret } -func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*SyncRun, string, error) { +func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*c1zstore.SyncRun, string, error) { ctx, span := tracer.Start(ctx, "C1File.ListSyncRuns") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -325,7 +325,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize ui q = q.Order(goqu.C("id").Asc()) q = q.Limit(uint(pageSize + 1)) - var ret []*SyncRun + var ret []*c1zstore.SyncRun query, args, err := q.ToSQL() if err != nil { @@ -347,7 +347,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize ui } statsBytes := &[]byte{} rowId := 0 - data := &SyncRun{} + data := &c1zstore.SyncRun{} err := rows.Scan(&rowId, &data.ID, &data.StartedAt, &data.EndedAt, &data.SyncToken, &data.Type, &data.ParentSyncID, &data.LinkedSyncID, &data.SupportsDiff, &statsBytes) if err != nil { return nil, "", err @@ -432,7 +432,7 @@ func (c *C1File) LatestFinishedSyncID(ctx context.Context, syncType connectorsto return s.ID, nil } -func (c *C1File) getSync(ctx context.Context, syncID string) (*SyncRun, error) { +func (c *C1File) getSync(ctx context.Context, syncID string) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -442,7 +442,7 @@ func (c *C1File) getSync(ctx context.Context, syncID string) (*SyncRun, error) { return nil, err } - ret := &SyncRun{} + ret := &c1zstore.SyncRun{} q := c.db.From(syncRuns.Name()) q = q.Select("sync_id", "started_at", "ended_at", "sync_token", "sync_type", "parent_sync_id", "linked_sync_id", "supports_diff", "stats") @@ -464,7 +464,7 @@ func (c *C1File) getSync(ctx context.Context, syncID string) (*SyncRun, error) { return ret, nil } -func (c *C1File) getCurrentSync(ctx context.Context) (*SyncRun, error) { +func (c *C1File) getCurrentSync(ctx context.Context) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getCurrentSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -885,7 +885,7 @@ func (c *C1File) Cleanup(ctx context.Context) error { return err } - var candidates []SyncRun + var candidates []c1zstore.SyncRun pageToken := "" for { runs, nextPageToken, err := c.ListSyncRuns(ctx, pageToken, 100) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go index d413bcc5..0b251e3e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go @@ -22,6 +22,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -140,7 +141,7 @@ func (c *C1File) ToPebble(ctx context.Context, outPath string, syncID string, op return nil, err } - if _, statErr := os.Stat(outPath); statErr == nil { + if _, statErr := os.Stat(outPath); statErr == nil { // #nosec G703 -- conversion output path is caller-controlled by API design. return nil, fmt.Errorf("to-pebble: output path (%s) must not exist", outPath) } else if !errors.Is(statErr, fs.ErrNotExist) { return nil, fmt.Errorf("to-pebble: stat output path %s: %w", outPath, statErr) @@ -168,7 +169,7 @@ func (c *C1File) ToPebble(ctx context.Context, outPath string, syncID string, op start := time.Now() l := ctxzap.Extract(ctx) - dest, err := NewStore(ctx, outPath, WithEngine(EnginePebble), WithTmpDir(cfg.tmpDir)) + dest, err := NewStore(ctx, outPath, WithEngine(c1zstore.EnginePebble), WithTmpDir(cfg.tmpDir)) if err != nil { return nil, fmt.Errorf("to-pebble: open destination: %w", err) } @@ -179,7 +180,7 @@ func (c *C1File) ToPebble(ctx context.Context, outPath string, syncID string, op defer func() { if cleanupDest { _ = dest.Close(ctx) - _ = os.Remove(outPath) + _ = os.Remove(outPath) // #nosec G703 -- cleanup of caller-selected conversion output. } }() @@ -259,7 +260,7 @@ func (c *C1File) ToPebble(ctx context.Context, outPath string, syncID string, op closeStart := time.Now() if err = dest.Close(ctx); err != nil { cleanupDest = false - _ = os.Remove(outPath) + _ = os.Remove(outPath) // #nosec G703 -- cleanup of caller-selected conversion output. return nil, fmt.Errorf("to-pebble: close destination: %w", err) } closeDur := time.Since(closeStart) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go index e4572664..40c4e551 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go @@ -264,6 +264,17 @@ var ( WithDescription("The path to the c1z file to sync external baton resources with"), WithPersistent(true), WithExportTarget(ExportTargetNone)) + // PreviousSyncC1ZField is hidden by default: source-cache replay is + // author-opt-in functionality (connectorrunner.WithKeepPreviousSyncC1Z), + // and non-replaying connectors shouldn't advertise it to operators. + // DefineConfiguration unhides it for connectors that declare the + // capability; the flag still parses everywhere (hidden ≠ disabled) so + // expert/debug use keeps working. + PreviousSyncC1ZField = StringField("previous-sync-c1z", + WithDescription("The path to the previous sync c1z file to use as a source-cache replay input"), + WithPersistent(true), + WithHidden(true), + WithExportTarget(ExportTargetNone)) externalResourceEntitlementIdFilter = StringField("external-resource-entitlement-id-filter", WithDescription("The entitlement that external users, groups must have access to sync external baton resources"), WithPersistent(true), @@ -276,11 +287,14 @@ var ( // connectors whose author also declared ETag-replay support at build // time (connectorrunner.WithKeepPreviousSyncC1Z) — both are // required. Costs one c1z of local disk. + // Hidden by default for the same reason as PreviousSyncC1ZField; unhidden + // when the connector author declares the replay capability. KeepPreviousSyncC1ZField = BoolField("keep-previous-sync-c1z", WithDescription("Keep the previously synced c1z on disk to enable ETag replay across service-mode syncs "+ "(requires a connector that supports ETag replay; costs one c1z of local disk)"), WithDefaultValue(false), WithPersistent(true), + WithHidden(true), WithExportTarget(ExportTargetNone)) LambdaServerClientIDField = StringField("lambda-client-id", WithRequired(true), WithDescription("The oauth client id to use with the configuration endpoint"), @@ -431,6 +445,7 @@ var DefaultFields = append([]SchemaField{ skipEntitlementsAndGrants, skipGrants, externalResourceC1ZField, + PreviousSyncC1ZField, externalResourceEntitlementIdFilter, KeepPreviousSyncC1ZField, diffSyncsField, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go index 1107861c..738d1ca6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go @@ -126,6 +126,34 @@ func (s SchemaField) ExportAs(et ExportTarget) SchemaField { return c } +// WithConnectorDefault returns a copy of a shared/default SDK field carrying +// a connector-specific default value. Include the copy in the connector's +// field.Configuration Fields — DefineConfiguration replaces the SDK's copy +// with it (same re-export mechanism as ExportAs), so --help, flag parsing, +// and exported config schemas all reflect the connector's default. +// +// Value-resolution precedence is unchanged and sentinel-free: an explicit +// flag beats the environment beats the config file beats this default (the +// default lives on the flag itself, so a user-supplied zero value remains +// distinguishable from "unset"). +// +// Example — a connector whose sync fans out well declares its own worker +// default without hiding the shared flag's semantics: +// +// field.NewConfiguration([]field.SchemaField{ +// field.WithConnectorDefault(field.WorkerCountField, 16), +// ... +// }) +// +// The value's type must match the field's variant (int for IntField, etc.); +// a mismatch fails loudly at startup when the flag is registered. +func WithConnectorDefault[T SchemaTypes](s SchemaField, defaultValue T) SchemaField { + c := s + c.DefaultValue = defaultValue + c.WasReExported = true + return c +} + // Go doesn't allow generic methods on a non-generic struct. func ValidateField[T SchemaTypes](s *SchemaField, value T) (bool, error) { return s.validate(value) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go index 81206188..df25813c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go @@ -11,6 +11,8 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -24,10 +26,18 @@ type lambdaTransport struct { } func (l *lambdaTransport) RoundTrip(ctx context.Context, req *Request) (*Response, error) { - payload, err := req.MarshalJSON() + payload, frameOnly, err := req.marshalPayload() if err != nil { return nil, fmt.Errorf("lambda_transport: failed to marshal frame: %w", err) } + if frameOnly != nil { + ctxzap.Extract(ctx).Warn( + "lambda_transport: request has no legacy encoding, sending v2 frame only; a connector on a pre-frame SDK cannot process this call", + zap.String("method", req.Method()), + zap.String("function_name", l.functionName), + zap.NamedError("legacy_encoding_error", frameOnly), + ) + } input := &lambda.InvokeInput{ LogType: types.LogTypeTail, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go index a8746242..2a3da875 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go @@ -236,7 +236,18 @@ func TimeoutForRequest(req *Request) (time.Duration, bool, error) { return 0, false, nil } +// Handler serves one transport request. The response echoes the request's +// wire version so v2 invokers get lossless frames and legacy invokers get +// protojson (see Response.MarshalJSON). func (s *Server) Handler(ctx context.Context, req *Request) (*Response, error) { + resp, err := s.handle(ctx, req) + if resp != nil { + resp.wireV2 = req.wireV2 + } + return resp, err +} + +func (s *Server) handle(ctx context.Context, req *Request) (*Response, error) { serviceName, methodName, err := parseMethod(req.Method()) if err != nil { return ErrorResponse(err), nil diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go index 28d1b9c0..7da6f05d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go @@ -20,10 +20,12 @@ import ( const annotationsFieldName = "annotations" /* -unmarshalTransportJSON unmarshals transport JSON into msg, discarding any -unknown fields. +unmarshalTransportJSON unmarshals transport JSON into msg. It reports whether +the payload carried a v2 wire frame (binary proto, see wireFrame), which is +decoded losslessly with no type resolution. -When the payload fails to unmarshal, it retries after filtering out any +Legacy payloads are protojson, unmarshaled discarding any unknown fields. +When a legacy payload fails to unmarshal, it retries after filtering out any annotations whose types are not known to the global registry. Annotation type skew happens frequently for new features and would otherwise require rolling every lambda function (and, in the response direction, would let an old @@ -39,7 +41,11 @@ payloads that already failed, where the alternative is a hard error. Our payloads are small relative to the work of the connector, so the performance impact is negligible. */ -func unmarshalTransportJSON(b []byte, msg proto.Message) error { +func unmarshalTransportJSON(b []byte, msg proto.Message) (bool, error) { + if ok, err := decodeWireFrame(b, msg); ok { + return true, err + } + unmarshalOptions := protojson.UnmarshalOptions{ DiscardUnknown: true, } @@ -47,19 +53,19 @@ func unmarshalTransportJSON(b []byte, msg proto.Message) error { // so any failure falls through to the annotation filter. originalErr := unmarshalOptions.Unmarshal(b, msg) if originalErr == nil { - return nil + return false, nil } filtered, changed := filterUnknownAnnotations(b) if !changed { - return originalErr + return false, originalErr } if err := unmarshalOptions.Unmarshal(filtered, msg); err != nil { - return errors.Join(originalErr, err) + return false, errors.Join(originalErr, err) } - return nil + return false, nil } // filterUnknownAnnotations recursively walks raw JSON and prunes entries from @@ -181,18 +187,61 @@ func filterAnnotationsArray(raw json.RawMessage) (json.RawMessage, bool) { type Request struct { msg *pbtransport.Request + + // wireV2 records that the request arrived as a v2 wire frame, proving + // the invoker can read one back. The server stamps it onto the Response. + wireV2 bool } -// UnmarshalJSON unmarshals the JSON into a Request, discarding unknown fields -// and filtering annotations with unresolvable types. See -// unmarshalTransportJSON. +// UnmarshalJSON unmarshals the JSON into a Request. v2 wire frames decode +// losslessly; legacy payloads are protojson, discarding unknown fields and +// filtering annotations with unresolvable types. See unmarshalTransportJSON. func (f *Request) UnmarshalJSON(b []byte) error { f.msg = &pbtransport.Request{} - return unmarshalTransportJSON(b, f.msg) + wireV2, err := unmarshalTransportJSON(b, f.msg) + if err != nil { + return err + } + f.wireV2 = wireV2 + return nil } +// MarshalJSON dual-encodes the request: the legacy protojson fields and the +// v2 wire frame share one JSON object, so legacy connectors keep working +// (they discard the unknown frame fields) while v2 connectors decode the +// frame and see annotations whose types this process cannot resolve. When no +// legacy view can be produced — protojson cannot represent an Any whose type +// is not linked into this process — the frame is sent alone: a legacy +// connector would have failed on that payload anyway. Oversized dual +// payloads fall back to legacy-only to stay under the Lambda invoke limit; +// v2 connectors accept those too. func (f *Request) MarshalJSON() ([]byte, error) { - return protojson.Marshal(f.msg) + payload, _, err := f.marshalPayload() + return payload, err +} + +// marshalPayload builds the invoke payload. The middle return reports the +// frame-only condition: when non-nil, the payload carries only the v2 frame +// and the value is the reason the legacy view could not be produced — +// callers with a context should surface it, since a legacy connector cannot +// process a frame-only payload. +func (f *Request) marshalPayload() ([]byte, error, error) { + legacy, legacyErr := protojson.Marshal(f.msg) + if legacyErr != nil { + payload, err := encodeWireFrame(f.msg) + if err != nil { + return nil, nil, errors.Join(legacyErr, err) + } + return payload, legacyErr, nil + } + dual, err := spliceWireFrame(legacy, f.msg) + if err != nil { + return nil, nil, err + } + if len(dual) > maxDualEncodedPayload { + return legacy, nil, nil + } + return dual, nil, nil } func (f *Request) Method() string { @@ -236,20 +285,40 @@ func NewRequest(method string, req proto.Message, headers metadata.MD) (*Request type Response struct { msg *pbtransport.Response + + // wireV2 selects the v2 wire frame encoding. The server sets it from + // the request: a frame in the request proves the invoker reads frames. + wireV2 bool } -// UnmarshalJSON unmarshals the JSON into a Response, discarding unknown -// fields and filtering annotations with unresolvable types. Responses carry -// annotations at the response level and nested inside rows (grants embed -// resources, etc.), so this protects an invoker from annotation types it -// does not know about — for example an older invoker receiving annotations -// from a connector built with a newer SDK. See unmarshalTransportJSON. +// UnmarshalJSON unmarshals the JSON into a Response. v2 wire frames decode +// losslessly with no type resolution. Legacy payloads are protojson, +// discarding unknown fields and filtering annotations with unresolvable +// types: responses carry annotations at the response level and nested inside +// rows (grants embed resources, etc.), so this protects an invoker from +// annotation types it does not know about — for example an older invoker +// receiving annotations from a connector built with a newer SDK. See +// unmarshalTransportJSON. func (f *Response) UnmarshalJSON(b []byte) error { f.msg = &pbtransport.Response{} - return unmarshalTransportJSON(b, f.msg) + wireV2, err := unmarshalTransportJSON(b, f.msg) + if err != nil { + return err + } + f.wireV2 = wireV2 + return nil } +// MarshalJSON encodes a v2 wire frame when the invoker proved it reads them +// (see wireV2), preserving annotations whose types this process cannot +// resolve. Legacy invokers get plain protojson, which fails on an Any whose +// type is not linked into this process — deliberately: the sender's registry +// is no authority on what the receiver understands or needs, so degrading +// the payload by silently dropping data is worse than failing loudly. func (f *Response) MarshalJSON() ([]byte, error) { + if f.wireV2 { + return encodeWireFrame(f.msg) + } return protojson.Marshal(f.msg) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go new file mode 100644 index 00000000..8be74902 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go @@ -0,0 +1,88 @@ +package grpc + +import ( + "bytes" + "encoding/json" + "fmt" + + "google.golang.org/protobuf/proto" +) + +const transportWireVersion = 2 + +// maxDualEncodedPayload caps dual-encoded requests below the 6MiB Lambda +// invoke payload limit. Past it the frame is dropped and the request goes +// out legacy-only, which v2 peers also accept. +var maxDualEncodedPayload = 5 << 20 + +/* +wireFrame is the v2 transport encoding: the binary proto bytes of a +transport Request or Response, carried base64-encoded in the JSON Lambda +payload. Binary proto copies google.protobuf.Any payloads verbatim instead +of resolving their type URLs the way protojson must, so annotation types +that are not linked into a process survive the transport intact — the fix +for connector-specific annotations (e.g. baton-jira's CustomField) being +dropped or crashing the marshal in runtimes that don't register them. + +Version skew is handled without negotiation state: + + - Requests are dual-encoded: the legacy protojson fields and the frame + share one JSON object. Legacy peers unmarshal with DiscardUnknown and + never see the frame; v2 peers prefer it. + - Responses carry the frame alone, but only when the request carried + one — a frame in the request proves the invoker can read it. Legacy + requests get legacy responses. + +The field names cannot collide with the legacy encoding: protojson emits +only "method"/"req"/"headers" for Requests and +"resp"/"status"/"headers"/"trailers" for Responses. +*/ +type wireFrame struct { + V int `json:"v"` + Frame []byte `json:"frame"` +} + +// decodeWireFrame reports whether raw carries a v2 wire frame and, if so, +// decodes it into msg. A false return means raw is a legacy payload: either +// it isn't shaped like a frame, or it doesn't parse as JSON at all — the +// legacy path owns reporting that error. +func decodeWireFrame(raw []byte, msg proto.Message) (bool, error) { + var wf wireFrame + if err := json.Unmarshal(raw, &wf); err != nil || len(wf.Frame) == 0 { + return false, nil //nolint:nilerr // not a v2 frame; the legacy path owns error reporting + } + if wf.V != transportWireVersion { + return true, fmt.Errorf("transport: unsupported wire frame version %d", wf.V) + } + return true, proto.Unmarshal(wf.Frame, msg) +} + +func encodeWireFrame(msg proto.Message) ([]byte, error) { + frame, err := proto.Marshal(msg) + if err != nil { + return nil, err + } + return json.Marshal(wireFrame{V: transportWireVersion, Frame: frame}) +} + +// spliceWireFrame appends the v2 frame fields to a legacy protojson object, +// producing the dual-encoded request payload. +func spliceWireFrame(legacy []byte, msg proto.Message) ([]byte, error) { + suffix, err := encodeWireFrame(msg) + if err != nil { + return nil, err + } + legacy = bytes.TrimSpace(legacy) + if len(legacy) < 2 || legacy[0] != '{' || legacy[len(legacy)-1] != '}' { + return nil, fmt.Errorf("transport: legacy payload is not a JSON object") + } + if len(legacy) == 2 { + return suffix, nil + } + var buf bytes.Buffer + buf.Grow(len(legacy) + len(suffix)) + buf.Write(legacy[:len(legacy)-1]) + buf.WriteByte(',') + buf.Write(suffix[1:]) + return buf.Bytes(), nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index bf9dfd5f..ff96f32a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.16.0" +const Version = "v0.18.2" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go new file mode 100644 index 00000000..266dd5c0 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go @@ -0,0 +1,20 @@ +package sourcecache + +import "context" + +type scopeContextKey struct{} + +// WithScope returns a context carrying the source-cache scope hash for +// rows written under it. The syncer wraps a page's store writes in this +// context when the page carried a SourceCacheScope annotation; the Pebble +// write path stamps the record's source_scope_hash from it. +func WithScope(ctx context.Context, scopeHash string) context.Context { + return context.WithValue(ctx, scopeContextKey{}, scopeHash) +} + +// ScopeFromContext returns the scope hash set by WithScope, or "" when +// the context carries none (the common, unstamped case). +func ScopeFromContext(ctx context.Context) string { + s, _ := ctx.Value(scopeContextKey{}).(string) + return s +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go new file mode 100644 index 00000000..78058fe2 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go @@ -0,0 +1,232 @@ +package sourcecache + +// Lookup continuation (ask/answer): the lookup transport for connector +// runtimes that cannot call back to the syncer mid-request (single-shot +// request/response tunnels, e.g. gRPC-over-Lambda). The connector's first +// execution of a page ("phase 1") records the scopes it needs and fails +// with ErrLookupDeferred; the SDK converts that into a +// SourceCacheLookupAsk response; the syncer resolves the queries against +// its local previous-sync store and re-invokes the same request with +// SourceCacheLookupAnswers attached ("phase 2"), where the same connector +// code gets real answers. See docs/tasks/source-cache-lambda-lookup.md. + +import ( + "context" + "errors" + "fmt" + "sync" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" +) + +// ErrLookupDeferred is returned by a deferring Lookup (phase 1 of the +// ask/answer continuation) when the answer is not yet available. The SDK +// intercepts it and answers the RPC with a SourceCacheLookupAsk. +// +// Propagation contract for connectors: wrap with %w if you must, NEVER +// swallow. The SDK matches with errors.Is, so idiomatic wrapping +// (fmt.Errorf("listing members: %w", err)) is fine. A connector that +// logs-and-continues past this error re-asks for scopes it already +// "handled" and turns every warm sync into a hard failure at the bounce +// cap. +var ErrLookupDeferred = errors.New("source cache lookup deferred: answer arrives on re-invoke") + +// Query identifies one scope to resolve against the previous sync. +type Query struct { + RowKind RowKind + ScopeHash string +} + +// Answer resolves one Query. Found=false means the previous sync has no +// entry for the scope: fetch fresh. (Distinct from a query that got no +// Answer at all, which means unresolved: ask again.) +type Answer struct { + Query + Found bool + ETag string +} + +// BatchLookup is optionally implemented by Lookup implementations that can +// resolve many scopes in one round trip. Connectors should not type-assert +// for it directly; call LookupMany, which falls back to per-query lookups. +type BatchLookup interface { + LookupPreviousSourceCacheMany(ctx context.Context, queries []Query) ([]Answer, error) +} + +// LookupMany resolves a batch of queries through lookup, using one round +// trip when the implementation supports it (BatchLookup) and a loop of +// single lookups otherwise. This is the topology-uniform batch API: +// in-process and subprocess lookups loop over local/loopback point-reads; +// a deferring lookup collects the whole batch into ONE ask. +// +// The returned answers are exact and complete for the queried set — one +// Answer per Query, order preserved, explicit Found per entry. (A +// deferring lookup returns ErrLookupDeferred instead of answers; phase 2 +// then answers the same calls. Answers dropped to the transport size +// budget surface as ErrLookupDeferred again on the affected queries, never +// as silent omissions or false not-founds.) +func LookupMany(ctx context.Context, lookup Lookup, queries []Query) ([]Answer, error) { + if bl, ok := lookup.(BatchLookup); ok { + return bl.LookupPreviousSourceCacheMany(ctx, queries) + } + answers := make([]Answer, 0, len(queries)) + for _, q := range queries { + entry, found, err := lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + if err != nil { + return nil, err + } + a := Answer{Query: q, Found: found} + if found { + a.ETag = entry.ETag + } + answers = append(answers, a) + } + return answers, nil +} + +// ContinuationLookup is the per-request Lookup installed for the +// ask/answer continuation. It serves lookups from the answers delivered on +// the request (phase 2) and defers everything else by recording the query +// and returning ErrLookupDeferred (phase 1, or an under-answered phase 2 — +// e.g. answers dropped to the size budget). +// +// It is constructed per RPC by the SDK; connectors only ever see it as +// SyncOpAttrs.SourceCache. +type ContinuationLookup struct { + mu sync.Mutex + answers map[Query]Answer + asked []Query + askedSet map[Query]struct{} +} + +var ( + _ Lookup = (*ContinuationLookup)(nil) + _ BatchLookup = (*ContinuationLookup)(nil) +) + +// NewContinuationLookup builds a ContinuationLookup pre-loaded with the +// request's answers (nil/empty on phase 1). +func NewContinuationLookup(answers []Answer) *ContinuationLookup { + m := make(map[Query]Answer, len(answers)) + for _, a := range answers { + m[a.Query] = a + } + return &ContinuationLookup{ + answers: m, + askedSet: map[Query]struct{}{}, + } +} + +func (c *ContinuationLookup) LookupPreviousSourceCache(_ context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { + q := Query{RowKind: rowKind, ScopeHash: scopeHash} + c.mu.Lock() + defer c.mu.Unlock() + if a, ok := c.answers[q]; ok { + if !a.Found { + return Entry{}, false, nil + } + return Entry{ETag: a.ETag}, true, nil + } + c.recordLocked(q) + return Entry{}, false, fmt.Errorf("lookup %s/%s: %w", rowKind, scopeHash, ErrLookupDeferred) +} + +func (c *ContinuationLookup) LookupPreviousSourceCacheMany(_ context.Context, queries []Query) ([]Answer, error) { + c.mu.Lock() + defer c.mu.Unlock() + answers := make([]Answer, 0, len(queries)) + missing := 0 + for _, q := range queries { + a, ok := c.answers[q] + if !ok { + c.recordLocked(q) + missing++ + continue + } + answers = append(answers, a) + } + if missing > 0 { + // Defer the whole batch: phase 2 re-runs the same call with the + // full answer set (already-answered queries remain answered on the + // re-invoked request). + return nil, fmt.Errorf("batch lookup: %d of %d queries unresolved: %w", missing, len(queries), ErrLookupDeferred) + } + return answers, nil +} + +func (c *ContinuationLookup) recordLocked(q Query) { + if _, seen := c.askedSet[q]; seen { + return + } + c.askedSet[q] = struct{}{} + c.asked = append(c.asked, q) +} + +// Asked returns the queries recorded by deferred lookups, deduplicated, in +// first-ask order. Empty means no lookup deferred and the handler's result +// stands. +func (c *ContinuationLookup) Asked() []Query { + c.mu.Lock() + defer c.mu.Unlock() + out := make([]Query, len(c.asked)) + copy(out, c.asked) + return out +} + +// --- proto conversions (shared by connectorbuilder and the syncer) ------ + +// AskProto builds the SourceCacheLookupAsk annotation for a set of +// deferred queries. +func AskProto(queries []Query) *v2.SourceCacheLookupAsk { + qs := make([]*v2.SourceCacheLookupAsk_Query, 0, len(queries)) + for _, q := range queries { + qs = append(qs, v2.SourceCacheLookupAsk_Query_builder{ + RowKind: string(q.RowKind), + ScopeHash: q.ScopeHash, + }.Build()) + } + return v2.SourceCacheLookupAsk_builder{Queries: qs}.Build() +} + +// QueriesFromProto extracts and validates the queries of an ask. +func QueriesFromProto(ask *v2.SourceCacheLookupAsk) ([]Query, error) { + out := make([]Query, 0, len(ask.GetQueries())) + for _, q := range ask.GetQueries() { + kind := RowKind(q.GetRowKind()) + if err := ValidateRowKind(kind); err != nil { + return nil, err + } + if err := ValidateScopeHash(q.GetScopeHash()); err != nil { + return nil, err + } + out = append(out, Query{RowKind: kind, ScopeHash: q.GetScopeHash()}) + } + return out, nil +} + +// AnswersProto builds the SourceCacheLookupAnswers annotation. +func AnswersProto(answers []Answer) *v2.SourceCacheLookupAnswers { + as := make([]*v2.SourceCacheLookupAnswers_Answer, 0, len(answers)) + for _, a := range answers { + as = append(as, v2.SourceCacheLookupAnswers_Answer_builder{ + RowKind: string(a.RowKind), + ScopeHash: a.ScopeHash, + Found: a.Found, + Etag: a.ETag, + }.Build()) + } + return v2.SourceCacheLookupAnswers_builder{Answers: as}.Build() +} + +// AnswersFromProto extracts the answers delivered on a request. +func AnswersFromProto(msg *v2.SourceCacheLookupAnswers) []Answer { + out := make([]Answer, 0, len(msg.GetAnswers())) + for _, a := range msg.GetAnswers() { + out = append(out, Answer{ + Query: Query{RowKind: RowKind(a.GetRowKind()), ScopeHash: a.GetScopeHash()}, + Found: a.GetFound(), + ETag: a.GetEtag(), + }) + } + return out +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go new file mode 100644 index 00000000..199ecda9 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go @@ -0,0 +1,54 @@ +package sourcecache + +import ( + "context" + "fmt" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCLookup is the connector-side Lookup implementation that talks to +// BatonSourceCacheService on the parent SDK. +// +// This is deliberately not routed through the session store: session data +// passes through the connector's local MemorySessionCache (otter), which +// would apply generic TTL/eviction policies to sync-scoped validator state. +// The dedicated service keeps the path uncached and the message shape +// explicit. +// +// The parent has exactly one active Lookup registered at a time (set per +// sync via SetSourceCache on the server), so the wire format carries no +// sync_id; routing is implicit. +type GRPCLookup struct { + client v1.BatonSourceCacheServiceClient +} + +// NewGRPCLookup returns a Lookup backed by the given client. A nil client +// yields NoopLookup so callers can configure the client optionally without +// nil checks at every call site. +func NewGRPCLookup(client v1.BatonSourceCacheServiceClient) Lookup { + if client == nil { + return NoopLookup{} + } + return &GRPCLookup{client: client} +} + +func (g *GRPCLookup) LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { + if err := ValidateRowKind(rowKind); err != nil { + return Entry{}, false, err + } + if err := ValidateScopeHash(scopeHash); err != nil { + return Entry{}, false, err + } + resp, err := g.client.Lookup(ctx, v1.LookupRequest_builder{ + RowKind: string(rowKind), + ScopeHash: scopeHash, + }.Build()) + if err != nil { + return Entry{}, false, fmt.Errorf("source cache rpc lookup: %w", err) + } + if !resp.GetFound() { + return Entry{}, false, nil + } + return Entry{ETag: resp.GetEtag()}, true, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go new file mode 100644 index 00000000..950aaf7c --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go @@ -0,0 +1,71 @@ +package sourcecache + +import ( + "context" + "fmt" + "sync/atomic" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCServer is the parent-side BatonSourceCacheService implementation. +// +// The parent SDK holds a single GRPCServer for the lifetime of the connector +// subprocess and swaps the active Lookup via SetSourceCache as syncs come +// and go. The syncer installs a real lookup once it has resolved a usable +// previous sync, and clears it when the sync ends so a late RPC can't serve +// from a store the syncer no longer owns. +// +// Until the first SetSourceCache call the server answers every lookup with +// found=false, which the connector treats as "no previous sync" and falls +// back to an unconditional fetch. +type GRPCServer struct { + v1.UnimplementedBatonSourceCacheServiceServer + lookup atomic.Pointer[Lookup] +} + +var _ v1.BatonSourceCacheServiceServer = (*GRPCServer)(nil) +var _ SetLookup = (*GRPCServer)(nil) + +// NewGRPCServer returns a GRPCServer with no active Lookup registered. +func NewGRPCServer() *GRPCServer { + return &GRPCServer{} +} + +// SetSourceCache replaces the active lookup. Safe to call concurrently with +// in-flight RPCs: existing RPCs continue against the value they read at +// entry; new RPCs see the swapped value. +func (s *GRPCServer) SetSourceCache(ctx context.Context, lookup Lookup) { + if lookup == nil { + s.lookup.Store(nil) + return + } + s.lookup.Store(&lookup) +} + +func (s *GRPCServer) Lookup(ctx context.Context, req *v1.LookupRequest) (*v1.LookupResponse, error) { + rowKind := RowKind(req.GetRowKind()) + if err := ValidateRowKind(rowKind); err != nil { + return nil, err + } + scopeHash := req.GetScopeHash() + if err := ValidateScopeHash(scopeHash); err != nil { + return nil, err + } + + lookupPtr := s.lookup.Load() + if lookupPtr == nil { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + entry, found, err := (*lookupPtr).LookupPreviousSourceCache(ctx, rowKind, scopeHash) + if err != nil { + return nil, fmt.Errorf("source cache lookup: %w", err) + } + if !found { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + return v1.LookupResponse_builder{ + Found: true, + Etag: entry.ETag, + }.Build(), nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go new file mode 100644 index 00000000..1892cda3 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go @@ -0,0 +1,140 @@ +// Package sourcecache defines the connector-facing surface of source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). +// +// A connector that can cheaply revalidate upstream data — HTTP conditional +// requests (GitHub), delta queries (Microsoft Graph) — opts in by attaching +// SourceCacheCapability MODE_READ_WRITE to its Validate response. During a +// sync it looks up the previous validator for a scope via the Lookup the SDK +// provides on SyncOpAttrs, revalidates upstream, and either emits fresh rows +// tagged with SourceCacheScope or asks the SDK to replay the previous rows +// with SourceCacheReplay. +// +// The connector owns scope computation; the SDK only keys storage by the +// connector-supplied scope hash. The validator (etag, delta token) is opaque +// to the SDK. +// +// Invariant that keeps replay safe: a connector must only emit +// SourceCacheReplay for a scope whose validator it received from THIS sync's +// Lookup. The lookup need not happen in the same call that emits the +// replay: a planning call may batch-resolve many scopes and pass the +// verdicts to sibling cursors through SpawnCursors page tokens — that +// satisfies the invariant, because the validator still originates from the +// consuming sync. What's forbidden is a validator that outlives a sync +// (connector-side caches, config, upstream echoes). When source cache is +// disabled or degraded (no capability, no usable previous sync, unsupported +// storage engine) the SDK installs NoopLookup, every lookup misses, and a +// well-behaved connector naturally falls back to full fetch. +// +// Replay equivalence: a cached sync must reproduce what a full resync +// would produce. Replayed rows are verbatim copies of the previous sync's +// rows with one deliberate exception — expander-written Sources on direct +// grants (classified by a self-source entry, mirroring RollbackExpansion) +// are stripped at copy time so the current sync's expansion recomputes +// them from true state; re-expansion only adds contributions, so carrying +// them verbatim would immortalize contributions removed upstream. +// Connector-set Sources (no self-source) are public connector data and +// survive replay byte-for-byte. +package sourcecache + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "time" +) + +// RowKind partitions source-cache scopes by the row type they produce. +// It doubles as the row_kind value stored in SourceCacheEntryRecord. +type RowKind string + +const ( + RowKindResources RowKind = "resources" + RowKindEntitlements RowKind = "entitlements" + RowKindGrants RowKind = "grants" +) + +// Valid reports whether k is one of the defined row kinds. +func (k RowKind) Valid() bool { + switch k { + case RowKindResources, RowKindEntitlements, RowKindGrants: + return true + } + return false +} + +// ValidateRowKind returns an error if rowKind is not one of the known +// RowKind* constants. +func ValidateRowKind(rowKind RowKind) error { + if !rowKind.Valid() { + return fmt.Errorf("invalid source cache row kind: %q", rowKind) + } + return nil +} + +// maxScopeHashLen bounds scope identifiers on the wire and in storage +// keys. Deliberately generous: the shape is a connector convention +// (HashScope produces 64 hex chars) and is not enforced beyond +// non-emptiness and this cap while the model is being proven out against +// real providers. +const maxScopeHashLen = 256 + +// ValidateScopeHash returns an error when scopeHash is empty or +// unreasonably long. Connectors conventionally use HashScope, but any +// stable identifier is accepted. +func ValidateScopeHash(scopeHash string) error { + if scopeHash == "" { + return fmt.Errorf("source cache scope hash is required") + } + if len(scopeHash) > maxScopeHashLen { + return fmt.Errorf("source cache scope hash too long: %d bytes (max %d)", len(scopeHash), maxScopeHashLen) + } + return nil +} + +// Entry is a previous sync's persisted validator for one scope. +type Entry struct { + // ETag is the opaque upstream validator: a literal HTTP ETag, a delta + // token, etc. Never interpreted by the SDK. + ETag string + + // DiscoveredAt is when the entry was written. + DiscoveredAt time.Time +} + +// Lookup resolves a scope's previous-sync validator. The SDK provides an +// implementation on SyncOpAttrs; connectors call it before revalidating +// upstream. +type Lookup interface { + // LookupPreviousSourceCache returns the previous sync's entry for + // (rowKind, scopeHash). found=false means no entry: fetch fresh. + // Implementations must treat internal read errors that leave fresh + // fetch available as misses rather than failing the connector call. + LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (entry Entry, found bool, err error) +} + +// NoopLookup is the Lookup installed when source cache is disabled or +// degraded. Every lookup misses. +type NoopLookup struct{} + +var _ Lookup = NoopLookup{} + +func (NoopLookup) LookupPreviousSourceCache(context.Context, RowKind, string) (Entry, bool, error) { + return Entry{}, false, nil +} + +// SetLookup is implemented by connector clients/servers that can receive a +// source-cache lookup implementation from the sync runner. The SDK calls +// SetSourceCache(lookup) at the start of each sync and SetSourceCache(nil) +// when the sync ends so a late RPC can't read stale state. +type SetLookup interface { + SetSourceCache(ctx context.Context, lookup Lookup) +} + +// HashScope returns the lowercase-hex sha256 of a canonical scope string. +// Convenience for connectors; any stable identifier is acceptable as a +// scope hash (only non-emptiness and a length cap are enforced). +func HashScope(canonicalScope string) string { + sum := sha256.Sum256([]byte(canonicalScope)) + return hex.EncodeToString(sum[:]) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/cycle.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/cycle.go index 52e2ba4c..aa4cc557 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/cycle.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/cycle.go @@ -142,7 +142,7 @@ func (g *EntitlementGraph) removeNode(nodeID int) { delete(g.Edges, edgeID) } } - delete(g.SourcesToDestinations, nodeID) + delete(g.DestinationsToSources, nodeID) } // FixCyclesFromComponents merges all provided cyclic components in one pass. @@ -172,8 +172,8 @@ func (g *EntitlementGraph) FixCyclesFromComponents(ctx context.Context, cyclic [ // single, new node. func (g *EntitlementGraph) fixCycle(nodeIDs []int) error { entitlementIDs := mapset.NewThreadUnsafeSet[string]() - outgoingEdgesToResourceTypeIDs := map[int]mapset.Set[string]{} - incomingEdgesToResourceTypeIDs := map[int]mapset.Set[string]{} + outgoing := map[int]*mergedEdgeSpec{} + incoming := map[int]*mergedEdgeSpec{} for _, nodeID := range nodeIDs { if node, ok := g.Nodes[nodeID]; ok { // Gather entitlements. @@ -185,14 +185,7 @@ func (g *EntitlementGraph) fixCycle(nodeIDs []int) error { if sources, ok := g.DestinationsToSources[nodeID]; ok { for sourceNodeID, edgeID := range sources { if edge, ok := g.Edges[edgeID]; ok { - resourceTypeIDs, ok := incomingEdgesToResourceTypeIDs[sourceNodeID] - if !ok { - resourceTypeIDs = mapset.NewThreadUnsafeSet[string]() - } - for _, resourceTypeID := range edge.ResourceTypeIDs { - resourceTypeIDs.Add(resourceTypeID) - } - incomingEdgesToResourceTypeIDs[sourceNodeID] = resourceTypeIDs + mergeEdgeSpec(incoming, sourceNodeID, edge) } } } @@ -201,14 +194,7 @@ func (g *EntitlementGraph) fixCycle(nodeIDs []int) error { if destinations, ok := g.SourcesToDestinations[nodeID]; ok { for destinationNodeID, edgeID := range destinations { if edge, ok := g.Edges[edgeID]; ok { - resourceTypeIDs, ok := outgoingEdgesToResourceTypeIDs[destinationNodeID] - if !ok { - resourceTypeIDs = mapset.NewThreadUnsafeSet[string]() - } - for _, resourceTypeID := range edge.ResourceTypeIDs { - resourceTypeIDs.Add(resourceTypeID) - } - outgoingEdgesToResourceTypeIDs[destinationNodeID] = resourceTypeIDs + mergeEdgeSpec(outgoing, destinationNodeID, edge) } } } @@ -228,15 +214,15 @@ func (g *EntitlementGraph) fixCycle(nodeIDs []int) error { } // Hook up edges - for destinationID, resourceTypeIDs := range outgoingEdgesToResourceTypeIDs { + for destinationID, spec := range outgoing { g.NextEdgeID++ edge := Edge{ EdgeID: g.NextEdgeID, SourceID: node.Id, DestinationID: destinationID, IsExpanded: false, - IsShallow: false, - ResourceTypeIDs: resourceTypeIDs.ToSlice(), + IsShallow: spec.allShallow, + ResourceTypeIDs: spec.resourceTypeIDs(), } g.Edges[edge.EdgeID] = edge if _, ok := g.SourcesToDestinations[node.Id]; !ok { @@ -248,15 +234,15 @@ func (g *EntitlementGraph) fixCycle(nodeIDs []int) error { } g.DestinationsToSources[destinationID][node.Id] = edge.EdgeID } - for sourceID, resourceTypeIDs := range incomingEdgesToResourceTypeIDs { + for sourceID, spec := range incoming { g.NextEdgeID++ edge := Edge{ EdgeID: g.NextEdgeID, SourceID: sourceID, DestinationID: node.Id, IsExpanded: false, - IsShallow: false, - ResourceTypeIDs: resourceTypeIDs.ToSlice(), + IsShallow: spec.allShallow, + ResourceTypeIDs: spec.resourceTypeIDs(), } g.Edges[edge.EdgeID] = edge @@ -279,3 +265,45 @@ func (g *EntitlementGraph) fixCycle(nodeIDs []int) error { return nil } + +// mergedEdgeSpec accumulates the attributes of the parallel edges one +// neighbor node carries into/out of a cycle being collapsed, with the +// widest-grant union semantics: +// +// - resource-type filters: an EMPTY filter means "match every principal +// type", so any unfiltered edge makes the merged edge unfiltered. Only +// when every edge is filtered is the union of their sets a faithful +// merge — set-unioning across an unfiltered edge would NARROW it. +// - shallow: a deep edge admits strictly more than a shallow one, so any +// deep edge makes the merged edge deep; only when every edge is +// shallow does the merged edge stay shallow. +type mergedEdgeSpec struct { + unfiltered bool + rtids mapset.Set[string] + allShallow bool +} + +func mergeEdgeSpec(into map[int]*mergedEdgeSpec, neighborID int, edge Edge) { + spec, ok := into[neighborID] + if !ok { + spec = &mergedEdgeSpec{rtids: mapset.NewThreadUnsafeSet[string](), allShallow: true} + into[neighborID] = spec + } + if len(edge.ResourceTypeIDs) == 0 { + spec.unfiltered = true + } else { + for _, rtid := range edge.ResourceTypeIDs { + spec.rtids.Add(rtid) + } + } + if !edge.IsShallow { + spec.allShallow = false + } +} + +func (s *mergedEdgeSpec) resourceTypeIDs() []string { + if s.unfiltered { + return nil + } + return s.rtids.ToSlice() +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go index 09cf8901..c54e25b4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go @@ -7,6 +7,8 @@ import ( "os" "strconv" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -61,12 +63,12 @@ var ErrMaxDepthExceeded = errors.New("max depth exceeded") // ExpanderStore defines the minimal store interface needed for grant expansion. // Implementations: -// - *dotc1z.C1File (via dotc1z.C1ZStore) for production syncs +// - *dotc1z.C1File (via c1zstore.Store) for production syncs // - mocks for unit tests // // StoreExpandedGrants writes a batch of expanded grants back to storage, // preserving existing expansion metadata columns on the underlying rows. -// See dotc1z.GrantStore.StoreExpandedGrants for the full contract. +// See c1zstore.GrantStore.StoreExpandedGrants for the full contract. type ExpanderStore interface { GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) ListGrantsForEntitlement(ctx context.Context, req *reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest) (*reader_v2.GrantsReaderServiceListGrantsForEntitlementResponse, error) @@ -75,11 +77,49 @@ type ExpanderStore interface { // GrantsForEntitlementPrincipalSorted reports whether ListGrantsForEntitlement // yields grants in non-decreasing principal (resource_type, resource) order // for a given entitlement. The topological evaluators require it. Pebble's - // by_entitlement index satisfies it; SQLite (orders by grant id) and the - // in-memory test doubles do not and report false. + // entitlement-first primary grant key satisfies it; SQLite (orders by grant + // id) and the in-memory test doubles do not and report false. GrantsForEntitlementPrincipalSorted() bool } +// newExpandedGrantStorer is an optional fast path for stores that can persist +// caller-proven-new expanded grants without read-before-write. Pebble implements +// it; SQLite and tests can omit it and fall back to StoreExpandedGrants. +type newExpandedGrantStorer interface { + StoreNewExpandedGrants(ctx context.Context, grants ...*v2.Grant) error +} + +// synthesizedContributionStorer is an optional store fast path for +// caller-proven-new synthesized grants expressed as raw contributions +// (destination + principal refs + sources) rather than materialized +// v2.Grants, skipping both the grant construction and the read-before-write. +// Pebble implements it; other stores fall back to StoreExpandedGrants via +// the caller. +type synthesizedContributionStorer interface { + StoreNewExpandedGrantContributions(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error +} + +// synthesizedContributionLayerStorer is an optional store fast path that +// accumulates one topological layer's synthesized grants into sorted bulk +// writes (SST ingests on Pebble) instead of committing each destination's +// rows out of key order as they are produced. +// +// Begin returning false routes the caller to the +// StoreNewExpandedGrantContributions fallback. Today the Pebble engine +// always returns true when a sync is open (the by_principal index is +// unconditionally rebuilt at EndSync, so a layer session never skips index +// maintenance); the boolean exists so a future store can decline. Rows +// streamed via Add become readable in bulk publishes no later than Finish +// returning — callers must not rely on visibility before Finish, nor on +// invisibility before it (large layers publish interior segments early). +// Abort discards an in-flight session. +type synthesizedContributionLayerStorer interface { + BeginExpandedGrantLayer(ctx context.Context) (bool, error) + AddExpandedGrantLayerContributions(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error + FinishExpandedGrantLayer(ctx context.Context) error + AbortExpandedGrantLayer(ctx context.Context) error +} + // entitlementGrantPrincipalKeyLister is an optional fast path for stores that // can list only descendant principal identities without materializing full // grants. Returned keys must use descendantGrantKey(resourceType, resource). @@ -131,9 +171,9 @@ func (e *Expander) Run(ctx context.Context) error { // This matches the syncer's step-by-step execution model. func (e *Expander) RunSingleStep(ctx context.Context) error { // The topological projection evaluator is the default whenever the store - // yields grants in principal order (Pebble's by_entitlement index). Stores - // that can't guarantee that ordering (SQLite, in-memory test doubles) fall - // through to the source-batched expander below. + // yields grants in principal order (Pebble's entitlement-first primary grant + // key). Stores that can't guarantee that ordering (SQLite, in-memory test + // doubles) fall through to the source-batched expander below. if e.store.GrantsForEntitlementPrincipalSorted() { if e.IsDone(ctx) { return nil diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expansion_plan.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expansion_plan.go index 03235710..3dab4930 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expansion_plan.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expansion_plan.go @@ -120,7 +120,13 @@ func (g *EntitlementGraph) BuildExpansionPlan(ctx context.Context) (*Entitlement } func (g *EntitlementGraph) ensureExpansionPlan(ctx context.Context) (*EntitlementGraphPlan, error) { - if g.ExpansionPlan != nil { + // A plan checkpointed by a different SDK version may encode different + // projection-source selection rules; rebuild rather than consume it. + // Correctness today is independent of the source set (rows are built + // into and routed through the same set), but the version gate keeps + // that a choice instead of an accident if a plan field ever becomes + // semantically load-bearing. + if g.ExpansionPlan != nil && g.ExpansionPlan.Version == expansionPlanVersion { return g.ExpansionPlan, nil } return g.BuildExpansionPlan(ctx) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go index 94cfccf2..23a9f9c9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/conductorone/baton-sdk/pkg/sync/expand/scc" + mapset "github.com/deckarep/golang-set/v2" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" ) @@ -377,10 +378,43 @@ func (g *EntitlementGraph) AddEdge( } if destinations, ok := g.SourcesToDestinations[srcNode.Id]; ok { - if _, ok = destinations[dstNode.Id]; ok { - // TODO: just do nothing? it's probably a mistake if we're adding the same edge twice + if edgeID, ok := destinations[dstNode.Id]; ok { + // A duplicate (src, dst) edge from the datasource. Fold the + // specs deterministically instead of keeping whichever the + // engine-dependent pagination order delivered first — the two + // engines can iterate pending expansions in different orders, + // and a first-wins rule would let them build different graphs + // from identical connector data. Union semantics match + // fixCycle's edge merge: any deep edge makes the pair deep + // (deep admits strictly more), and any unfiltered edge makes + // the pair unfiltered (an empty filter means match-all, so + // set-unioning across it would narrow it). + existing := g.Edges[edgeID] + changed := false + if existing.IsShallow && !isShallow { + existing.IsShallow = false + changed = true + } + switch { + case len(existing.ResourceTypeIDs) == 0: + // Already unfiltered; nothing wider exists. + case len(resourceTypeIDs) == 0: + existing.ResourceTypeIDs = nil + changed = true + default: + merged := mapset.NewThreadUnsafeSet[string](existing.ResourceTypeIDs...) + before := merged.Cardinality() + merged.Append(resourceTypeIDs...) + if merged.Cardinality() != before { + existing.ResourceTypeIDs = merged.ToSlice() + changed = true + } + } + if changed { + g.Edges[edgeID] = existing + } ctxzap.Extract(ctx).Warn( - "duplicate edge from datasource", + "duplicate edge from datasource; merged specs (deep-wins, unfiltered-wins)", zap.String("src_entitlement_id", srcEntitlementID), zap.String("dst_entitlement_id", dstEntitlementID), zap.Bool("shallow", isShallow), @@ -394,13 +428,13 @@ func (g *EntitlementGraph) AddEdge( if sources, ok := g.DestinationsToSources[dstNode.Id]; ok { if _, ok = sources[srcNode.Id]; ok { - // TODO: just do nothing? it's probably a mistake if we're adding the same edge twice + // Unreachable when the maps are consistent (the + // SourcesToDestinations check above already handled the + // duplicate), kept as a guard against map divergence. ctxzap.Extract(ctx).Warn( - "duplicate edge from datasource", + "duplicate edge from datasource (reverse map only)", zap.String("src_entitlement_id", srcEntitlementID), zap.String("dst_entitlement_id", dstEntitlementID), - zap.Bool("shallow", isShallow), - zap.Strings("resource_type_ids", resourceTypeIDs), ) return nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge.go index c33a4825..594df7c2 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge.go @@ -7,7 +7,9 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/annotations" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" "google.golang.org/grpc/codes" @@ -30,41 +32,148 @@ import ( // multi-flush path on small fixtures; production never mutates it. var expansionDirtyFlushChunk = 10000 -// destinationSink persists a batch of dirty grants for the destination currently -// being reduced. Reduce implementations call it repeatedly (via dirtyFlusher) so -// the write buffer stays bounded regardless of how many grants a destination -// produces. -type destinationSink func(ctx context.Context, dirty []*v2.Grant) error +// destinationSink persists dirty grants for the destination currently being +// reduced. It carries both the generic v2-grant path and an optional direct +// synthesized-contribution path for Pebble. +type destinationSink struct { + store func(ctx context.Context, dirty []*v2.Grant, allNew bool) error + storeSynth func(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error +} -// dirtyFlusher buffers dirty grants and flushes them through a destinationSink -// once the buffer reaches its limit. The buffer is reused across flushes; the -// sink consumes each batch synchronously before the buffer is truncated. +// dirtyFlusher buffers synthesized and base-update grants separately so each +// flush is homogeneous for the optional all-new fast path. type dirtyFlusher struct { - sink destinationSink - limit int - buf []*v2.Grant + sink *destinationSink + dest *v2.Entitlement + limit int + synth []*v2.Grant + synthPrincipals []*v3.PrincipalRef + synthSources []batonGrant.Sources + synthSourceArena []batonGrant.Source + // refSlab hands out the PrincipalRef protos synthPrincipals point into. + // Reset after each flush: the storeSynth implementations encode + // everything during the call and retain nothing. + refSlab principalRefSlab + update []*v2.Grant +} + +// principalRefSlab hands out reusable *v3.PrincipalRef values from +// fixed-size chunks. Chunks are value slices allocated once and never +// resliced or copied (generated protos must not be moved), so growth only +// appends chunk headers. reset() recycles every handed-out ref; consumers +// re-set all fields on reuse (principalRefData.fillRef). +type principalRefSlab struct { + chunks [][]v3.PrincipalRef + chunk int + used int +} + +const principalRefSlabChunk = 4096 + +func (s *principalRefSlab) next() *v3.PrincipalRef { + if s.chunk == len(s.chunks) { + s.chunks = append(s.chunks, make([]v3.PrincipalRef, principalRefSlabChunk)) + } + ref := &s.chunks[s.chunk][s.used] + s.used++ + if s.used == principalRefSlabChunk { + s.chunk++ + s.used = 0 + } + return ref +} + +func (s *principalRefSlab) reset() { + s.chunk, s.used = 0, 0 } -func newDirtyFlusher(sink destinationSink) *dirtyFlusher { - return &dirtyFlusher{sink: sink, limit: expansionDirtyFlushChunk} +func newDirtyFlusher(dest *v2.Entitlement, sink *destinationSink) *dirtyFlusher { + return &dirtyFlusher{dest: dest, sink: sink, limit: expansionDirtyFlushChunk} } -func (f *dirtyFlusher) add(ctx context.Context, grant *v2.Grant) error { - f.buf = append(f.buf, grant) - if len(f.buf) >= f.limit { - return f.flush(ctx) +func (f *dirtyFlusher) add(ctx context.Context, grant *v2.Grant, isNew bool) error { + if isNew { + f.synth = append(f.synth, grant) + if len(f.synth) >= f.limit { + return f.flushSynth(ctx) + } + return nil + } + f.update = append(f.update, grant) + if len(f.update) >= f.limit { + return f.flushUpdate(ctx) + } + return nil +} + +func (f *dirtyFlusher) addSynthesizedContribution(ctx context.Context, contrib *topoContribution, sources batonGrant.Sources) error { + if f.sink.storeSynth == nil { + principal, err := contrib.principalResource() + if err != nil { + return err + } + if principal == nil { + return nil + } + grant, err := newExpandedGrantWithSources(f.dest, principal, sources) + if err != nil { + return err + } + return f.add(ctx, grant, true) + } + data, ok := contrib.principalRefForStore() + if !ok { + return nil + } + sourceStart := len(f.synthSourceArena) + f.synthSourceArena = append(f.synthSourceArena, sources...) + ref := f.refSlab.next() + data.fillRef(ref) + f.synthPrincipals = append(f.synthPrincipals, ref) + f.synthSources = append(f.synthSources, f.synthSourceArena[sourceStart:]) + if len(f.synthPrincipals) >= f.limit { + return f.flushSynth(ctx) } return nil } func (f *dirtyFlusher) flush(ctx context.Context) error { - if len(f.buf) == 0 { + if err := f.flushSynth(ctx); err != nil { + return err + } + return f.flushUpdate(ctx) +} + +func (f *dirtyFlusher) flushSynth(ctx context.Context) error { + if len(f.synthPrincipals) > 0 { + if err := f.sink.storeSynth(ctx, f.dest, f.synthPrincipals, f.synthSources); err != nil { + return err + } + f.synthPrincipals = f.synthPrincipals[:0] + f.synthSources = f.synthSources[:0] + f.synthSourceArena = f.synthSourceArena[:0] + // storeSynth encoded everything during the call; the slab's refs are + // free to reuse for the next batch. + f.refSlab.reset() + } + if len(f.synth) == 0 { return nil } - if err := f.sink(ctx, f.buf); err != nil { + if err := f.sink.store(ctx, f.synth, true); err != nil { return err } - f.buf = f.buf[:0] + f.synth = f.synth[:0] + return nil +} + +func (f *dirtyFlusher) flushUpdate(ctx context.Context) error { + if len(f.update) == 0 { + return nil + } + if err := f.sink.store(ctx, f.update, false); err != nil { + return err + } + f.update = f.update[:0] return nil } @@ -76,10 +185,11 @@ type topologicalRun struct { // reduce evaluates one destination entitlement and streams its dirty grants // to sink in bounded chunks (rather than returning the whole set), given the // destination's finalized incoming edges and the resolved entitlement set. - reduce func(ctx context.Context, dest *v2.Entitlement, incoming []topoIncomingEdge, entitlements map[string]*v2.Entitlement, sink destinationSink) error + reduce func(ctx context.Context, dest *v2.Entitlement, incoming []topoIncomingEdge, entitlements map[string]*v2.Entitlement, sink *destinationSink) error // onStored, when set, runs after each batch of dirty grants is persisted // (projection appends matching projection rows so deeper nodes can read them). - onStored func(ctx context.Context, dirty []*v2.Grant) error + onStored func(ctx context.Context, dirty []*v2.Grant) error + onStoredSynth func(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error // checkBudget, when set, is polled before each node and each destination so a // cancelled context aborts promptly. checkBudget func() error @@ -89,15 +199,15 @@ type topologicalRun struct { progress func(nodeIdx, nodeTotal int) } -// prepareTopological builds the expansion plan, computes the node topological -// order, and resolves every graph entitlement once. All three evaluators share -// it so ordering and the (possibly missing) entitlement set are derived -// identically. -func (e *Expander) prepareTopological(ctx context.Context) (map[string]*v2.Entitlement, []int, error) { +// prepareTopological builds the expansion plan, computes the topological +// layer decomposition, and resolves every graph entitlement once. All +// evaluators share it so ordering and the (possibly missing) entitlement set +// are derived identically. +func (e *Expander) prepareTopological(ctx context.Context) (map[string]*v2.Entitlement, [][]int, error) { if _, err := e.graph.ensureExpansionPlan(ctx); err != nil { return nil, nil, err } - order, err := topologicalNodeOrder(e.graph) + layers, err := topologicalNodeLayers(e.graph) if err != nil { return nil, nil, err } @@ -105,7 +215,7 @@ func (e *Expander) prepareTopological(ctx context.Context) (map[string]*v2.Entit if err != nil { return nil, nil, err } - return entitlements, order, nil + return entitlements, layers, nil } func (e *Expander) loadExpansionEntitlements(ctx context.Context) (map[string]*v2.Entitlement, error) { @@ -122,44 +232,154 @@ func (e *Expander) loadExpansionEntitlements(ctx context.Context) (map[string]*v return entitlements, nil } -// driveTopological walks the collapsed DAG in topological order and reduces each -// destination entitlement of every node whose parents are finalized. It is the -// single scheduling loop behind RunTopologicalMergeStreaming and -// RunTopologicalMergeProjection; only the per-destination reduce strategy and -// the optional projection hooks differ between them. +// driveTopological walks the collapsed DAG layer by layer (Kahn levels) and +// reduces each destination entitlement of every node whose parents are +// finalized. It is the single scheduling loop behind +// RunTopologicalMergeStreaming and RunTopologicalMergeProjection; only the +// per-destination reduce strategy and the optional projection hooks differ +// between them. +// +// When the store supports layer sessions (Pebble), each layer's synthesized +// grants are streamed into one session and published as sorted bulk writes at +// segment/layer boundaries. That is safe because every parent of a layer-k +// node sits in a layer < k, so no reduce in the current layer reads rows the +// session is still holding. func (e *Expander) driveTopological( ctx context.Context, entitlements map[string]*v2.Entitlement, - order []int, + layers [][]int, run topologicalRun, ) error { - logFanInWidth(ctx, e.graph, order) + logFanInWidth(ctx, e.graph, layers) + + totalNodes := 0 + for _, layer := range layers { + totalNodes += len(layer) + } - sink := func(ctx context.Context, dirty []*v2.Grant) error { - if len(dirty) == 0 { + // activeLayer is non-nil only while a layer session is open; the + // storeSynth closure below routes synthesized rows into it. + var activeLayer synthesizedContributionLayerStorer + sink := &destinationSink{ + store: func(ctx context.Context, dirty []*v2.Grant, allNew bool) error { + if len(dirty) == 0 { + return nil + } + if allNew { + if fast, ok := e.store.(newExpandedGrantStorer); ok { + if err := fast.StoreNewExpandedGrants(ctx, dirty...); err != nil { + return fmt.Errorf("topological merge: store new expanded grants: %w", err) + } + } else if err := e.store.StoreExpandedGrants(ctx, dirty...); err != nil { + return fmt.Errorf("topological merge: store expanded grants: %w", err) + } + } else { + if err := e.store.StoreExpandedGrants(ctx, dirty...); err != nil { + return fmt.Errorf("topological merge: store expanded grants: %w", err) + } + } + if run.onStored != nil { + if err := run.onStored(ctx, dirty); err != nil { + return err + } + } + if run.metrics != nil { + run.metrics.DirtyGrantsWritten += int64(len(dirty)) + } + return nil + }, + } + if fast, ok := e.store.(synthesizedContributionStorer); ok { + sink.storeSynth = func(ctx context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error { + if len(principals) == 0 { + return nil + } + if activeLayer != nil { + if err := activeLayer.AddExpandedGrantLayerContributions(ctx, dest, principals, sources); err != nil { + return fmt.Errorf("topological merge: add synthesized layer contributions: %w", err) + } + } else { + if err := fast.StoreNewExpandedGrantContributions(ctx, dest, principals, sources); err != nil { + return fmt.Errorf("topological merge: store new expanded grant contributions: %w", err) + } + } + if run.onStoredSynth != nil { + if err := run.onStoredSynth(ctx, dest, principals, sources); err != nil { + return err + } + } + if run.metrics != nil { + run.metrics.DirtyGrantsWritten += int64(len(principals)) + } return nil } - if err := e.store.StoreExpandedGrants(ctx, dirty...); err != nil { - return fmt.Errorf("topological merge: store expanded grants: %w", err) + } + + var layerCandidate synthesizedContributionLayerStorer + if sink.storeSynth != nil { + layerCandidate, _ = e.store.(synthesizedContributionLayerStorer) + } + + nodeIdx := 0 + for _, layer := range layers { + if layerCandidate != nil { + ok, err := layerCandidate.BeginExpandedGrantLayer(ctx) + if err != nil { + return fmt.Errorf("topological merge: begin synthesized layer: %w", err) + } + if ok { + activeLayer = layerCandidate + } } - if run.onStored != nil { - if err := run.onStored(ctx, dirty); err != nil { - return err + if err := e.driveTopologicalLayer(ctx, entitlements, layer, run, sink, nodeIdx, totalNodes); err != nil { + if activeLayer != nil { + _ = activeLayer.AbortExpandedGrantLayer(ctx) + activeLayer = nil } + return err } - if run.metrics != nil { - run.metrics.DirtyGrantsWritten += int64(len(dirty)) + nodeIdx += len(layer) + if activeLayer != nil { + if err := activeLayer.FinishExpandedGrantLayer(ctx); err != nil { + // Symmetric with the mid-layer error path above: if the + // finish failed before the engine detached the session + // (e.g. an error thrown by a wrapper before reaching the + // engine, or an engine failure that left the session + // attached), a same-process retry's BeginExpandedGrantLayer + // would hit "session already open". Abort is a no-op when + // the session is already gone. + _ = activeLayer.AbortExpandedGrantLayer(ctx) + activeLayer = nil + return fmt.Errorf("topological merge: finish synthesized layer: %w", err) + } + activeLayer = nil } - return nil } - for nodeIdx, nodeID := range order { + return nil +} + +// driveTopologicalLayer reduces every node of one topological layer against +// the shared sink. startIdx is the number of nodes completed in earlier +// layers, used only for progress reporting. Split out of driveTopological so +// a layer's error unwinds through one place where the caller can abort the +// layer's open session. +func (e *Expander) driveTopologicalLayer( + ctx context.Context, + entitlements map[string]*v2.Entitlement, + layer []int, + run topologicalRun, + sink *destinationSink, + startIdx int, + totalNodes int, +) error { + for i, nodeID := range layer { if run.checkBudget != nil { if err := run.checkBudget(); err != nil { return err } } if run.progress != nil { - run.progress(nodeIdx, len(order)) + run.progress(startIdx+i, totalNodes) } node, ok := e.graph.Nodes[nodeID] if !ok { @@ -178,10 +398,15 @@ func (e *Expander) driveTopological( } destEntitlement := entitlements[destID] if destEntitlement == nil { + // Same drop-don't-fail policy as the legacy expander's + // missing-descendant handling, but say so: silently zeroing + // a subtree of expansion is undiagnosable in production. + ctxzap.Extract(ctx).Warn("topological expansion: destination entitlement not in store; skipping its reduction", + zap.String("entitlement_id", destID)) continue } if err := run.reduce(ctx, destEntitlement, incoming, entitlements, sink); err != nil { - return err + return fmt.Errorf("reduce destination entitlement %q: %w", destID, err) } reducedAny = true if run.metrics != nil { @@ -224,28 +449,30 @@ func sortedCopy(in []string) []string { // evaluator re-reads each unprojected source once per destination it feeds, so a // high-fan-out "broadcast" source is a read-amplification hotspot. This answers // whether projection-source selection should cover fan-out, not just fan-in. -func logFanInWidth(ctx context.Context, g *EntitlementGraph, order []int) { - widths := make([]int, 0, len(order)) - outDegrees := make([]int, 0, len(order)) +func logFanInWidth(ctx context.Context, g *EntitlementGraph, layers [][]int) { + widths := make([]int, 0, len(g.Nodes)) + outDegrees := make([]int, 0, len(g.Nodes)) maxParentEntitlements := 0 - for _, nodeID := range order { - incoming := incomingEdgesSorted(g, nodeID) - if len(incoming) > 0 { - width := 1 // base(D) stream - for _, in := range incoming { - parent, ok := g.Nodes[in.sourceNodeID] - if !ok { - continue - } - width += len(parent.EntitlementIDs) - if len(parent.EntitlementIDs) > maxParentEntitlements { - maxParentEntitlements = len(parent.EntitlementIDs) + for _, layer := range layers { + for _, nodeID := range layer { + incoming := incomingEdgesSorted(g, nodeID) + if len(incoming) > 0 { + width := 1 // base(D) stream + for _, in := range incoming { + parent, ok := g.Nodes[in.sourceNodeID] + if !ok { + continue + } + width += len(parent.EntitlementIDs) + if len(parent.EntitlementIDs) > maxParentEntitlements { + maxParentEntitlements = len(parent.EntitlementIDs) + } } + widths = append(widths, width) + } + if outDegree := len(g.SourcesToDestinations[nodeID]); outDegree > 0 { + outDegrees = append(outDegrees, outDegree) } - widths = append(widths, width) - } - if outDegree := len(g.SourcesToDestinations[nodeID]); outDegree > 0 { - outDegrees = append(outDegrees, outDegree) } } @@ -350,11 +577,15 @@ func incomingEdgesSorted(g *EntitlementGraph, nodeID int) []topoIncomingEdge { return out } -func topologicalNodeOrder(g *EntitlementGraph) ([]int, error) { - ids := make([]int, 0, len(g.Nodes)) +// topologicalNodeLayers returns the graph's Kahn level decomposition: layer k +// holds every node whose parents all sit in layers < k, so nodes within one +// layer never depend on each other. The layer boundary is where synthesized +// grant writes can be published (and, later, checkpointed) — a node's reduce +// only ever reads parent output from strictly earlier layers. Each layer is +// sorted by node id for deterministic iteration. +func topologicalNodeLayers(g *EntitlementGraph) ([][]int, error) { inDegree := make(map[int]int, len(g.Nodes)) for id := range g.Nodes { - ids = append(ids, id) inDegree[id] = 0 } for _, edge := range g.Edges { @@ -367,35 +598,49 @@ func topologicalNodeOrder(g *EntitlementGraph) ([]int, error) { inDegree[edge.DestinationID]++ } - sort.Ints(ids) - queue := make([]int, 0, len(ids)) - for _, id := range ids { - if inDegree[id] == 0 { - queue = append(queue, id) + frontier := make([]int, 0, len(g.Nodes)) + for id, deg := range inDegree { + if deg == 0 { + frontier = append(frontier, id) } } - - order := make([]int, 0, len(ids)) - for len(queue) > 0 { - id := queue[0] - queue = queue[1:] - order = append(order, id) - - children := make([]int, 0, len(g.SourcesToDestinations[id])) - for child := range g.SourcesToDestinations[id] { - children = append(children, child) - } - sort.Ints(children) - for _, child := range children { - inDegree[child]-- - if inDegree[child] == 0 { - queue = append(queue, child) + sort.Ints(frontier) + + var layers [][]int + seen := 0 + for len(frontier) > 0 { + layers = append(layers, frontier) + seen += len(frontier) + var next []int + for _, id := range frontier { + for child := range g.SourcesToDestinations[id] { + inDegree[child]-- + if inDegree[child] == 0 { + next = append(next, child) + } } } + sort.Ints(next) + frontier = next } - if len(order) != len(ids) { + if seen != len(g.Nodes) { return nil, fmt.Errorf("topological merge: graph contains a cycle or dangling edge") } + return layers, nil +} + +// topologicalNodeOrder flattens topologicalNodeLayers into a single valid +// topological order. Kept for callers that only need a linear order (the +// expansion plan builder and benchmarks). +func topologicalNodeOrder(g *EntitlementGraph) ([]int, error) { + layers, err := topologicalNodeLayers(g) + if err != nil { + return nil, err + } + order := make([]int, 0, len(g.Nodes)) + for _, layer := range layers { + order = append(order, layer...) + } return order, nil } @@ -421,16 +666,20 @@ func principalKeyLess(a, b topoPrincipalKey) bool { return a.resource < b.resource } +// topoContribution accumulates the source entitlements contributing to one +// principal on one destination. sources is a small slice, not a map: fan-in is +// tiny in practice (p50 ≈ 4, p99 ≈ 14 source entitlements per principal), so a +// linear-scan slice avoids the per-group map allocation that dominated the +// expansion allocation profile. addSource keeps entries unique. type topoContribution struct { - sources map[string]bool - principal *v2.Resource - principalBytes []byte + sources batonGrant.Sources + principal topoPrincipal } func (c *topoContribution) add(sourceEntitlementID string, isDirect bool, principal *v2.Resource) { c.addSource(sourceEntitlementID, isDirect) - if c.principal == nil && principal != nil { - c.principal = proto.Clone(principal).(*v2.Resource) + if c.principal.empty() && principal != nil { + c.principal.setResource(principal) } } @@ -438,52 +687,188 @@ func (c *topoContribution) add(sourceEntitlementID string, isDirect bool, princi // upgrading an existing indirect entry to direct. It never touches the // principal. func (c *topoContribution) addSource(sourceEntitlementID string, isDirect bool) { - if c.sources == nil { - c.sources = make(map[string]bool) - } - if existing, ok := c.sources[sourceEntitlementID]; !ok || (isDirect && !existing) { - c.sources[sourceEntitlementID] = isDirect + for i := range c.sources { + if c.sources[i].EntitlementID == sourceEntitlementID { + if isDirect && !c.sources[i].IsDirect { + c.sources[i].IsDirect = true + } + return + } } + c.sources = append(c.sources, batonGrant.Source{EntitlementID: sourceEntitlementID, IsDirect: isDirect}) } func (c *topoContribution) merge(other *topoContribution) { if other == nil { return } - for sourceID, isDirect := range other.sources { - c.addSource(sourceID, isDirect) - } - // Take ownership of the principal exactly once, from the first contributor - // that carries one. other.principalBytes may alias a stream-owned reusable - // buffer (see projectionContributionStream), which is overwritten on the - // next stream advance. merge runs during consume, before that advance, so - // copy the bytes into a slice this contribution owns rather than aliasing. - if c.principal == nil && c.principalBytes == nil { - if other.principal != nil { - c.principal = other.principal - } else if len(other.principalBytes) > 0 { - c.principalBytes = append([]byte(nil), other.principalBytes...) - } + for _, src := range other.sources { + c.addSource(src.EntitlementID, src.IsDirect) + } + if c.principal.empty() { + c.principal.take(other.principal) } } +// resetForReuse clears the accumulated sources and principal while keeping +// backing storage, so stream-owned contributions can be recycled across +// principal groups without reallocating. +func (c *topoContribution) resetForReuse() { + c.sources = c.sources[:0] + c.principal.reset() +} + +func (c *topoContribution) principalRefForStore() (principalRefData, bool) { + if c == nil { + return principalRefData{}, false + } + return c.principal.refForStore() +} + func (c *topoContribution) principalResource() (*v2.Resource, error) { if c == nil { return nil, nil } - if c.principal != nil { - return c.principal, nil + return c.principal.resource() +} + +// principalRefData is the identity-only principal form as plain strings — +// the value-type equivalent of v3.PrincipalRef. Contributions carry this +// instead of a proto so per-row proto allocation happens exactly once, in +// the flusher's reusable slab, rather than at every projection-row decode +// (54M+ rows per whale expansion made PrincipalRef_builder.Build a top +// allocation site). +type principalRefData struct { + resourceTypeID string + resourceID string + parentResourceTypeID string + parentResourceID string + ok bool +} + +func (d principalRefData) fillRef(ref *v3.PrincipalRef) { + // Every field is assigned unconditionally: refs come from a reused slab + // and a conditionally-set field would leak the previous occupant's value. + ref.SetResourceTypeId(d.resourceTypeID) + ref.SetResourceId(d.resourceID) + ref.SetParentResourceTypeId(d.parentResourceTypeID) + ref.SetParentResourceId(d.parentResourceID) +} + +type topoPrincipal struct { + // full is the full principal payload used by generic fallback stores. + full *v2.Resource + // refData is the identity-only form Pebble can persist without + // unmarshalling the full Resource. resourceBytes is kept only so fallback + // stores preserve rich principal payload when projection rows are the + // source. + refData principalRefData + resourceBytes []byte +} + +func (p *topoPrincipal) empty() bool { + return p.full == nil && !p.refData.ok && len(p.resourceBytes) == 0 +} + +func (p *topoPrincipal) reset() { + p.full = nil + p.refData = principalRefData{} + // Keep capacity: setRef appends into this buffer on the next group. + p.resourceBytes = p.resourceBytes[:0] +} + +func (p *topoPrincipal) setResource(resource *v2.Resource) { + p.full = proto.Clone(resource).(*v2.Resource) + p.refData = principalRefData{} + p.resourceBytes = nil +} + +func (p *topoPrincipal) setRef(data principalRefData, resourceBytes []byte) { + p.full = nil + p.refData = data + p.resourceBytes = append(p.resourceBytes[:0], resourceBytes...) +} + +func (p *topoPrincipal) take(other topoPrincipal) { + if other.full != nil { + p.full = other.full + p.refData = principalRefData{} + p.resourceBytes = nil + return + } + p.full = nil + p.refData = other.refData + p.resourceBytes = append(p.resourceBytes[:0], other.resourceBytes...) +} + +func (p *topoPrincipal) refForStore() (principalRefData, bool) { + if p.refData.ok { + return p.refData, true + } + if p.full == nil || p.full.GetId() == nil { + return principalRefData{}, false } - if len(c.principalBytes) == 0 { + parent := p.full.GetParentResourceId() + return principalRefData{ + resourceTypeID: p.full.GetId().GetResourceType(), + resourceID: p.full.GetId().GetResource(), + parentResourceTypeID: parent.GetResourceType(), + parentResourceID: parent.GetResource(), + ok: true, + }, true +} + +func (p *topoPrincipal) resource() (*v2.Resource, error) { + if p.full != nil { + return p.full, nil + } + if len(p.resourceBytes) > 0 { + resource := &v2.Resource{} + if err := proto.Unmarshal(p.resourceBytes, resource); err != nil { + return nil, err + } + p.full = resource + p.resourceBytes = nil + return resource, nil + } + if !p.refData.ok { return nil, nil } - p := &v2.Resource{} - if err := proto.Unmarshal(c.principalBytes, p); err != nil { - return nil, err + resource := principalResourceFromRefData(p.refData) + p.full = resource + return resource, nil +} + +// principalResourceFromRef adapts a proto ref for consumers that hold one +// (fallback stores handed []*v3.PrincipalRef). +func principalResourceFromRef(ref *v3.PrincipalRef) *v2.Resource { + if ref == nil { + return nil } - c.principal = p - c.principalBytes = nil - return p, nil + return principalResourceFromRefData(principalRefData{ + resourceTypeID: ref.GetResourceTypeId(), + resourceID: ref.GetResourceId(), + parentResourceTypeID: ref.GetParentResourceTypeId(), + parentResourceID: ref.GetParentResourceId(), + ok: true, + }) +} + +func principalResourceFromRefData(data principalRefData) *v2.Resource { + var parent *v2.ResourceId + if data.parentResourceID != "" { + parent = v2.ResourceId_builder{ + ResourceType: data.parentResourceTypeID, + Resource: data.parentResourceID, + }.Build() + } + return v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: data.resourceTypeID, + Resource: data.resourceID, + }.Build(), + ParentResourceId: parent, + }.Build() } func grantContributesOverEdge(grant *v2.Grant, sourceEntitlementID string, edge Edge) bool { @@ -514,7 +899,7 @@ func isGrantDirectOnEntitlement(grant *v2.Grant, entitlementID string) bool { return len(sources) == 0 || sources[entitlementID] != nil } -func mergeContributionIntoExistingGrant(baseGrant *v2.Grant, destEntitlementID string, contrib map[string]bool) *v2.Grant { +func mergeContributionIntoExistingGrant(baseGrant *v2.Grant, destEntitlementID string, contrib batonGrant.Sources) *v2.Grant { if baseGrant == nil || len(contrib) == 0 { return nil } @@ -529,14 +914,14 @@ func mergeContributionIntoExistingGrant(baseGrant *v2.Grant, destEntitlementID s sourcesMap[destEntitlementID] = &v2.GrantSources_GrantSource{IsDirect: true} updated = true } - for sourceID, isDirect := range contrib { - existingSource := sourcesMap[sourceID] + for _, src := range contrib { + existingSource := sourcesMap[src.EntitlementID] if existingSource == nil { - sourcesMap[sourceID] = &v2.GrantSources_GrantSource{IsDirect: isDirect} + sourcesMap[src.EntitlementID] = &v2.GrantSources_GrantSource{IsDirect: src.IsDirect} updated = true continue } - if isDirect && !existingSource.GetIsDirect() { + if src.IsDirect && !existingSource.GetIsDirect() { existingSource.SetIsDirect(true) updated = true } @@ -548,29 +933,26 @@ func mergeContributionIntoExistingGrant(baseGrant *v2.Grant, destEntitlementID s return grant } -func newExpandedGrantWithSources(descEntitlement *v2.Entitlement, principal *v2.Resource, sources map[string]bool) (*v2.Grant, error) { +func newExpandedGrantWithSources(descEntitlement *v2.Entitlement, principal *v2.Resource, sources batonGrant.Sources) (*v2.Grant, error) { if len(sources) == 0 { return nil, fmt.Errorf("newExpandedGrantWithSources: empty sources") } sourceMap := make(map[string]*v2.GrantSources_GrantSource, len(sources)) - sourceIDs := make([]string, 0, len(sources)) - for sourceID := range sources { - sourceIDs = append(sourceIDs, sourceID) - } - sort.Strings(sourceIDs) - for _, sourceID := range sourceIDs { - isDirect := sources[sourceID] - sourceMap[sourceID] = &v2.GrantSources_GrantSource{IsDirect: isDirect} + for _, src := range sources { + sourceMap[src.EntitlementID] = &v2.GrantSources_GrantSource{IsDirect: src.IsDirect} } enResource := descEntitlement.GetResource() if enResource == nil { return nil, fmt.Errorf("newExpandedGrantWithSources: entitlement has no resource") } - if principal == nil { - return nil, fmt.Errorf("newExpandedGrantWithSources: principal is nil") + // Guard the id too: batonGrant.NewGrantID panics on a nil resource id + // (its callers are connector code where that is a programming error); + // here the principal came out of a store and malformed data must be an + // error, not a panic. + if principal == nil || principal.GetId() == nil { + return nil, fmt.Errorf("newExpandedGrantWithSources: principal has no resource id") } - pid := principal.GetId() - grantID := descEntitlement.GetId() + ":" + pid.GetResourceType() + ":" + pid.GetResource() + grantID := batonGrant.NewGrantID(principal, descEntitlement) return v2.Grant_builder{ Id: grantID, Entitlement: descEntitlement, @@ -579,3 +961,9 @@ func newExpandedGrantWithSources(descEntitlement *v2.Entitlement, principal *v2. Annotations: annotations.Annotations{immutableAnnotationAny}, }.Build(), nil } + +// NewExpandedGrantForStore builds the generic v2 expanded grant used by store +// adapters that do not implement the direct synthesized-contribution fast path. +func NewExpandedGrantForStore(descEntitlement *v2.Entitlement, principal *v2.Resource, sources batonGrant.Sources) (*v2.Grant, error) { + return newExpandedGrantWithSources(descEntitlement, principal, sources) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge_projection.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge_projection.go index d2948889..d2081d93 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge_projection.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge_projection.go @@ -16,7 +16,9 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" "google.golang.org/protobuf/proto" @@ -28,8 +30,8 @@ const projectionProgressInterval = 15 * time.Second // RunTopologicalMergeProjection runs topological pull expansion using a // temporary source projection DB for finalized source streams. The projection -// avoids repeatedly materializing parent grants through the permanent -// by_entitlement index while keeping the canonical grant store as the sink. +// avoids repeatedly materializing parent grants through permanent grant scans +// while keeping the canonical grant store as the sink. func (e *Expander) RunTopologicalMergeProjection(ctx context.Context) error { plan, err := e.graph.ensureExpansionPlan(ctx) if err != nil { @@ -51,10 +53,14 @@ func (e *Expander) RunTopologicalMergeProjection(ctx context.Context) error { } defer projDB.Close() - entitlements, order, err := e.prepareTopological(ctx) + entitlements, layers, err := e.prepareTopological(ctx) if err != nil { return err } + totalNodes := 0 + for _, layer := range layers { + totalNodes += len(layer) + } l := ctxzap.Extract(ctx) buildStart := time.Now() @@ -66,14 +72,15 @@ func (e *Expander) RunTopologicalMergeProjection(ctx context.Context) error { l.Info("topological projection: built source projection", zap.Int("projection_rows", rows), zap.Int("projection_sources", len(projectionSources)), - zap.Int("nodes", len(order)), + zap.Int("nodes", totalNodes), + zap.Int("layers", len(layers)), zap.Duration("elapsed", time.Since(buildStart)), ) expandStart := time.Now() lastLog := expandStart - err = e.driveTopological(ctx, entitlements, order, topologicalRun{ - reduce: func(ctx context.Context, dest *v2.Entitlement, incoming []topoIncomingEdge, ents map[string]*v2.Entitlement, sink destinationSink) error { + err = e.driveTopological(ctx, entitlements, layers, topologicalRun{ + reduce: func(ctx context.Context, dest *v2.Entitlement, incoming []topoIncomingEdge, ents map[string]*v2.Entitlement, sink *destinationSink) error { return e.mergeDestinationStreams(ctx, dest, incoming, ents, projDB, projectionSources, sink) }, onStored: func(_ context.Context, dirty []*v2.Grant) error { @@ -84,6 +91,14 @@ func (e *Expander) RunTopologicalMergeProjection(ctx context.Context) error { metrics.ProjectionRowsBuilt += int64(addedRows) return nil }, + onStoredSynth: func(_ context.Context, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources) error { + addedRows, err := addProjectionRowsFromSynthesized(projDB, dest, principals, sources, projectionSources) + if err != nil { + return fmt.Errorf("topological projection: append synthesized projection rows for %q: %w", dest.GetId(), err) + } + metrics.ProjectionRowsBuilt += int64(addedRows) + return nil + }, // The context deadline is the projection's budget: a caller (e.g. a // bounded profiling run) sets a timeout, and a cancelled context aborts // the pass promptly. There is no separate budget knob in the production @@ -110,7 +125,7 @@ func (e *Expander) RunTopologicalMergeProjection(ctx context.Context) error { } l.Info("topological projection: expansion complete", - zap.Int("nodes_total", len(order)), + zap.Int("nodes_total", totalNodes), zap.Int64("dirty_grants_written", metrics.DirtyGrantsWritten), zap.Int64("projection_rows", metrics.ProjectionRowsBuilt), zap.Int64("nodes_reduced", metrics.NodesReduced), @@ -133,7 +148,7 @@ func (e *Expander) mergeDestinationStreams( entitlements map[string]*v2.Entitlement, projDB *pebble.DB, projectionSources map[string]struct{}, - sink destinationSink, + sink *destinationSink, ) error { streams := make([]contributionGroupStream, 0, 1+len(incoming)) streams = append(streams, &baseContributionStream{ @@ -147,6 +162,10 @@ func (e *Expander) mergeDestinationStreams( for _, sourceID := range sortedCopy(sourceNode.EntitlementIDs) { sourceEntitlement := entitlements[sourceID] if sourceEntitlement == nil { + // Drop-don't-fail, but loudly (see the destination-side + // warning in driveTopologicalLayer). + ctxzap.Extract(ctx).Warn("topological expansion: source entitlement not in store; its contributions are skipped", + zap.String("entitlement_id", sourceID)) continue } if projDB != nil { @@ -172,7 +191,7 @@ func (e *Expander) mergeDestinationStreams( // Entitlements are processed in sorted id order, and the tuple-encoded // projection key is prefix-ordered by entitlement, so each entitlement's rows // form a contiguous, globally-ordered block. When the store yields grants in -// principal order (Pebble's by_entitlement index — see +// principal order (Pebble's entitlement-first primary grant key — see // ExpanderStore.GrantsForEntitlementPrincipalSorted) the rows arrive already in // projection-key order and are streamed straight into // the SST writer with only one page resident. Otherwise (SQLite orders by grant @@ -284,15 +303,10 @@ func encodeProjectionKV(grant *v2.Grant, entitlementID string) ([]byte, []byte, resourceType: pid.GetResourceType(), resource: pid.GetResource(), }, grant.GetId()) - principalBytes, err := proto.Marshal(grant.GetPrincipal()) + val, err := encodeProjectionValueFromResource(grant.GetPrincipal(), isGrantDirectOnEntitlement(grant, entitlementID)) if err != nil { - return nil, nil, false, fmt.Errorf("topological projection: marshal principal: %w", err) - } - val := make([]byte, 1+len(principalBytes)) - if isGrantDirectOnEntitlement(grant, entitlementID) { - val[0] = 1 + return nil, nil, false, err } - copy(val[1:], principalBytes) return key, val, true, nil } @@ -323,6 +337,115 @@ func addProjectionRows(projDB *pebble.DB, grants []*v2.Grant, projectionSources return count, batch.Commit(pebble.NoSync) } +func addProjectionRowsFromSynthesized(projDB *pebble.DB, dest *v2.Entitlement, principals []*v3.PrincipalRef, sources []batonGrant.Sources, projectionSources map[string]struct{}) (int, error) { + if dest == nil { + return 0, nil + } + entID := dest.GetId() + if _, ok := projectionSources[entID]; !ok { + return 0, nil + } + batch := projDB.NewBatch() + defer batch.Close() + count := 0 + for i, principal := range principals { + if principal == nil { + continue + } + if principal.GetResourceTypeId() == "" || principal.GetResourceId() == "" { + continue + } + grantID := grantIDForPrincipalRef(dest, principal) + key := projectionKey(entID, topoPrincipalKey{ + resourceType: principal.GetResourceTypeId(), + resource: principal.GetResourceId(), + }, grantID) + val := encodeProjectionValueFromPrincipalRef(principal, sources[i].DirectFor(entID)) + if err := batch.Set(key, val, nil); err != nil { + return 0, err + } + count++ + } + if count == 0 { + return 0, nil + } + return count, batch.Commit(pebble.NoSync) +} + +func encodeProjectionValueFromResource(principal *v2.Resource, direct bool) ([]byte, error) { + var parent *v2.ResourceId + if principal != nil { + parent = principal.GetParentResourceId() + } + val := []byte{0} + if direct { + val[0] = 1 + } + val = codec.AppendTupleStrings(val, parent.GetResourceType(), parent.GetResource()) + if principal != nil { + principalBytes, err := proto.Marshal(principal) + if err != nil { + // A dropped payload would silently degrade fallback stores to a + // refs-only principal reconstruction; surface the failure instead. + return nil, fmt.Errorf("topological projection: marshal principal payload: %w", err) + } + val = codec.AppendTupleSeparator(val) + val = append(val, principalBytes...) + } + return val, nil +} + +func encodeProjectionValueFromPrincipalRef(principal *v3.PrincipalRef, direct bool) []byte { + val := []byte{0} + if direct { + val[0] = 1 + } + if principal == nil { + return codec.AppendTupleStrings(val, "", "") + } + return codec.AppendTupleStrings(val, principal.GetParentResourceTypeId(), principal.GetParentResourceId()) +} + +func projectionPrincipalRefFromValue(key topoPrincipalKey, val []byte) (principalRefData, []byte, bool) { + data := principalRefData{ + resourceTypeID: key.resourceType, + resourceID: key.resource, + ok: true, + } + if len(val) <= 1 { + return data, nil, true + } + parentRT, off, ok := codec.DecodeTupleStringAlias(val, 1) + if !ok { + return principalRefData{}, nil, false + } + if off < len(val) && val[off] == 0 { + off++ + } + parentID, off, ok := codec.DecodeTupleStringAlias(val, off) + if !ok { + return principalRefData{}, nil, false + } + data.parentResourceTypeID = string(parentRT) + data.parentResourceID = string(parentID) + var principalBytes []byte + if off < len(val) && val[off] == 0 { + principalBytes = val[off+1:] + } + return data, principalBytes, true +} + +// grantIDForPrincipalRef builds the legacy public grant id (raw concat) — +// byte-identical to what batonGrant.NewGrantID emits. External ids are an +// external-consumer contract; identity is structural and never derives +// from this string. +func grantIDForPrincipalRef(dest *v2.Entitlement, principal *v3.PrincipalRef) string { + if principal == nil { + return "" + } + return dest.GetId() + ":" + principal.GetResourceTypeId() + ":" + principal.GetResourceId() +} + func projectionSourceSet(plan *EntitlementGraphPlan) map[string]struct{} { out := make(map[string]struct{}, len(plan.GetProjectionSources())) for _, source := range plan.GetProjectionSources() { @@ -332,8 +455,8 @@ func projectionSourceSet(plan *EntitlementGraphPlan) map[string]struct{} { } // projectionKey encodes (entitlement, principal_rt, principal_id, external_id) -// with the engine's order-preserving tuple codec — the same encoding the -// permanent by_entitlement index uses. This keeps the projection key +// with the engine's order-preserving tuple codec — the same entitlement-first +// ordering as primary grant keys. This keeps the projection key // unambiguous when any component carries a raw 0x00/0x01 (which the codec // escapes), and makes the projection key order byte-for-byte identical to // Pebble's by_entitlement iteration order, so a principal-sorted store yields @@ -418,10 +541,11 @@ type projectionContributionStream struct { iter *pebble.Iterator started bool valid bool - // principalBuf stabilizes the current group's principal payload across the - // iterator advances in next's inner loop. It is reused for every group this - // stream yields; see next for the safety argument. - principalBuf []byte + // contrib is reused across groups: the k-way merge fully consumes a + // stream's current group before pulling the next one from that stream, so + // at most one group per stream is live at a time. The PrincipalRef handed + // to it is still allocated per group because downstream sinks retain it. + contrib topoContribution } func newProjectionContributionStream(db *pebble.DB, sourceID string, edge Edge) *projectionContributionStream { @@ -469,18 +593,14 @@ func (s *projectionContributionStream) next(_ context.Context) (contributionGrou s.valid = s.iter.Next() continue } - // Allocate the principal-key strings exactly once per group. key := topoPrincipalKey{resourceType: string(rt), resource: string(resource)} - contrib := &topoContribution{} - // Pebble's Value() bytes are only valid until the iterator advances, and - // the inner loop below advances it. Copy the principal payload once into - // this stream's reusable buffer so it survives the scan and reuses - // memory across groups. mergeContributionGroupStreams copies these bytes - // into the combined contribution during consume — before this stream is - // advanced again — so the per-group reuse is safe (see merge). - s.principalBuf = append(s.principalBuf[:0], val[1:]...) - contrib.principalBytes = s.principalBuf - contrib.addSource(s.sourceID, isDirect) + principalData, principalBytes, ok := projectionPrincipalRefFromValue(key, val) + if !ok { + return contributionGroup{}, false, fmt.Errorf("topological projection: decode principal ref for source %q", s.sourceID) + } + s.contrib.resetForReuse() + s.contrib.principal.setRef(principalData, principalBytes) + s.contrib.addSource(s.sourceID, isDirect) for s.valid = s.iter.Next(); s.valid; s.valid = s.iter.Next() { nextRT, nextResource, ok := decodeProjectionPrincipal(s.iter.Key(), s.prefix) if !ok { @@ -494,9 +614,9 @@ func (s *projectionContributionStream) next(_ context.Context) (contributionGrou if s.edge.IsShallow && !nextDirect { continue } - contrib.addSource(s.sourceID, nextDirect) + s.contrib.addSource(s.sourceID, nextDirect) } - return contributionGroup{key: key, contrib: contrib}, true, nil + return contributionGroup{key: key, contrib: &s.contrib}, true, nil } if s.iter == nil { return contributionGroup{}, false, nil diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge_streaming.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge_streaming.go index 35d1fade..770f992f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge_streaming.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge_streaming.go @@ -1,7 +1,6 @@ package expand import ( - "container/heap" "context" "fmt" "sort" @@ -14,12 +13,12 @@ import ( // only one principal group per active input stream, instead of materializing all // source entitlement grant groups into maps for each destination. func (e *Expander) RunTopologicalMergeStreaming(ctx context.Context) error { - entitlements, order, err := e.prepareTopological(ctx) + entitlements, layers, err := e.prepareTopological(ctx) if err != nil { return err } - if err := e.driveTopological(ctx, entitlements, order, topologicalRun{ - reduce: func(ctx context.Context, dest *v2.Entitlement, incoming []topoIncomingEdge, ents map[string]*v2.Entitlement, sink destinationSink) error { + if err := e.driveTopological(ctx, entitlements, layers, topologicalRun{ + reduce: func(ctx context.Context, dest *v2.Entitlement, incoming []topoIncomingEdge, ents map[string]*v2.Entitlement, sink *destinationSink) error { return e.mergeDestinationStreams(ctx, dest, incoming, ents, nil, nil, sink) }, }); err != nil { @@ -87,7 +86,10 @@ func (s *streamingPrincipalGroupStream) rawNext(ctx context.Context) (*v2.Grant, PageToken: s.pageToken, }.Build()) if err != nil { - return nil, false, err + // Identify which of potentially hundreds of thousands of + // entitlement streams failed; a raw engine error is + // undiagnosable at that scale. + return nil, false, fmt.Errorf("list grants for entitlement %q: %w", s.entitlement.GetId(), err) } s.buf = resp.GetList() s.pos = 0 @@ -235,6 +237,10 @@ type contributionStream struct { sourceEntitlementID string edge Edge stream principalGroupStream + // contrib is reused across groups: the k-way merge fully consumes a + // stream's current group before pulling the next one from that stream, so + // at most one group per stream is live at a time. + contrib topoContribution } func (s *contributionStream) next(ctx context.Context) (contributionGroup, bool, error) { @@ -243,17 +249,17 @@ func (s *contributionStream) next(ctx context.Context) (contributionGroup, bool, if err != nil || !ok { return contributionGroup{}, ok, err } - contrib := &topoContribution{} + s.contrib.resetForReuse() for _, grant := range group.grants { if !grantContributesOverEdge(grant, s.sourceEntitlementID, s.edge) { continue } - contrib.add(s.sourceEntitlementID, isGrantDirectOnEntitlement(grant, s.sourceEntitlementID), grant.GetPrincipal()) + s.contrib.add(s.sourceEntitlementID, isGrantDirectOnEntitlement(grant, s.sourceEntitlementID), grant.GetPrincipal()) } - if len(contrib.sources) == 0 { + if len(s.contrib.sources) == 0 { continue } - return contributionGroup{key: group.key, contrib: contrib}, true, nil + return contributionGroup{key: group.key, contrib: &s.contrib}, true, nil } } @@ -278,22 +284,57 @@ type mergeHeapItem struct { streamID int } +// mergeHeap is a hand-rolled binary min-heap of concrete mergeHeapItem values. +// container/heap boxes every Push/Pop through interface{}, which allocated per +// heap operation on the expansion hot path; the concrete implementation is +// allocation-free. type mergeHeap []mergeHeapItem -func (h mergeHeap) Len() int { return len(h) } -func (h mergeHeap) Less(i, j int) bool { +func (h mergeHeap) less(i, j int) bool { if h[i].group.key != h[j].group.key { return principalKeyLess(h[i].group.key, h[j].group.key) } return h[i].streamID < h[j].streamID } -func (h mergeHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *mergeHeap) Push(x any) { *h = append(*h, x.(mergeHeapItem)) } -func (h *mergeHeap) Pop() any { - old := *h - n := len(old) - item := old[n-1] - *h = old[:n-1] + +func (h *mergeHeap) push(item mergeHeapItem) { + *h = append(*h, item) + s := *h + i := len(s) - 1 + for i > 0 { + parent := (i - 1) / 2 + if !s.less(i, parent) { + break + } + s[i], s[parent] = s[parent], s[i] + i = parent + } +} + +func (h *mergeHeap) pop() mergeHeapItem { + s := *h + n := len(s) - 1 + s[0], s[n] = s[n], s[0] + item := s[n] + s[n] = mergeHeapItem{} // release contribution references + s = s[:n] + *h = s + i := 0 + for { + l := 2*i + 1 + if l >= n { + break + } + m := l + if r := l + 1; r < n && s.less(r, l) { + m = r + } + if !s.less(m, i) { + break + } + s[i], s[m] = s[m], s[i] + i = m + } return item } @@ -305,7 +346,7 @@ func mergeContributionGroupStreams( ctx context.Context, destEntitlement *v2.Entitlement, streams []contributionGroupStream, - sink destinationSink, + sink *destinationSink, ) error { for _, stream := range streams { defer stream.close() @@ -319,41 +360,42 @@ func mergeContributionGroupStreams( if !ok { continue } - heap.Push(&h, mergeHeapItem{group: group, streamID: i}) + h.push(mergeHeapItem{group: group, streamID: i}) } - flusher := newDirtyFlusher(sink) - for h.Len() > 0 { + flusher := newDirtyFlusher(destEntitlement, sink) + contrib := &topoContribution{} + var base []*v2.Grant + consume := func(group contributionGroup) { + if group.isBase { + base = append(base, group.base...) + return + } + contrib.merge(group.contrib) + } + for len(h) > 0 { if err := ctx.Err(); err != nil { return err } - item := heap.Pop(&h).(mergeHeapItem) + item := h.pop() key := item.group.key - var base []*v2.Grant - contrib := &topoContribution{} - - consume := func(group contributionGroup) { - if group.isBase { - base = append(base, group.base...) - return - } - contrib.merge(group.contrib) - } + base = base[:0] + contrib.resetForReuse() consume(item.group) if next, ok, err := streams[item.streamID].next(ctx); err != nil { return err } else if ok { - heap.Push(&h, mergeHeapItem{group: next, streamID: item.streamID}) + h.push(mergeHeapItem{group: next, streamID: item.streamID}) } - for h.Len() > 0 && h[0].group.key == key { - same := heap.Pop(&h).(mergeHeapItem) + for len(h) > 0 && h[0].group.key == key { + same := h.pop() consume(same.group) if next, ok, err := streams[same.streamID].next(ctx); err != nil { return err } else if ok { - heap.Push(&h, mergeHeapItem{group: next, streamID: same.streamID}) + h.push(mergeHeapItem{group: next, streamID: same.streamID}) } } @@ -361,18 +403,7 @@ func mergeContributionGroupStreams( continue } if len(base) == 0 { - principal, err := contrib.principalResource() - if err != nil { - return err - } - if principal == nil { - continue - } - grant, err := newExpandedGrantWithSources(destEntitlement, principal, contrib.sources) - if err != nil { - return err - } - if err := flusher.add(ctx, grant); err != nil { + if err := flusher.addSynthesizedContribution(ctx, contrib, contrib.sources); err != nil { return err } continue @@ -380,7 +411,7 @@ func mergeContributionGroupStreams( for _, baseGrant := range base { updated := mergeContributionIntoExistingGrant(baseGrant, destEntitlement.GetId(), contrib.sources) if updated != nil { - if err := flusher.add(ctx, updated); err != nil { + if err := flusher.add(ctx, updated, false); err != nil { return err } } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go index 734ccd54..51be5a36 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go @@ -38,6 +38,7 @@ type ProgressLog struct { lastEntitlementLog map[string]time.Time grantsProgress map[string]int lastGrantLog map[string]time.Time + grantsCountOnly map[string]bool mu sync.RWMutex l *zap.Logger maxLogFrequency time.Duration @@ -150,6 +151,7 @@ func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { lastEntitlementLog: make(map[string]time.Time), grantsProgress: make(map[string]int), lastGrantLog: make(map[string]time.Time), + grantsCountOnly: make(map[string]bool), l: ctxzap.Extract(ctx), maxLogFrequency: defaultMaxLogFrequency, mu: sync.RWMutex{}, @@ -256,18 +258,46 @@ func (p *ProgressLog) LogEntitlementsProgress(ctx context.Context, resourceType } } +// SetGrantsCountOnly marks a resource type's grant progress as a plain +// count with no resources-covered denominator. Used for type-scoped grant +// enumeration (v2.TypeScopedGrants), where cursors don't map 1:1 to +// resources: the per-cursor accounting would exceed the type's resource +// total and trip the "more grant resources than resources" warning for a +// perfectly healthy sync. Count-only types log synced row counts +// periodically and never compute the ratio. +func (p *ProgressLog) SetGrantsCountOnly(resourceType string) { + p.mu.Lock() + defer p.mu.Unlock() + p.grantsCountOnly[resourceType] = true +} + +// GrantsProgress returns the current grant-coverage counter for a resource +// type. For per-resource types this is "resources covered" and must never +// exceed the type's resource total — spawned sibling cursors +// (v2.SpawnCursors) don't increment it, only the origin action's chain end +// does. Exposed for tests pinning that accounting. +func (p *ProgressLog) GrantsProgress(resourceType string) int { + p.mu.RLock() + defer p.mu.RUnlock() + return p.grantsProgress[resourceType] +} + func (p *ProgressLog) LogGrantsProgress(ctx context.Context, resourceType string) { var grantsProgress, resources int var lastLogTime time.Time + var countOnly bool p.mu.RLock() grantsProgress = p.grantsProgress[resourceType] resources = p.resources[resourceType] lastLogTime = p.lastGrantLog[resourceType] + countOnly = p.grantsCountOnly[resourceType] p.mu.RUnlock() - if resources == 0 { - // if resuming sync, resource counts will be zero, so don't calculate percentage. just log every 10 seconds. + if resources == 0 || countOnly { + // Count-only: either a resumed sync (resource counts are zero) or a + // type-scoped grants type (no meaningful denominator). Log the raw + // synced count every log window; never compute a percentage. if time.Since(lastLogTime) > p.maxLogFrequency { p.l.Info("Syncing grants", zap.String("resource_type_id", resourceType), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go new file mode 100644 index 00000000..7ae42348 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go @@ -0,0 +1,354 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "fmt" + stdsync "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// Source-cache replay, syncer side. See +// proto/c1/connector/v2/annotation_source_cache.proto for the contract. +// +// Setup degrades, replay fails loudly. Any setup problem (capability +// absent, store engine unsupported, no usable previous sync) installs the +// no-op lookup: the connector never sees a previous validator, never gets +// a conditional-request hit, and therefore never emits SourceCacheReplay +// — which is what makes it safe to treat a replay annotation arriving +// while degraded as a hard error (the connector already skipped row +// generation; there is nothing to fall back to). + +// syncerSourceCache is the per-sync source-cache state resolved by +// configureSourceCache. +// +// Write side and read side enable independently: the FIRST sync of a chain +// has no previous sync but must still stamp rows and write manifest +// entries, or the second sync would have nothing to replay. enabled covers +// the write side (capability declared + current store supports it); prev +// is non-nil only when a usable previous sync exists (read side — lookup +// hits and replay). +type syncerSourceCache struct { + enabled bool + // current is the writable output store's source-cache capability. + current dotc1z.SourceCacheStore + // prev is the previous sync's lookup/replay source (read-only). Nil + // when no usable previous sync exists; lookups then miss and replay + // annotations are hard errors. + prev dotc1z.SourceCacheStore + // prevReader is the same store as prev, typed for ReplaySourceCache. + prevReader connectorstore.Reader + // lookup is the connector-facing lookup built from prev (NoopLookup + // when prev is nil). The syncer also uses it to answer lookup asks + // from connectors on single-shot transports (the ask/answer + // continuation); both paths see identical results by construction. + lookup sourcecache.Lookup + // contStats accumulates ask/answer continuation counters for the + // sync-complete log line. + contStats *continuationStats +} + +// prevStoreLookup adapts the previous store's manifest to the +// connector-facing Lookup. Mid-sync read errors are logged once and +// treated as misses: at lookup time the connector can still fetch fresh, +// so degrading beats failing the sync. +type prevStoreLookup struct { + prev dotc1z.SourceCacheStore + logOnce *stdsync.Once +} + +var _ sourcecache.Lookup = prevStoreLookup{} + +func (p prevStoreLookup) LookupPreviousSourceCache(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { + entry, found, err := p.prev.LookupSourceCacheEntry(ctx, kind, scopeHash) + if err != nil { + p.logOnce.Do(func() { + ctxzap.Extract(ctx).Warn("source cache lookup failed; treating as miss", zap.Error(err)) + }) + return sourcecache.Entry{}, false, nil //nolint:nilerr // intentional: a failed lookup degrades to a miss (connector fetches fresh) rather than failing the connector call + } + return entry, found, nil +} + +// configureSourceCache resolves per-sync source-cache state from the +// connector's Validate response and installs the connector-facing lookup. +// Called once per Sync, after Validate. +func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorServiceValidateResponse) error { + l := ctxzap.Extract(ctx) + s.sourceCache = syncerSourceCache{} + + setLookup, canSetLookup := s.connector.(sourcecache.SetLookup) + degrade := func(reason string) error { + if canSetLookup { + setLookup.SetSourceCache(ctx, sourcecache.NoopLookup{}) + } + if reason != "" { + l.Info("source cache disabled", zap.String("reason", reason)) + } + return nil + } + + capability := &v2.SourceCacheCapability{} + annos := annotations.Annotations(resp.GetAnnotations()) + ok, err := annos.Pick(capability) + if err != nil { + return fmt.Errorf("error parsing source cache capability annotation: %w", err) + } + if !ok || capability.GetMode() != v2.SourceCacheCapability_MODE_READ_WRITE { + // The common case; stay quiet. + return degrade("") + } + current, ok := s.store.(dotc1z.SourceCacheStore) + if !ok { + return degrade("storage engine does not support source cache") + } + + // Write side enabled: rows produced under a scope get stamped and + // manifest entries get written, so this sync is usable as the NEXT + // sync's replay source even when this one has nothing to replay from. + s.sourceCache = syncerSourceCache{enabled: true, current: current} + + // Read side: a usable previous sync makes lookups hit and replay legal. + var readReason string + if s.previousSyncReader == nil { + readReason = "no previous-sync c1z configured" + } else if prev, ok := s.previousSyncReader.(dotc1z.SourceCacheStore); !ok { + readReason = "previous-sync store engine does not support source cache" + } else { + s.sourceCache.prev = prev + s.sourceCache.prevReader = s.previousSyncReader + } + + lookup := sourcecache.Lookup(sourcecache.NoopLookup{}) + if s.sourceCache.prev != nil { + lookup = prevStoreLookup{prev: s.sourceCache.prev, logOnce: &stdsync.Once{}} + } + s.sourceCache.lookup = lookup + s.sourceCache.contStats = &continuationStats{} + if canSetLookup { + setLookup.SetSourceCache(ctx, lookup) + } else { + // The connector declared the capability but the client offers no + // way to deliver lookups. Its own lookup stays no-op, so every + // scope misses and no replay annotations can legally arrive. + l.Warn("source cache capability declared but connector client cannot receive lookups") + } + l.Info("source cache enabled", + zap.Bool("replay_available", s.sourceCache.prev != nil), + zap.String("replay_unavailable_reason", readReason), + ) + return nil +} + +// clearSourceCacheLookup detaches the per-sync lookup so a late RPC from +// the connector cannot read a store the syncer no longer owns. +func (s *syncer) clearSourceCacheLookup(ctx context.Context) { + if setLookup, ok := s.connector.(sourcecache.SetLookup); ok { + setLookup.SetSourceCache(ctx, nil) + } +} + +// sourceCachePage carries one list response's source-cache instructions +// from beginSourceCachePage (before rows are written) to +// finishSourceCachePage (after rows are written). +type sourceCachePage struct { + kind sourcecache.RowKind + scopeHash string + etag string + // replayed reports that beginSourceCachePage copied the previous + // sync's rows for this scope into the current sync BEFORE the page's + // own rows commit. Consumers that dedupe against "already synced this + // sync" state (the resources path) must not skip this page's rows: + // they are the overlay, and the already-present row is the stale + // replayed base they exist to overwrite. + replayed bool + // deletedIDs are canonical-id tombstones (grant/entitlement ids, + // resource BIDs); deletedPrincipalIDs are bare-object-id tombstones + // applied scope-relatively. Both may arrive on any page of a scope + // (replay annotation on the first page, scope annotation on every + // page) and apply after the page's rows commit. + deletedIDs []string + deletedPrincipalIDs []string +} + +// beginSourceCachePage inspects a list response's annotations, performs +// any requested replay, and returns the context to write the page's rows +// under (stamped with the scope when one is present). A nil page means the +// response carried no source-cache instructions. +// +// rowCount is the number of rows in the response; a non-overlay replay +// that also returned rows gets a warning (the rows are upserted anyway). +func (s *syncer) beginSourceCachePage( + ctx context.Context, + kind sourcecache.RowKind, + respAnnos annotations.Annotations, + rowCount int, +) (context.Context, *sourceCachePage, error) { + replay := &v2.SourceCacheReplay{} + hasReplay, err := respAnnos.Pick(replay) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing replay annotation: %w", err) + } + scope := &v2.SourceCacheScope{} + hasScope, err := respAnnos.Pick(scope) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing scope annotation: %w", err) + } + if !hasReplay && !hasScope { + return ctx, nil, nil + } + + if !s.sourceCache.enabled { + if hasReplay { + // The connector skipped row generation expecting a replay; with + // source cache disabled there is nothing to replay from. This is + // a connector bug (replay for a scope it never got a lookup hit + // on), not a degradable condition. + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but source cache is disabled", replay.GetScopeHash()) + } + // Scope annotations without the capability handshake are ignored. + return ctx, nil, nil + } + + page := &sourceCachePage{kind: kind} + switch { + case hasReplay && hasScope: + if replay.GetScopeHash() != scope.GetScopeHash() { + return ctx, nil, fmt.Errorf("source cache: replay scope %q and page scope %q disagree", replay.GetScopeHash(), scope.GetScopeHash()) + } + page.scopeHash = replay.GetScopeHash() + case hasReplay: + page.scopeHash = replay.GetScopeHash() + default: + page.scopeHash = scope.GetScopeHash() + } + if err := sourcecache.ValidateScopeHash(page.scopeHash); err != nil { + return ctx, nil, fmt.Errorf("source cache: %w", err) + } + // Prefer the scope annotation's etag (the freshest validator on + // overlay pages); fall back to the replay's. + page.etag = scope.GetEtag() + if page.etag == "" { + page.etag = replay.GetEtag() + } + // Tombstones may ride either annotation — the replay annotation on a + // round's first page, the scope annotation on every page (so a + // multi-page delta round never buffers deletions). + page.deletedIDs = append(replay.GetDeletedIds(), scope.GetDeletedIds()...) + page.deletedPrincipalIDs = append(replay.GetDeletedPrincipalIds(), scope.GetDeletedPrincipalIds()...) + + if hasReplay { + if s.sourceCache.prev == nil { + // Same invariant violation as the disabled case: the connector + // can only have gotten a lookup hit if a previous source exists. + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but no previous sync is available", replay.GetScopeHash()) + } + page.replayed = true + if !replay.GetOverlay() && rowCount > 0 { + // The contract says a 304-style replay page is empty, but rows + // arriving here are more data, not less — upsert them on top of + // the replayed base (overlay semantics) rather than failing the + // sync. Kept lenient while the model is proven against real + // providers. + ctxzap.Extract(ctx).Warn("source cache: non-overlay replay returned rows; treating them as an overlay", + zap.String("scope_hash", page.scopeHash), + zap.Int("rows", rowCount), + ) + } + // Advisory check: a well-behaved connector only replays a scope + // whose validator came from this sync's lookup, so a missing + // previous manifest entry is suspicious — but not by itself data + // loss (the stamped rows may still exist, e.g. a partially carried + // file). The hard error below is reserved for a replay that + // produces nothing. + _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeHash) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error reading previous manifest for scope %q: %w", page.scopeHash, err) + } + if !entryFound { + ctxzap.Extract(ctx).Warn("source cache: replay requested for scope with no previous manifest entry", + zap.String("scope_hash", page.scopeHash)) + } + res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeHash) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: replay for scope %q failed: %w", page.scopeHash, err) + } + if res.Rows == 0 && !entryFound { + // The connector skipped row generation expecting a base that + // does not exist anywhere in the previous file — this sync + // would silently drop the scope's rows. + return ctx, nil, fmt.Errorf("source cache: replay for scope %q found no previous rows and no manifest entry; the connector replayed a scope it never looked up", page.scopeHash) + } + // Replay bypasses the connector-response path that normally arms + // grant expansion (seeing GrantExpandable on returned rows), so a + // sync whose expandable pages all replay would silently skip the + // expansion phase without this. + if kind == sourcecache.RowKindGrants && res.NeedsExpansion && !s.dontExpandGrants { + s.state.SetNeedsExpansion() + } + ctxzap.Extract(ctx).Debug("source cache replayed scope", + zap.String("row_kind", string(kind)), + zap.String("scope_hash", page.scopeHash), + zap.Int64("rows", res.Rows), + zap.Bool("needs_expansion", res.NeedsExpansion), + zap.Int("deleted_ids", len(page.deletedIDs)), + zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), + ) + } + + return sourcecache.WithScope(ctx, page.scopeHash), page, nil +} + +// finishSourceCachePage runs after the page's rows committed: applies +// delta tombstones and, when the page carried a validator, writes the +// current sync's manifest entry. The entry write is last so a failed page +// can never leave a phantom hit for the next sync. +func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePage) error { + if page == nil { + return nil + } + if len(page.deletedIDs) > 0 { + if page.kind == sourcecache.RowKindGrants { + // Grant-id tombstones resolve within the scope's own rows so + // connector-custom grant-id shapes (unreachable by the global + // bounded delete) work, and the cost stays bounded by the + // scope's size. + deleted, err := s.sourceCache.current.DeleteSourceCacheGrantsByIDInScope(ctx, page.scopeHash, page.deletedIDs) + if err != nil { + return fmt.Errorf("source cache: error applying grant deletions for scope %q: %w", page.scopeHash, err) + } + ctxzap.Extract(ctx).Debug("source cache applied grant-id deletions", + zap.String("scope_hash", page.scopeHash), + zap.Int("tombstones", len(page.deletedIDs)), + zap.Int64("rows_deleted", deleted), + ) + } else if err := s.sourceCache.current.DeleteSourceCacheRows(ctx, page.kind, page.deletedIDs); err != nil { + return fmt.Errorf("source cache: error applying deletions for scope %q: %w", page.scopeHash, err) + } + } + if len(page.deletedPrincipalIDs) > 0 { + deleted, err := s.sourceCache.current.DeleteSourceCacheRowsInScope(ctx, page.kind, page.scopeHash, page.deletedPrincipalIDs) + if err != nil { + return fmt.Errorf("source cache: error applying scoped deletions for scope %q: %w", page.scopeHash, err) + } + ctxzap.Extract(ctx).Debug("source cache applied scoped deletions", + zap.String("row_kind", string(page.kind)), + zap.String("scope_hash", page.scopeHash), + zap.Int("tombstones", len(page.deletedPrincipalIDs)), + zap.Int64("rows_deleted", deleted), + ) + } + if page.etag != "" { + if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeHash, page.etag); err != nil { + return fmt.Errorf("source cache: error writing manifest entry for scope %q: %w", page.scopeHash, err) + } + } + return nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go new file mode 100644 index 00000000..76fb7276 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go @@ -0,0 +1,248 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Syncer side of the source-cache lookup continuation (ask/answer). On +// single-shot transports (gRPC-over-Lambda) the connector cannot call the +// lookup service mid-request, so it answers a list RPC with a +// SourceCacheLookupAsk instead of rows; the syncer resolves the queries +// against its LOCAL previous-sync store — the same store replay copies +// from, so lookup and replay can never disagree — and re-invokes the same +// request with SourceCacheLookupAnswers attached. See +// docs/tasks/source-cache-lambda-lookup.md and the annotation contract in +// proto/c1/connector/v2/annotation_source_cache.proto. + +import ( + "context" + "fmt" + stdsync "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +const ( + // sourceCacheBounceCap bounds consecutive asks for the SAME request + // (same page token; only the answers annotation differs between + // re-invokes). A connector that keeps asking without progressing is + // broken (most commonly: swallowing ErrLookupDeferred and re-asking + // for scopes it already "handled"), and silence would be the + // stale-data failure mode — so fail loudly. + // + // Deliberately NOT per action: a multi-page action that asks once per + // page (e.g. a delta planner pre-resolving each planning page's chunk + // scopes) bounces once per page with monotonic progress — every + // NextPageToken advance is a new request and resets the counter. + sourceCacheBounceCap = 4 + + // sourceCacheAnswerBudget caps the FOUND-etag payload attached to one + // re-invoke, keeping the request under single-shot transport payload + // limits (Lambda invokes cap at 6MB; the dual-encoded frame at 5MiB). + // Not-found answers are always complete for the queried set — only + // found answers with large etags are dropped, and a dropped answer is + // ABSENT (re-askable, subject to the cap), never a false not-found. + sourceCacheAnswerBudget = 2 << 20 +) + +// continuationStats accumulates ask/answer counters across a sync for the +// sync-complete log line and bounce-cap diagnostics. Per-op-kind bounce +// counts let a rollout review distinguish planner asks (expected, one per +// planning page) from per-row asks (a batching opportunity). +type continuationStats struct { + mu stdsync.Mutex + requests int // RPCs that bounced at least once + bounces int + bouncesByOp map[string]int + asked int + found int + notFound int + truncated int + capFailures int +} + +func (c *continuationStats) record(op string, bounces, asked, found, notFound, truncated int) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if bounces > 0 { + c.requests++ + if c.bouncesByOp == nil { + c.bouncesByOp = map[string]int{} + } + c.bouncesByOp[op] += bounces + } + c.bounces += bounces + c.asked += asked + c.found += found + c.notFound += notFound + c.truncated += truncated +} + +func (c *continuationStats) recordCapFailure() { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.capFailures++ +} + +// logTotals emits the continuation counters when any bounces happened. +func (c *continuationStats) logTotals(ctx context.Context) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.bounces == 0 && c.capFailures == 0 { + return + } + fields := []zap.Field{ + zap.Int("requests_bounced", c.requests), + zap.Int("bounces", c.bounces), + zap.Int("scopes_asked", c.asked), + zap.Int("answered_found", c.found), + zap.Int("answered_not_found", c.notFound), + zap.Int("answers_truncated", c.truncated), + zap.Int("bounce_cap_failures", c.capFailures), + } + for op, n := range c.bouncesByOp { + fields = append(fields, zap.Int("bounces_"+op, n)) + } + ctxzap.Extract(ctx).Info("source-cache lookup continuation totals", fields...) +} + +// listAttempt is one list-RPC attempt as observed by the continuation +// loop: enough of the response to detect and validate an ask. +type listAttempt struct { + annos annotations.Annotations + rows int + nextToken string +} + +// withSourceCacheContinuation drives the ask/answer loop around one list +// RPC. issue performs the RPC with extra request annotations (the lookup +// offer, plus accumulated answers on re-invokes) and reports the response +// surface; the loop returns once a response carries no ask — that final +// response is the one the caller processes. +// +// The offer is attached only when the syncer can actually answer (warm +// previous-sync lookup). Old or cold syncers never send it, and a +// compliant connector never asks without it — that pairing is what makes +// version skew degrade to a cold sync instead of a misread response. +func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, issue func(extra annotations.Annotations) (listAttempt, error)) error { + l := ctxzap.Extract(ctx) + + warm := s.sourceCache.prev != nil + extra := annotations.Annotations{} + if warm { + extra.Update(&v2.SourceCacheLookupOffer{}) + } + + // Answers accumulate across bounces: the connector re-executes from + // scratch each phase, so every re-invoke must carry the union of all + // resolved queries, in first-resolved order (deterministic requests). + var ordered []sourcecache.Answer + seen := map[sourcecache.Query]bool{} + + asked, found, notFound, truncated := 0, 0, 0, 0 + for bounce := 0; ; bounce++ { + attempt, err := issue(extra) + if err != nil { + return err + } + + ask := &v2.SourceCacheLookupAsk{} + hasAsk, err := attempt.annos.Pick(ask) + if err != nil { + return fmt.Errorf("%s: error parsing source-cache lookup ask: %w", op, err) + } + if !hasAsk { + s.sourceCache.contStats.record(op, bounce, asked, found, notFound, truncated) + return nil + } + + // Ask legality. Failing loudly here is deliberate: every branch + // is a connector bug that would otherwise surface as silently + // wrong data or an unexplained stall. + if !warm { + return fmt.Errorf("%s: connector sent a source-cache lookup ask on a request that carried no offer (connector must gate asks on SourceCacheLookupOffer)", op) + } + if attempt.rows > 0 || attempt.nextToken != "" || + attempt.annos.Contains(&v2.SourceCacheScope{}) || attempt.annos.Contains(&v2.SourceCacheReplay{}) || + attempt.annos.Contains(&v2.SpawnCursors{}) { + return fmt.Errorf("%s: source-cache lookup ask response must carry ONLY the ask: "+ + "no rows, no next page token, no scope/replay annotations, no spawned cursors "+ + "(spawn on the re-invoked request's real response instead)", op) + } + if bounce >= sourceCacheBounceCap { + s.sourceCache.contStats.recordCapFailure() + return fmt.Errorf("%s: source-cache lookup bounce cap (%d) exceeded for one request; "+ + "connector kept asking without progressing (%d scopes still unresolved) — "+ + "check for swallowed ErrLookupDeferred or nondeterministic scope computation", + op, sourceCacheBounceCap, len(ask.GetQueries())) + } + + queries, err := sourcecache.QueriesFromProto(ask) + if err != nil { + return fmt.Errorf("%s: invalid source-cache lookup ask: %w", op, err) + } + + budget := sourceCacheAnswerBudget + for _, a := range ordered { + budget -= len(a.ETag) + } + newAsked, newFound, newNotFound, newTruncated := 0, 0, 0, 0 + for _, q := range queries { + if seen[q] { + continue + } + newAsked++ + entry, ok, err := s.sourceCache.lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + if err != nil { + return fmt.Errorf("%s: resolving source-cache lookup ask: %w", op, err) + } + if !ok { + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: false}) + newNotFound++ + continue + } + if len(entry.ETag) > budget { + // Dropped to budget: the query stays ABSENT from the + // answers (re-askable), never a false not-found. + newTruncated++ + continue + } + budget -= len(entry.ETag) + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: true, ETag: entry.ETag}) + newFound++ + } + asked += newAsked + found += newFound + notFound += newNotFound + truncated += newTruncated + + if newAsked == 0 { + // Every query was already answered on the request the + // connector just saw; re-invoking cannot make progress. + return fmt.Errorf("%s: connector re-asked only already-answered scopes (%d queries); connector lookup handling is broken", op, len(queries)) + } + + extra.Update(sourcecache.AnswersProto(ordered)) + + l.Debug("source-cache lookup bounce", + zap.String("op", op), + zap.Int("bounce", bounce+1), + zap.Int("asked", newAsked), + zap.Int("found", newFound), + zap.Int("not_found", newNotFound), + zap.Int("truncated_to_budget", newTruncated), + ) + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go index cb1d30ca..bcf89bb7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go @@ -202,6 +202,15 @@ type Action struct { ResourceID string `json:"resource_id,omitempty"` ParentResourceTypeID string `json:"parent_resource_type_id,omitempty"` ParentResourceID string `json:"parent_resource_id,omitempty"` + + // Spawned marks an action enqueued by a connector's SpawnCursors + // annotation (a sibling cursor) rather than by the syncer's own + // planners. Progress accounting uses it: per-resource grant coverage + // counts a resource once, when its ORIGIN action's chain ends — + // spawned siblings (and their NextPage continuations, which inherit + // the action) don't count, or one resource would count N times. + // omitempty keeps old checkpointed sync tokens decoding unchanged. + Spawned bool `json:"spawned,omitempty"` } var _ State = &state{} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index bf0a4a27..d64e99a9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -16,8 +16,11 @@ import ( "time" "github.com/Masterminds/semver/v3" + storage_v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/conductorone/baton-sdk/pkg/types/entitlement" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" @@ -114,7 +117,7 @@ type syncer struct { externalResourceEntitlementIdFilter string previousSyncC1ZPath string previousSyncC1ZPathOptional bool - store dotc1z.C1ZStore + store c1zstore.Store externalResourceReader connectorstore.Reader previousSyncReader connectorstore.Reader connector types.ConnectorClient @@ -123,7 +126,7 @@ type syncer struct { transitionHandler func(s Action) progressHandler func(p *Progress) tmpDir string - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine skipFullSync bool lastCheckPointTime time.Time counts *progresslog.ProgressLog @@ -133,6 +136,7 @@ type syncer struct { syncID string skipEGForResourceType syncMap[string, bool] skipEntitlementsForResourceType syncMap[string, bool] + typeScopedGrantsForResourceType syncMap[string, bool] skipEntitlementsAndGrants bool skipGrants bool resourceTypeTraits syncMap[string, []v2.ResourceType_Trait] @@ -143,6 +147,7 @@ type syncer struct { workerCount int // If 1, sync is sequential (default). If > 1, sync operations are done in parallel. metricsHandler metrics.Handler syncIdentity uotel.SyncIdentity + sourceCache syncerSourceCache } var _ Syncer = (*syncer)(nil) @@ -151,17 +156,37 @@ var _ Syncer = (*syncer)(nil) // GrantStore.StoreExpandedGrants so the expander package can depend on // a single narrow interface without knowing about C1ZStore. type expanderStoreAdapter struct { - store dotc1z.C1ZStore + store c1zstore.Store } func (a expanderStoreAdapter) GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) { return a.store.GetEntitlement(ctx, req) } +// requireEntitlementRefs guards the expansion → store boundary: grant +// listings during expansion must address the entitlement by its structured +// resource refs (from a fetched entitlement record), never by a bare id +// string. Bare-id resolution is a query-planning convenience reserved for +// interactive edges (CLI, explorer) where an ambiguity error is acceptable; +// expansion must not depend on it. Every expansion call site fetches the +// entitlement record first, so this only fires on a regression. +func requireEntitlementRefs(ent *v2.Entitlement) error { + if ent == nil || ent.GetId() == "" { + return errors.New("grant expansion: missing entitlement") + } + if res := ent.GetResource(); res.GetId().GetResourceType() == "" || res.GetId().GetResource() == "" { + return fmt.Errorf("grant expansion: entitlement %q has no resource refs; expansion must not resolve bare id strings", ent.GetId()) + } + return nil +} + func (a expanderStoreAdapter) ListGrantsForEntitlement( ctx context.Context, req *reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest, ) (*reader_v2.GrantsReaderServiceListGrantsForEntitlementResponse, error) { + if err := requireEntitlementRefs(req.GetEntitlement()); err != nil { + return nil, err + } return a.store.ListGrantsForEntitlement(ctx, req) } @@ -171,6 +196,9 @@ func (a expanderStoreAdapter) ListGrantPrincipalKeysForEntitlement( pageToken string, pageSize uint32, ) ([]string, string, error) { + if err := requireEntitlementRefs(entitlement); err != nil { + return nil, "", err + } // Preserve Pebble's compact prefetch path through this wrapper. Non-Pebble // stores fall back to regular grant listing and local key extraction. if store, ok := a.store.(interface { @@ -201,6 +229,93 @@ func (a expanderStoreAdapter) StoreExpandedGrants(ctx context.Context, grants .. return a.store.Grants().StoreExpandedGrants(ctx, grants...) } +func (a expanderStoreAdapter) StoreNewExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { + if fast, ok := a.store.Grants().(interface { + StoreNewExpandedGrants(context.Context, ...*v2.Grant) error + }); ok { + return fast.StoreNewExpandedGrants(ctx, grants...) + } + return a.store.Grants().StoreExpandedGrants(ctx, grants...) +} + +func (a expanderStoreAdapter) StoreNewExpandedGrantContributions(ctx context.Context, dest *v2.Entitlement, principals []*storage_v3.PrincipalRef, sources []batonGrant.Sources) error { + if fast, ok := a.store.Grants().(interface { + StoreNewExpandedGrantContributions(context.Context, *v2.Entitlement, []*storage_v3.PrincipalRef, []batonGrant.Sources) error + }); ok { + return fast.StoreNewExpandedGrantContributions(ctx, dest, principals, sources) + } + grants := make([]*v2.Grant, 0, len(principals)) + for i, principalRef := range principals { + principal := resourceFromPrincipalRef(principalRef) + grant, err := expand.NewExpandedGrantForStore(dest, principal, sources[i]) + if err != nil { + return err + } + grants = append(grants, grant) + } + return a.store.Grants().StoreExpandedGrants(ctx, grants...) +} + +// expandedGrantLayerStorer is the layer-scoped synthesized-grant layer session +// surface the store's GrantStore may implement (Pebble). Local interface so +// the adapter can pass sessions through without importing engine internals. +type expandedGrantLayerStorer interface { + BeginExpandedGrantLayer(ctx context.Context) (bool, error) + AddExpandedGrantLayerContributions(ctx context.Context, dest *v2.Entitlement, principals []*storage_v3.PrincipalRef, sources []batonGrant.Sources) error + FinishExpandedGrantLayer(ctx context.Context) error + AbortExpandedGrantLayer(ctx context.Context) error +} + +func (a expanderStoreAdapter) BeginExpandedGrantLayer(ctx context.Context) (bool, error) { + if fast, ok := a.store.Grants().(expandedGrantLayerStorer); ok { + return fast.BeginExpandedGrantLayer(ctx) + } + return false, nil +} + +func (a expanderStoreAdapter) AddExpandedGrantLayerContributions(ctx context.Context, dest *v2.Entitlement, principals []*storage_v3.PrincipalRef, sources []batonGrant.Sources) error { + fast, ok := a.store.Grants().(expandedGrantLayerStorer) + if !ok { + return errors.New("expanded grant layer: store does not support layer sessions") + } + return fast.AddExpandedGrantLayerContributions(ctx, dest, principals, sources) +} + +func (a expanderStoreAdapter) FinishExpandedGrantLayer(ctx context.Context) error { + fast, ok := a.store.Grants().(expandedGrantLayerStorer) + if !ok { + return errors.New("expanded grant layer: store does not support layer sessions") + } + return fast.FinishExpandedGrantLayer(ctx) +} + +func (a expanderStoreAdapter) AbortExpandedGrantLayer(ctx context.Context) error { + if fast, ok := a.store.Grants().(expandedGrantLayerStorer); ok { + return fast.AbortExpandedGrantLayer(ctx) + } + return nil +} + +func resourceFromPrincipalRef(ref *storage_v3.PrincipalRef) *v2.Resource { + if ref == nil { + return nil + } + var parent *v2.ResourceId + if ref.GetParentResourceId() != "" { + parent = v2.ResourceId_builder{ + ResourceType: ref.GetParentResourceTypeId(), + Resource: ref.GetParentResourceId(), + }.Build() + } + return v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: ref.GetResourceTypeId(), + Resource: ref.GetResourceId(), + }.Build(), + ParentResourceId: parent, + }.Build() +} + // GrantsForEntitlementPrincipalSorted forwards the underlying engine's // principal-sort guarantee (Pebble) so the topological merge can stream grant // groups instead of buffering and sorting each entitlement. Engines that do not @@ -450,6 +565,11 @@ func (s *syncer) Sync(ctx context.Context) error { } } + err = s.configureSourceCache(ctx, resp) + if err != nil { + return err + } + syncResourceTypeMap := make(map[string]bool) if len(s.syncResourceTypes) > 0 { for _, rt := range s.syncResourceTypes { @@ -575,6 +695,7 @@ func (s *syncer) Sync(ctx context.Context) error { return err } + s.sourceCache.contStats.logTotals(ctx) l.Info("Sync complete.") _, err = s.connector.Cleanup(ctx, v2.ConnectorServiceCleanupRequest_builder{ @@ -930,23 +1051,54 @@ func (s *syncer) SyncResources(ctx context.Context, action *Action) error { // owns a span — the duplicate inflated trace span counts without adding // information. func (s *syncer) syncResources(ctx context.Context, action *Action) error { - req := v2.ResourcesServiceListResourcesRequest_builder{ - ResourceTypeId: action.ResourceTypeID, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build() - if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { - req.SetParentResourceId(v2.ResourceId_builder{ - ResourceType: action.ParentResourceTypeID, - Resource: action.ParentResourceID, - }.Build()) + var resp *v2.ResourcesServiceListResourcesResponse + err := s.withSourceCacheContinuation(ctx, "sync-resources", func(extra annotations.Annotations) (listAttempt, error) { + req := v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: action.ResourceTypeID, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: extra, + }.Build() + if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { + req.SetParentResourceId(v2.ResourceId_builder{ + ResourceType: action.ParentResourceTypeID, + Resource: action.ParentResourceID, + }.Build()) + } + r, err := s.connector.ListResources(ctx, req) + if err != nil { + return listAttempt{}, err + } + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) + if err != nil { + return err } - resp, err := s.connector.ListResources(ctx, req) + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindResources, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) if err != nil { return err } + // On any source-cache-scoped page the "already synced this sync" + // dedupe below is wrong — scoped pages are upsert streams whose rows + // are always authoritative: + // - Replayed rounds copy the previous sync's rows in on the round's + // FIRST page, so an overlay row's identity ALWAYS hits the store, + // on every page of the round — but the stored row is the stale + // base and the overlay row is the update that must overwrite it. + // Keying off the replay annotation alone would only protect page + // one (the annotation fires once per round). + // - Cold delta enumerations may legally return the same object + // multiple times (changed mid-walk), later occurrences + // authoritative; deduping would keep the STALE first occurrence. + pageScoped := scPage != nil + bulkPutResoruces := []*v2.Resource{} for _, r := range resp.GetList() { validatedResource := false @@ -963,7 +1115,7 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { validatedResource = true // We must *ALSO* check if we have any child resources. - if !s.hasChildResources(r) { + if !pageScoped && !s.hasChildResources(r) { // Since we only have the resource type IDs of child resources, // we can't tell if we already have synced those child resources. // Those children may also have their own child resources, @@ -992,12 +1144,16 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { } if len(bulkPutResoruces) > 0 { - err = s.store.PutResources(ctx, bulkPutResoruces...) + err = s.store.PutResources(putCtx, bulkPutResoruces...) if err != nil { return err } } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) s.counts.AddResources(action.ResourceTypeID, len(resp.GetList())) if resp.GetNextPageToken() == "" { @@ -1205,22 +1361,45 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action resource := resourceResponse.GetResource() - resp, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build()) + var resp *v2.EntitlementsServiceListEntitlementsResponse + err = s.withSourceCacheContinuation(ctx, "sync-entitlements", func(extra annotations.Annotations) (listAttempt, error) { + r, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: extra, + }.Build()) + if err != nil { + return listAttempt{}, err + } + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return err } if err := s.validateEntitlementExclusionGroups(resp.GetList()); err != nil { return err } - err = s.store.PutEntitlements(ctx, resp.GetList()...) + + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindEntitlements, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) + if err != nil { + return err + } + + err = s.store.PutEntitlements(putCtx, resp.GetList()...) if err != nil { return err } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) if resp.GetNextPageToken() == "" { s.counts.AddEntitlementsProgress(resourceID.ResourceType, 1) @@ -1646,6 +1825,9 @@ func (s *syncer) fixEntitlementGraphCycles(ctx context.Context, graph *expand.En // SyncGrants fetches the grants for each resource from the connector. It iterates each resource // from the datastore, and pushes a new action to sync the grants for each individual resource. +// Resource types annotated with TypeScopedGrants are excluded from the per-resource fan-out and +// get a single type-scoped action instead (empty ResourceID); the connector enumerates the whole +// type, optionally spawning additional cursors via the SpawnCursors annotation. func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { ctx, span := uotel.StartWithLink(ctx, tracer, "syncer.SyncGrants") uotel.SetSyncIdentityAttrs(ctx, span) @@ -1653,9 +1835,20 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { defer func() { uotel.EndSpanWithError(span, err) }() if action.ResourceTypeID == "" && action.ResourceID == "" { + actions := make([]Action, 0) if action.PageToken == "" { ctxzap.Extract(ctx).Info("Syncing grants...") s.handleInitialActionForStep(ctx, *action) + + // One type-scoped action per annotated resource type, enqueued + // exactly once (the planner's first page). + typeScoped, err := s.typeScopedGrantsResourceTypes(ctx) + if err != nil { + return fmt.Errorf("sync-grants: error listing type-scoped resource types: %w", err) + } + for _, rtID := range typeScoped { + actions = append(actions, Action{Op: SyncGrantsOp, ResourceTypeID: rtID}) + } } resp, err := s.store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ @@ -1666,16 +1859,25 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return fmt.Errorf("sync-grants: error listing resources: %w", err) } - actions := make([]Action, 0) for _, r := range resp.GetList() { shouldSkip, err := s.shouldSkipGrants(ctx, r) if err != nil { return err } - if shouldSkip { continue } + + // Types with type-scoped grants are excluded from the + // per-resource fan-out; their single action is enqueued above. + typeScoped, err := s.resourceTypeHasTypeScopedGrants(ctx, r.GetId().GetResourceType()) + if err != nil { + return err + } + if typeScoped { + continue + } + actions = append(actions, Action{Op: SyncGrantsOp, ResourceID: r.GetId().GetResource(), ResourceTypeID: r.GetId().GetResourceType()}) } @@ -1689,27 +1891,107 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return nil } +// resourceTypeHasTypeScopedGrants reports (cached per sync) whether the +// resource type carries the TypeScopedGrants annotation. +func (s *syncer) resourceTypeHasTypeScopedGrants(ctx context.Context, resourceTypeID string) (bool, error) { + if v, ok := s.typeScopedGrantsForResourceType.Load(resourceTypeID); ok { + return v, nil + } + rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: resourceTypeID, + }.Build()) + if err != nil { + return false, err + } + rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) + s.typeScopedGrantsForResourceType.Store(resourceTypeID, typeScoped) + return typeScoped, nil +} + +// typeScopedGrantsResourceTypes lists every synced resource type annotated +// with TypeScopedGrants. +func (s *syncer) typeScopedGrantsResourceTypes(ctx context.Context) ([]string, error) { + var out []string + pageToken := "" + for { + resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return nil, err + } + for _, rt := range resp.GetList() { + rtAnnos := annotations.Annotations(rt.GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) + s.typeScopedGrantsForResourceType.Store(rt.GetId(), typeScoped) + if typeScoped { + out = append(out, rt.GetId()) + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return out, nil + } + } +} + // syncGrantsForResource fetches the grants for a specific resource from the connector. +// An action with an empty ResourceID is a TYPE-SCOPED grants cursor: the connector +// enumerates grants for the whole resource type (no single resource backs the call), +// and may spawn sibling cursors via the SpawnCursors response annotation. // No span here: only call site is SyncGrants, which already owns a span. func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) error { + typeScoped := action.ResourceID == "" resourceID := v2.ResourceId_builder{ ResourceType: action.ResourceTypeID, Resource: action.ResourceID, }.Build() - resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ - ResourceId: resourceID, - }.Build()) - if err != nil { - return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) - } - resource := resourceResponse.GetResource() + var resource *v2.Resource + var reqAnnos annotations.Annotations + if typeScoped { + // Wire validation requires a non-empty resource id, so the stub is + // self-referential ({type, type}) and the request carries the + // TypeScopedGrants annotation as the routing marker. + resource = v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: action.ResourceTypeID, + Resource: action.ResourceTypeID, + }.Build(), + }.Build() + reqAnnos.Update(&v2.TypeScopedGrants{}) + } else { + resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ + ResourceId: resourceID, + }.Build()) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) + } + resource = resourceResponse.GetResource() + } - resp, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build()) + var resp *v2.GrantsServiceListGrantsResponse + err := s.withSourceCacheContinuation(ctx, "sync-grants-for-resource", func(extra annotations.Annotations) (listAttempt, error) { + annos := make(annotations.Annotations, 0, len(reqAnnos)+len(extra)) + annos = append(annos, reqAnnos...) + annos = append(annos, extra...) + r, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: annos, + }.Build()) + if err != nil { + return listAttempt{}, err + } + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return fmt.Errorf("sync-grants-for-resource: error listing grants: %w", err) } @@ -1721,6 +2003,15 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro respAnnos := annotations.Annotations(resp.GetAnnotations()) insertResourceGrants := respAnnos.Contains(&v2.InsertResourceGrants{}) + // Source-cache replay/stamping for this grants page. putCtx applies + // ONLY to the PutGrants call below — the related-resource PutResources + // writes in this function are resource rows and must not inherit a + // grants-scope stamp. + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, respAnnos, len(grants)) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + // Stamp InsertResourceGrants per-grant so the slim-blob writer's // gate sees it. The annotation is response-level, but the writer // needs it per-row to avoid stripping the Resource this path @@ -1803,19 +2094,79 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro } } - err = s.store.PutGrants(ctx, grants...) + err = s.store.PutGrants(putCtx, grants...) if err != nil { return fmt.Errorf("sync-grants-for-resource: error putting grants: %w", err) } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + s.handleProgress(ctx, action, len(grants)) - if resp.GetNextPageToken() == "" { + if typeScoped { + // Cursors don't map 1:1 to resources (one cursor can cover many + // groups and may also emit cross-type grants), so the per-resource + // "N of M resources covered" accounting is meaningless here and + // would trip the "more grant resources than resources" warning on + // healthy syncs. Count raw grant rows instead. + s.counts.SetGrantsCountOnly(resourceID.GetResourceType()) + s.counts.AddGrantsProgress(resourceID.GetResourceType(), len(grants)) + s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) + } else if resp.GetNextPageToken() == "" && !action.Spawned { + // A resource counts as covered exactly once: when its ORIGIN + // action's page chain ends. Spawned sibling cursors for the same + // resource also end with an empty token but must not count, or a + // resource with N spawned pages would count N times and trip the + // progress anomaly warning on healthy syncs. s.counts.AddGrantsProgress(resourceID.GetResourceType(), 1) s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } - return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken()) + // SpawnCursors: the response may enqueue sibling cursors — each runs + // as its own action, scheduled by the worker pool, rate-limited, and + // checkpointed like any other pagination. + // + // - Type-scoped calls: one cursor per connector-defined shard (e.g. + // 50-id delta chunks); actions carry only the resource type. + // - Per-resource calls: parallel page fan-out for page-numbered APIs + // (the connector knows every page URL from page one, so a warm + // sync can revalidate all pages concurrently instead of serially); + // actions carry the resource identity. + // + // Spawned cursors are ORDINARY pages that happen to be enqueued + // eagerly: the SDK assumes nothing about replay — a spawned page may + // hit its lookup and replay, miss and fetch cold (page boundary + // shifted since last sync), and may chain further via NextPageToken. + spawn := &v2.SpawnCursors{} + hasSpawn, err := respAnnos.Pick(spawn) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: error parsing spawn-cursors annotation: %w", err) + } + var spawned []Action + if hasSpawn { + for _, tok := range spawn.GetPageTokens() { + if tok == "" { + continue + } + spawned = append(spawned, Action{ + Op: SyncGrantsOp, + ResourceTypeID: action.ResourceTypeID, + ResourceID: action.ResourceID, // empty on type-scoped actions + PageToken: tok, + Spawned: true, + }) + } + l.Debug("sync-grants-for-resource: spawned sibling grant cursors", + zap.String("resource_type_id", action.ResourceTypeID), + zap.String("resource_id", action.ResourceID), + zap.Bool("type_scoped", typeScoped), + zap.Int("cursors", len(spawned)), + zap.Int64("estimated_total", spawn.GetEstimatedTotal())) + } + + return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) } func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) error { @@ -2223,7 +2574,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi principalMap[principalID] = principal } - grantsToDelete := make([]string, 0) + grantsToDelete := make([]*v2.Grant, 0) expandedGrants := make([]*v2.Grant, 0) for ga, err := range s.store.Grants().ListWithAnnotations(ctx) { @@ -2256,7 +2607,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi newGrant := newGrantForExternalPrincipal(grant, principal) expandedGrants = append(expandedGrants, newGrant) } - grantsToDelete = append(grantsToDelete, grant.GetId()) + grantsToDelete = append(grantsToDelete, grant) continue } @@ -2332,7 +2683,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi // We still want to delete the grant even if there are no matches // Since it does not correspond to any known user - grantsToDelete = append(grantsToDelete, grant.GetId()) + grantsToDelete = append(grantsToDelete, grant) } // Match by key/val @@ -2414,7 +2765,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi } // We still want to delete the grant even if there are no matches - grantsToDelete = append(grantsToDelete, grant.GetId()) + grantsToDelete = append(grantsToDelete, grant) } } @@ -2428,11 +2779,19 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi return err } - for _, grantId := range grantsToDelete { - if newGrantIDs.ContainsOne(grantId) { + // Prefer the refs-based delete (exact structural identity) when the + // store supports it; external ids are a lossy external contract and + // stores keyed by structural identity cannot always resolve them. + refsDeleter, _ := s.store.(grantByRefsDeleter) + for _, grantToDelete := range grantsToDelete { + if newGrantIDs.ContainsOne(grantToDelete.GetId()) { continue } - err = s.store.DeleteGrant(ctx, grantId) + if refsDeleter != nil { + err = refsDeleter.DeleteGrantByRefs(ctx, grantToDelete) + } else { + err = s.store.DeleteGrant(ctx, grantToDelete.GetId()) + } if err != nil { return err } @@ -2447,6 +2806,13 @@ func userTraitContainsEmail(emails []*v2.UserTrait_Email, address string) bool { }) } +// grantByRefsDeleter is the optional store fast path for deleting a grant +// by its structured refs instead of its lossy public id (Pebble implements +// it; SQLite resolves ids by exact string and does not need it). +type grantByRefsDeleter interface { + DeleteGrantByRefs(ctx context.Context, grant *v2.Grant) error +} + func newGrantForExternalPrincipal(grant *v2.Grant, principal *v2.Resource) *v2.Grant { newGrant := v2.Grant_builder{ Entitlement: grant.GetEntitlement(), @@ -2608,6 +2974,10 @@ func (s *syncer) Close(ctx context.Context) error { var errs []error + // Detach the source-cache lookup before the stores go away so a late + // connector RPC can't read a store the syncer no longer owns. + s.clearSourceCacheLookup(ctx) + var storeCloseErr error if s.store != nil { storeCloseErr = s.store.Close(finalizeCtx) @@ -2673,7 +3043,7 @@ func WithProgressHandler(f func(s *Progress)) SyncOpt { // WithConnectorStore sets the connector store to use. This is the preferred option. // Either this or WithC1ZPath must be provided to create a new syncer. -func WithConnectorStore(store dotc1z.C1ZStore) SyncOpt { +func WithConnectorStore(store c1zstore.Store) SyncOpt { return func(s *syncer) { s.store = store } @@ -2695,7 +3065,7 @@ func WithTmpDir(path string) SyncOpt { // WithStorageEngine selects the dotc1z storage engine when opening the c1z // file via WithC1ZPath. Empty uses the baton-sdk default. -func WithStorageEngine(engine dotc1z.Engine) SyncOpt { +func WithStorageEngine(engine c1zstore.Engine) SyncOpt { return func(s *syncer) { s.storageEngine = engine } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go index d1b904f7..97c1e50b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go @@ -8,6 +8,7 @@ import ( reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" ) @@ -25,7 +26,7 @@ type Compactor struct { // Both arguments are C1ZStore; the constructor type-asserts to *dotc1z.C1File // and returns an error on mismatch. This keeps the public entry point clean // while confining the SQLite-specific concern to the attached package. -func NewAttachedCompactor(base, applied dotc1z.C1ZStore) (*Compactor, error) { +func NewAttachedCompactor(base, applied c1zstore.Store) (*Compactor, error) { baseFile, ok := dotc1z.AsSQLiteStore(base) if !ok { return nil, fmt.Errorf("attached compactor requires SQLite-backed base store, got %T", base) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go index 6a298f37..48514230 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go @@ -13,6 +13,7 @@ import ( reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/sdk" "github.com/conductorone/baton-sdk/pkg/sync" "github.com/conductorone/baton-sdk/pkg/synccompactor/attached" @@ -35,7 +36,7 @@ const ( type Compactor struct { compactorType CompactorType entries []*CompactableSync - compactedC1z dotc1z.C1ZStore + compactedC1z c1zstore.Store tmpDir string destDir string @@ -47,7 +48,7 @@ type Compactor struct { // Empty means EngineSQLite (the default; behavior is unchanged and // the output is byte-identical to the pre-engine-option compactor). // EnginePebble produces a v3 Pebble c1z via a native record merge. - engine dotc1z.Engine + engine c1zstore.Engine // pebbleMode optionally forces the Pebble merge strategy; the zero // value (Auto) lets the compactor choose. See WithPebbleCompactorMode. pebbleMode PebbleCompactorMode @@ -76,9 +77,9 @@ type Compactor struct { // resolvedEngine returns the configured engine, treating the zero value // as EngineSQLite. Compact calls inferEngineFromInputs first so the zero // value can follow existing c1z inputs instead of always producing SQLite. -func (c *Compactor) resolvedEngine() dotc1z.Engine { +func (c *Compactor) resolvedEngine() c1zstore.Engine { if c.engine == "" { - return dotc1z.EngineSQLite + return c1zstore.EngineSQLite } return c.engine } @@ -103,7 +104,7 @@ var ErrEnginePolicyConflict = errors.New("compactor: engine policy conflict: can // // Constraint: an explicit SQLite request (WithEngine(EngineSQLite)) when any // input is Pebble/v3 returns ErrEnginePolicyConflict. -func (c *Compactor) inferEngineFromInputs() (dotc1z.Engine, error) { +func (c *Compactor) inferEngineFromInputs() (c1zstore.Engine, error) { hasPebble := false hasSQLite := false for _, entry := range c.entries { @@ -134,7 +135,7 @@ func (c *Compactor) inferEngineFromInputs() (dotc1z.Engine, error) { // Explicit engine: validate and return. if c.engine != "" { - if c.engine == dotc1z.EngineSQLite && hasPebble { + if c.engine == c1zstore.EngineSQLite && hasPebble { return "", fmt.Errorf("%w: caller requested SQLite but at least one input is Pebble/v3", ErrEnginePolicyConflict) } return c.engine, nil @@ -142,13 +143,13 @@ func (c *Compactor) inferEngineFromInputs() (dotc1z.Engine, error) { // Auto-select: any Pebble input → Pebble output. if hasPebble { - return dotc1z.EnginePebble, nil + return c1zstore.EnginePebble, nil } if hasSQLite { - return dotc1z.EngineSQLite, nil + return c1zstore.EngineSQLite, nil } // No readable inputs: default to SQLite to preserve historical behavior. - return dotc1z.EngineSQLite, nil + return c1zstore.EngineSQLite, nil } type CompactableSync struct { @@ -310,7 +311,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { // and partials are merged into the base keyspace via keep-newer // writes; the folded output is then re-keyed to a fresh sync id. // The original base file is never mutated. See compactPebbleFold. - if c.resolvedEngine() == dotc1z.EnginePebble { + if c.resolvedEngine() == c1zstore.EnginePebble { c.pebbleMode = c.resolvePebbleMode(ctx) } foldMode := c.pebbleMode == PebbleCompactorModeFold @@ -336,7 +337,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { } } - if c.resolvedEngine() == dotc1z.EnginePebble { + if c.resolvedEngine() == c1zstore.EnginePebble { // One payload-decoder pool for the whole compaction: the merge // opens every source's envelope (selection + per-chunk unpack), // and reusing one decoder across those opens avoids a fresh @@ -347,10 +348,10 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { opts = append(opts, dotc1z.WithDecoderPool(c.decoderPool)) } - if c.resolvedEngine() == dotc1z.EnginePebble { + if c.resolvedEngine() == c1zstore.EnginePebble { // Force the resolved engine last so a stray engine passed via // WithC1ZOptions cannot mislabel the artifact. - c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(dotc1z.EnginePebble))...) + c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(c1zstore.EnginePebble))...) } else { c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, opts...) } @@ -380,7 +381,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { } return nil, fmt.Errorf("failed to compact (pebble fold): %w", err) } - case c.resolvedEngine() == dotc1z.EnginePebble: + case c.resolvedEngine() == c1zstore.EnginePebble: newSyncId, err = c.runPebbleRebuild(ctx, runCtx) if err != nil { if cause := context.Cause(runCtx); errors.Is(cause, context.DeadlineExceeded) && c.runDuration > 0 && ctx.Err() == nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go index 6258309f..00fc64a4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go @@ -19,6 +19,7 @@ import ( v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" enginepkg "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" mergepkg "github.com/conductorone/baton-sdk/pkg/synccompactor/pebble" @@ -33,7 +34,7 @@ import ( // This is the only supported way to choose the engine; an engine // passed through WithC1ZOptions does not select the compaction // strategy and is overridden. -func WithEngine(engine dotc1z.Engine) Option { +func WithEngine(engine c1zstore.Engine) Option { return func(c *Compactor) { c.engine = engine } @@ -339,7 +340,7 @@ func fileSizeOrZero(path string) int64 { // pre-static-registration era. Pebble is now registered by dotc1z init, so this // is a cheap sanity check. func ensurePebbleRegistered() error { - if _, ok := dotc1z.EngineDriverFor(dotc1z.EnginePebble); ok { + if _, ok := dotc1z.EngineDriverFor(c1zstore.EnginePebble); ok { return nil } return dotc1z.ErrEngineNotAvailable @@ -792,6 +793,26 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { return errors.New("compactPebble: compacted store is not a pebble engine") } + // runPebbleRebuild's StartNewSync→EndSync left the dest engine SEALED + // (writes refused, compactions paused). The merge below writes the whole + // compacted dataset — overlay mode through raw memtable batches — so + // bind the sync first: unseals the engine and resumes the compaction + // scheduler. Without this, L0 accumulates with no compactions granted + // until pebble stalls writes at L0StopWritesThreshold, permanently + // (nothing else resumes the scheduler mid-merge). + // + // Yes, "SetCurrentSync to restart compactions" is an odd spelling. It + // is deliberate: unseal/resume is not a public engine operation, because + // the sealed state exists precisely to guarantee "no record writes + // without a bound sync". Binding the sync we're about to write under is + // the one sanctioned way to declare that intent, and unseal+resume ride + // along as consequences (see Engine.SetCurrentSync / Engine.seal). An + // exported ResumeCompactions-style escape hatch would let callers write + // on a sealed engine again, recreating the very hang this fixes. + if err := destEng.SetCurrentSync(newSyncId); err != nil { + return fmt.Errorf("compactPebble: bind dest sync: %w", err) + } + pebbleCompactorMode := c.pebbleMode if pebbleCompactorMode == PebbleCompactorModeAuto { pebbleCompactorMode = c.resolvePebbleMode(ctx) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go index 7c29f484..2cd44d8e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go @@ -50,31 +50,44 @@ func buildBucketPlans() []bucketPlan { lower: enginepkg.GrantLowerBound(), upper: enginepkg.GrantUpperBound(), }, - { - name: "grant_by_entitlement", - lower: enginepkg.GrantByEntitlementLowerBound(), - upper: enginepkg.GrantByEntitlementUpperBound(), - }, - { - name: "grant_by_entitlement_resource", - lower: enginepkg.GrantByEntitlementResourceLowerBound(), - upper: enginepkg.GrantByEntitlementResourceUpperBound(), - }, { name: "grant_by_principal", lower: enginepkg.GrantByPrincipalLowerBound(), upper: enginepkg.GrantByPrincipalUpperBound(), }, - { - name: "grant_by_principal_resource_type", - lower: enginepkg.GrantByPrincipalResourceTypeLowerBound(), - upper: enginepkg.GrantByPrincipalResourceTypeUpperBound(), - }, { name: "grant_by_needs_expansion", lower: enginepkg.GrantByNeedsExpansionLowerBound(), upper: enginepkg.GrantByNeedsExpansionUpperBound(), }, + // Source-cache state MUST fold with its rows. Excising these + // ranges alongside the record buckets replaces the base sync's + // manifest/index entries with the applied sync's: scopes the + // applied sync touched keep a consistent (entry, rows) pair, and + // scopes it never touched become clean lookup misses. Leaving + // them out would strand the BASE sync's validators and index + // entries on top of the applied sync's rows — a stale manifest + // hit could then replay rows that no longer belong to the scope. + { + name: "grant_by_source_scope", + lower: enginepkg.GrantBySourceScopeLowerBound(), + upper: enginepkg.GrantBySourceScopeUpperBound(), + }, + { + name: "entitlement_by_source_scope", + lower: enginepkg.EntitlementBySourceScopeLowerBound(), + upper: enginepkg.EntitlementBySourceScopeUpperBound(), + }, + { + name: "resource_by_source_scope", + lower: enginepkg.ResourceBySourceScopeLowerBound(), + upper: enginepkg.ResourceBySourceScopeUpperBound(), + }, + { + name: "source_cache_entry", + lower: enginepkg.SourceCacheEntryLowerBound(), + upper: enginepkg.SourceCacheEntryUpperBound(), + }, { name: "asset", lower: enginepkg.AssetLowerBound(), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/compactor.go index b05f4076..4cfa720c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/compactor.go @@ -67,10 +67,9 @@ func NewCompactor(base *enginepkg.Engine, tmpDir string) (*Compactor, error) { // source's view of that sync. // // Atomicity: per-bucket atomic, NOT whole-Compact atomic. Each -// IngestAndExcise call (one per record-type bucket: grants, -// by_entitlement index, by_principal index) is atomic from base's -// perspective — a concurrent reader sees the old-or-new state of that -// bucket, never a mixture. However, the multi-bucket loop is NOT +// IngestAndExcise call (one per record-type bucket: grants, by_principal index, +// etc.) is atomic from base's perspective — a concurrent reader sees the +// old-or-new state of that bucket, never a mixture. However, the multi-bucket loop is NOT // transactional as a whole: a crash or hard cancellation mid-loop // leaves base with new data in some buckets and old data in others // (and the same for the DeleteRange-only path used for empty @@ -97,6 +96,10 @@ func (c *Compactor) Compact(ctx context.Context, source *enginepkg.Engine, syncI if dstDB == nil { return errors.New("synccompactor/pebble.Compact: base engine has no DB (closed?)") } + // This function writes the base keyspace through the raw DB handle; + // invalidate the engine's bare-id lookup state on the way out (even on + // error — earlier buckets may already have been replaced). + defer c.base.InvalidateBareIDLookups() // The full key range belonging to the file's one sync, across all // record types and indexes. v3 keys carry no sync_id, so each diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/kway.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/kway.go index 91e76226..787029aa 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/kway.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/kway.go @@ -103,6 +103,9 @@ func mergeBucketsInto(ctx context.Context, dest *enginepkg.Engine, sources []Sou if cfg.fanIn < 2 { cfg.fanIn = defaultKWayFanIn } + // Every bucket lands through the raw DB handle; invalidate the dest + // engine's bare-id lookup state on the way out (even on error). + defer dest.InvalidateBareIDLookups() l := ctxzap.Extract(ctx) mergeStart := time.Now() direct := len(sources) <= cfg.fanIn @@ -490,7 +493,7 @@ func entitlementBucket() bucketSpec { syncRange: func() ([]byte, []byte) { return enginepkg.EntitlementLowerBound(), enginepkg.EntitlementUpperBound() }, - key: func(m proto.Message) string { return m.(*v3.EntitlementRecord).GetExternalId() }, + key: func(m proto.Message) string { return enginepkg.EntitlementRecordIdentityKey(m.(*v3.EntitlementRecord)) }, ts: func(m proto.Message) *timestamppb.Timestamp { return m.(*v3.EntitlementRecord).GetDiscoveredAt() }, indexKeys: func(m proto.Message) [][]byte { return enginepkg.EntitlementIndexKeys(m.(*v3.EntitlementRecord)) @@ -509,7 +512,7 @@ func grantBucket() bucketSpec { syncRange: func() ([]byte, []byte) { return enginepkg.GrantLowerBound(), enginepkg.GrantUpperBound() }, - key: func(m proto.Message) string { return m.(*v3.GrantRecord).GetExternalId() }, + key: func(m proto.Message) string { return enginepkg.GrantRecordIdentityKey(m.(*v3.GrantRecord)) }, ts: func(m proto.Message) *timestamppb.Timestamp { return m.(*v3.GrantRecord).GetDiscoveredAt() }, indexKeys: func(m proto.Message) [][]byte { return enginepkg.GrantIndexKeys(m.(*v3.GrantRecord)) @@ -752,6 +755,7 @@ func materializeSourceBucketToPebble( if err := dest.DB().Ingest(ctx, []string{primaryPath}); err != nil { return fmt.Errorf("ingest direct primary %s: %w", bucket.name, err) } + dest.InvalidateBareIDLookups() return indexWriters.closeSortAndIngest(ctx, dest) } @@ -1259,6 +1263,7 @@ func materializeRunFileBucket(ctx context.Context, dest *enginepkg.Engine, tmpDi if err := dest.DB().Ingest(ctx, []string{primaryPath}); err != nil { return fmt.Errorf("ingest primary %s: %w", bucket.name, err) } + dest.InvalidateBareIDLookups() return indexWriters.closeSortAndIngest(ctx, dest) } @@ -1379,6 +1384,7 @@ func (w *indexRunWriter) closeSortAndIngest(ctx context.Context, dest *enginepkg if err := dest.DB().Ingest(ctx, []string{sstPath}); err != nil { return fmt.Errorf("ingest index %s: %w", w.name, err) } + dest.InvalidateBareIDLookups() return nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/merge.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/merge.go index e5a94920..56b0a8e4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/merge.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/merge.go @@ -81,6 +81,10 @@ func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync if err := dest.SetCurrentSync(destSyncID); err != nil { return stats, fmt.Errorf("synccompactor/pebble.MergeInto: bind dest sync: %w", err) } + // Folds write the dest keyspace through the raw DB handle; invalidate + // the engine's bare-id lookup state on the way out (even on error — + // earlier sources may already have been folded in). + defer dest.InvalidateBareIDLookups() for i := range sources { if err := ctx.Err(); err != nil { return stats, err diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/overlay.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/overlay.go index 16077776..392b83f6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/overlay.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/overlay.go @@ -272,22 +272,72 @@ func overlayPreGateFires(seen *seenSuffixSet, srcStats *reader_v2.SyncStats, buc // holds one sync and keys carry no sync_id, so each is one contiguous // range. func bucketIndexRanges(bucket bucketSpec) [][2][]byte { + ranges := make([][2][]byte, 0, len(bucketIndexCopySpecs(bucket))) + for _, spec := range bucketIndexCopySpecs(bucket) { + ranges = append(ranges, spec.bounds) + } + return ranges +} + +// indexFamilyCopySpec describes how one secondary-index family participates in +// the whole-source copy: its key range, whether copied entries must be +// filtered against the primary seen set, and — when filtering — how many +// trailing tuple elements of the index key reconstruct the record's +// primary-key suffix (the exact bytes the seen set hashed at admission). +type indexFamilyCopySpec struct { + bounds [2][]byte + // filter=false families are copied verbatim: their index key is a pure + // function of the record's structural identity, so a "seen" identity + // implies the admitted winner produces the byte-identical index key — + // the copy is an exact duplicate Set, never a stale entry. + filter bool + elems int +} + +func bucketIndexCopySpecs(bucket bucketSpec) []indexFamilyCopySpec { switch bucket.id { case runBucketResources: - return [][2][]byte{ - {enginepkg.ResourceByParentLowerBound(), enginepkg.ResourceByParentUpperBound()}, - } + // by_parent tail is (parent_rt, parent_id, child_rt, child_id); + // the resource primary suffix is the trailing (child_rt, child_id). + return []indexFamilyCopySpec{{ + bounds: [2][]byte{enginepkg.ResourceByParentLowerBound(), enginepkg.ResourceByParentUpperBound()}, + filter: true, + elems: 2, + }} case runBucketEntitlements: - return [][2][]byte{ - {enginepkg.EntitlementByResourceLowerBound(), enginepkg.EntitlementByResourceUpperBound()}, - } + // Retired family: the structured-identity migration deletes this + // range, so it is empty on every source this copy can see. Copied + // verbatim (i.e. not at all) for symmetry. + return []indexFamilyCopySpec{{ + bounds: [2][]byte{enginepkg.EntitlementByResourceLowerBound(), enginepkg.EntitlementByResourceUpperBound()}, + filter: false, + }} case runBucketGrants: - return [][2][]byte{ - {enginepkg.GrantByEntitlementLowerBound(), enginepkg.GrantByEntitlementUpperBound()}, - {enginepkg.GrantByEntitlementResourceLowerBound(), enginepkg.GrantByEntitlementResourceUpperBound()}, - {enginepkg.GrantByPrincipalLowerBound(), enginepkg.GrantByPrincipalUpperBound()}, - {enginepkg.GrantByPrincipalResourceTypeLowerBound(), enginepkg.GrantByPrincipalResourceTypeUpperBound()}, - {enginepkg.GrantByNeedsExpansionLowerBound(), enginepkg.GrantByNeedsExpansionUpperBound()}, + return []indexFamilyCopySpec{ + { + // by_principal permutes the identity components + // (prt, pid, rt, rid, flag, tail): a trailing-element walk + // CANNOT reconstruct the primary suffix. It doesn't need + // to — the key is a pure function of the grant identity, + // so filtered (seen) identities would produce the same + // bytes the winner's own index entry carries. Copy + // verbatim. + bounds: [2][]byte{enginepkg.GrantByPrincipalLowerBound(), enginepkg.GrantByPrincipalUpperBound()}, + filter: false, + }, + { + // by_needs_expansion tail IS the grant primary tail + // (rt, rid, flag, tail, prt, pid): the whole 6-element + // suffix reconstructs the primary suffix byte-for-byte. + // This one MUST filter: the flag is record state, not + // identity — a newer source can admit the same identity + // with needs_expansion=false, and copying the older + // source's entry verbatim would bake a phantom + // pending-expansion row into the artifact. + bounds: [2][]byte{enginepkg.GrantByNeedsExpansionLowerBound(), enginepkg.GrantByNeedsExpansionUpperBound()}, + filter: true, + elems: 6, + }, } default: return nil @@ -366,6 +416,9 @@ func MergeFilesIntoOverlay(ctx context.Context, dest *enginepkg.Engine, sources for _, opt := range opts { opt(&cfg) } + // Overlay writers commit batches through the raw DB handle; invalidate + // the dest engine's bare-id lookup state on the way out (even on error). + defer dest.InvalidateBareIDLookups() hardLimit := cfg.hardLimit() gateThreshold := cfg.gateThreshold() buckets := allBuckets() @@ -1091,6 +1144,7 @@ func overlayMaterializeWholeSourceBucketSST( if err := dest.DB().Ingest(ctx, []string{primaryPath}); err != nil { return fmt.Errorf("overlay merge: ingest whole primary %s: %w", bucket.name, err) } + dest.InvalidateBareIDLookups() } return overlayCopySourceIndexesSST(ctx, dest, source, bucket, seen, tmpDir) } @@ -1145,19 +1199,15 @@ func overlayCopySourceIndexesSST( seen *seenSuffixSet, tmpDir string, ) error { - ranges := bucketIndexRanges(bucket) - if len(ranges) == 0 { + specs := bucketIndexCopySpecs(bucket) + if len(specs) == 0 { return nil } // SST keys must be appended in ascending order; the family ranges // are disjoint, so ordering them by lower bound suffices. - sort.Slice(ranges, func(i, j int) bool { - return bytes.Compare(ranges[i][0], ranges[j][0]) < 0 + sort.Slice(specs, func(i, j int) bool { + return bytes.Compare(specs[i].bounds[0], specs[j].bounds[0]) < 0 }) - suffixElems := 1 - if bucket.id == runBucketResources { - suffixElems = 2 - } sstPath := filepath.Join(tmpDir, fmt.Sprintf("overlay-whole-%s-index.sst", bucket.name)) w, err := newSSTBuilder(sstPath) if err != nil { @@ -1170,10 +1220,10 @@ func overlayCopySourceIndexesSST( } _ = os.Remove(sstPath) }() - checkSeen := seen.size() > 0 wrote := false - for _, r := range ranges { - if err := copyIndexRangeFiltered(ctx, source, r[0], r[1], checkSeen, seen, suffixElems, w, &wrote); err != nil { + for _, spec := range specs { + checkSeen := spec.filter && seen.size() > 0 + if err := copyIndexRangeFiltered(ctx, source, spec.bounds[0], spec.bounds[1], checkSeen, seen, spec.elems, w, &wrote); err != nil { return err } } @@ -1187,6 +1237,7 @@ func overlayCopySourceIndexesSST( if err := dest.DB().Ingest(ctx, []string{sstPath}); err != nil { return fmt.Errorf("overlay merge: ingest whole index %s: %w", bucket.name, err) } + dest.InvalidateBareIDLookups() return nil } @@ -1272,76 +1323,33 @@ func forEachIndexKeyFromRaw( scratch.key = enginepkg.AppendResourceIndexKeyRawBytes(scratch.key[:0], parentRT, parentID, rt, id) return emit(scratch.key) case runBucketEntitlements: - ext, err := decodePrimaryTailBytes1(destKey, destLower, scratch) - if err != nil { - return err - } - resourceRT, resourceID, err := scanEntitlementResourceBytes(value) + resourceRT, _, err := scanEntitlementResourceBytes(value) if err != nil { return err } stats.groupEntitlement(resourceRT) - if len(resourceID) == 0 { - return nil - } - scratch.key = enginepkg.AppendEntitlementIndexKeyRawBytes(scratch.key[:0], resourceRT, resourceID, ext) - return emit(scratch.key) + return nil case runBucketGrants: - ext, err := decodePrimaryTailBytes1(destKey, destLower, scratch) - if err != nil { - return err - } entRT, entRID, entID, principalRT, principalID, needsExpansion, err := scanGrantIndexFieldsBytes(value) if err != nil { return err } stats.groupGrant(entRT) - if len(entID) != 0 && len(principalRT) != 0 && len(principalID) != 0 { - scratch.key = enginepkg.AppendGrantByEntitlementIndexKeyRawBytes(scratch.key[:0], entID, principalRT, principalID, ext) - if err := emit(scratch.key); err != nil { - return err - } - } - if len(entRID) != 0 { - scratch.key = enginepkg.AppendGrantByEntitlementResourceIndexKeyRawBytes(scratch.key[:0], entRT, entRID, ext) - if err := emit(scratch.key); err != nil { - return err - } - } - if len(principalRT) != 0 && len(principalID) != 0 { - scratch.key = enginepkg.AppendGrantByPrincipalIndexKeyRawBytes(scratch.key[:0], principalRT, principalID, ext) - if err := emit(scratch.key); err != nil { - return err - } - scratch.key = enginepkg.AppendGrantByPrincipalResourceTypeIndexKeyRawBytes(scratch.key[:0], principalRT, ext) - if err := emit(scratch.key); err != nil { - return err - } - } - if needsExpansion { - scratch.key = enginepkg.AppendGrantByNeedsExpansionIndexKeyRawBytes(scratch.key[:0], ext) - if err := emit(scratch.key); err != nil { - return err - } - } - return nil + return enginepkg.ForEachGrantIndexKeyRaw( + string(entRT), + string(entRID), + string(entID), + string(principalRT), + string(principalID), + "", + needsExpansion, + emit, + ) default: return nil } } -func decodePrimaryTailBytes1(key []byte, lower []byte, scratch *rawIndexScratch) ([]byte, error) { - tail, err := primaryTail(key, lower) - if err != nil { - return nil, err - } - scratch.tail1, _, err = codec.DecodeTupleStringTo(scratch.tail1[:0], tail, 0) - if err != nil { - return nil, err - } - return scratch.tail1, nil -} - func decodePrimaryTailBytes2(key []byte, lower []byte, scratch *rawIndexScratch) ([]byte, []byte, error) { tail, err := primaryTail(key, lower) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go index 7b4c56d8..ed2fe9fc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go @@ -16,7 +16,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" "github.com/conductorone/baton-sdk/pkg/annotations" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" "github.com/conductorone/baton-sdk/pkg/session" sdkSync "github.com/conductorone/baton-sdk/pkg/sync" @@ -42,7 +42,7 @@ type fullSyncTaskHandler struct { targetedSyncResources []*v2.Resource syncResourceTypeIDs []string workerCount int - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine // previousSyncSparePath is the connector's ETag-replay opt-in: when // non-empty, the handler retains one spare c1z (the last successfully @@ -160,7 +160,7 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { } engine := c.storageEngine if engine == "" && c.task.GetSyncFull().GetStorageEngine() != "" { - engine = dotc1z.Engine(c.task.GetSyncFull().GetStorageEngine()) + engine = c1zstore.Engine(c.task.GetSyncFull().GetStorageEngine()) } if engine != "" { syncOpts = append(syncOpts, sdkSync.WithStorageEngine(engine)) @@ -359,7 +359,7 @@ func newFullSyncTaskHandler( targetedSyncResources []*v2.Resource, syncResourceTypeIDs []string, workerCount int, - storageEngine dotc1z.Engine, + storageEngine c1zstore.Engine, previousSyncSparePath string, ) tasks.TaskHandler { return &fullSyncTaskHandler{ diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go index 5ed25a90..c0df2348 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go @@ -14,7 +14,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/conductorone/baton-sdk/pkg/annotations" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" "github.com/conductorone/baton-sdk/pkg/uotel/uotelzap" @@ -68,7 +68,7 @@ type c1ApiTaskManager struct { targetedSyncResources []*v2.Resource syncResourceTypeIDs []string workerCount int - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine // previousSyncSparePath is non-empty when the connector opted into // ETag replay (keepPreviousSyncC1Z): the fixed, client-id-namespaced @@ -500,7 +500,7 @@ func NewC1TaskManager( targetedSyncResources []*v2.Resource, syncResourceTypeIDs []string, workerCount int, - storageEngine dotc1z.Engine, + storageEngine c1zstore.Engine, taskConcurrency int, keepPreviousSyncC1Z bool, ) (BootstrappingTaskManager, error) { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go index 2c55874e..f9ee3c04 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go @@ -6,7 +6,7 @@ import ( "time" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/synccompactor" "github.com/conductorone/baton-sdk/pkg/tasks" "github.com/conductorone/baton-sdk/pkg/types" @@ -23,12 +23,12 @@ type localCompactor struct { compactableSyncs []*synccompactor.CompactableSync outputPath string tmpDir string - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine } type CompactorOption func(*localCompactor) -func WithCompactorStorageEngine(engine dotc1z.Engine) CompactorOption { +func WithCompactorStorageEngine(engine c1zstore.Engine) CompactorOption { return func(m *localCompactor) { m.storageEngine = engine } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go index 137be49e..916b166d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go @@ -10,7 +10,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/session" sdkSync "github.com/conductorone/baton-sdk/pkg/sync" "github.com/conductorone/baton-sdk/pkg/tasks" @@ -24,13 +24,14 @@ type localSyncer struct { o sync.Once tmpDir string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string targetedSyncResources []*v2.Resource skipEntitlementsAndGrants bool skipGrants bool syncResourceTypeIDs []string workerCount int - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine } type Option func(*localSyncer) @@ -47,6 +48,12 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(m *localSyncer) { + m.previousSyncC1Z = previousSyncC1Z + } +} + func WithExternalResourceEntitlementIdFilter(entitlementId string) Option { return func(m *localSyncer) { m.externalResourceEntitlementIdFilter = entitlementId @@ -83,7 +90,7 @@ func WithWorkerCount(workerCount int) Option { } } -func WithStorageEngine(engine dotc1z.Engine) Option { +func WithStorageEngine(engine c1zstore.Engine) Option { return func(m *localSyncer) { m.storageEngine = engine } @@ -122,6 +129,7 @@ func (m *localSyncer) Process(ctx context.Context, task *v1.Task, cc types.Conne sdkSync.WithC1ZPath(m.dbPath), sdkSync.WithTmpDir(m.tmpDir), sdkSync.WithExternalResourceC1ZPath(m.externalResourceC1Z), + sdkSync.WithPreviousSyncC1ZPath(m.previousSyncC1Z), sdkSync.WithExternalResourceEntitlementIdFilter(m.externalResourceEntitlementIdFilter), sdkSync.WithTargetedSyncResources(m.targetedSyncResources), sdkSync.WithSkipEntitlementsAndGrants(m.skipEntitlementsAndGrants), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/entitlement/entitlement.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/entitlement/entitlement.go index 37485e69..3515bcb6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/entitlement/entitlement.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/entitlement/entitlement.go @@ -76,6 +76,12 @@ func WithExclusionGroupDefault(exclusionGroupID string, order uint32) Entitlemen }) } +// NewEntitlementID returns the public entitlement id for a resource and +// permission. The raw ":"-join is an external-consumer contract: ids must +// stay byte-identical across SDK versions (C1 carries configuration keyed on +// them), so this join is deliberately lossy and MUST NOT be used as an +// internal identity — storage keys derive from the structured resource +// fields instead (see pkg/dotc1z/engine/pebble/identity.go). func NewEntitlementID(resource *v2.Resource, permission string) string { return fmt.Sprintf("%s:%s:%s", resource.GetId().GetResourceType(), resource.GetId().GetResource(), permission) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/grant/grant.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/grant/grant.go index b472ca99..230784ba 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/grant/grant.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/grant/grant.go @@ -20,6 +20,34 @@ type GrantPrincipal interface { // Sometimes C1 doesn't have the grant ID, but does have the principal and entitlement. const UnknownGrantId string = "🧸_UNKNOWN_GRANT_ID" +// Source records one source entitlement's contribution to an expanded grant. +type Source struct { + EntitlementID string + IsDirect bool +} + +type Sources []Source + +func (s Sources) ToBoolMap() map[string]bool { + if len(s) == 0 { + return nil + } + out := make(map[string]bool, len(s)) + for _, src := range s { + out[src.EntitlementID] = src.IsDirect + } + return out +} + +func (s Sources) DirectFor(entitlementID string) bool { + for _, src := range s { + if src.EntitlementID == entitlementID { + return src.IsDirect + } + } + return false +} + func WithGrantMetadata(metadata map[string]interface{}) GrantOption { return func(g *v2.Grant) error { md, err := structpb.NewStruct(metadata) @@ -94,6 +122,13 @@ func NewGrant(resource *v2.Resource, entitlementName string, principal GrantPrin return grant } +// NewGrantID returns the public grant id for a principal on an entitlement. +// The raw ":"-join is an external-consumer contract: ids must stay +// byte-identical across SDK versions (C1 carries configuration keyed on +// them), so this join is deliberately lossy and MUST NOT be used as an +// internal identity — storage keys derive from the structured +// entitlement/principal refs instead (see +// pkg/dotc1z/engine/pebble/identity.go). func NewGrantID(principal GrantPrincipal, entitlement *v2.Entitlement) string { var resourceID *v2.ResourceId switch p := principal.(type) { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go index 16585ed3..49a01e0f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go @@ -8,6 +8,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/sessions" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -524,9 +525,14 @@ func NewManagedDeviceResource( } type SyncOpAttrs struct { - Session sessions.SessionStore - SyncID string - PageToken pagination.Token + Session sessions.SessionStore + // SourceCache resolves a scope's previous-sync validator (etag / + // delta token) for source-cache replay. Never nil for framework-built + // connectors: when source cache is disabled or degraded the SDK + // supplies sourcecache.NoopLookup, which always misses. + SourceCache sourcecache.Lookup + SyncID string + PageToken pagination.Token } type SyncOpResults struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 624e6b05..d385633d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -266,7 +266,7 @@ github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-sdk v0.17.0 +# github.com/conductorone/baton-sdk v0.17.0 => /Users/matt.kaniaris/work/baton-sdk-2 ## explicit; go 1.25.2 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1 @@ -287,6 +287,7 @@ github.com/conductorone/baton-sdk/pkg/bid github.com/conductorone/baton-sdk/pkg/cli github.com/conductorone/baton-sdk/pkg/config github.com/conductorone/baton-sdk/pkg/connectorbuilder +github.com/conductorone/baton-sdk/pkg/connectorclient github.com/conductorone/baton-sdk/pkg/connectorrunner github.com/conductorone/baton-sdk/pkg/connectorstore github.com/conductorone/baton-sdk/pkg/crypto @@ -310,6 +311,7 @@ github.com/conductorone/baton-sdk/pkg/ratelimit github.com/conductorone/baton-sdk/pkg/retry github.com/conductorone/baton-sdk/pkg/sdk github.com/conductorone/baton-sdk/pkg/session +github.com/conductorone/baton-sdk/pkg/sourcecache github.com/conductorone/baton-sdk/pkg/sync github.com/conductorone/baton-sdk/pkg/sync/expand github.com/conductorone/baton-sdk/pkg/sync/expand/scc @@ -1015,3 +1017,4 @@ modernc.org/memory modernc.org/sqlite modernc.org/sqlite/lib modernc.org/sqlite/vtab +# github.com/conductorone/baton-sdk => /Users/matt.kaniaris/work/baton-sdk-2