Skip to content

Implement centralized TSO phase D #1114

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bootjp wants to merge 1 commit into
base: design/dedicated-tso-fsm
Choose a base branch
from
Open

Implement centralized TSO phase D #1114

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 108 additions & 21 deletions adapter/distribution_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,18 +147,15 @@ func (s *DistributionServer) GetTimestamp(ctx context.Context, req *pb.GetTimest
if err != nil {
return nil, err
}
activateCutover := req != nil && req.GetActivateCutover()
activateCutover, activatePhaseD, err := timestampActivationValues(req)
if err != nil {
return nil, err
}
if s.timestampAllocator == nil {
if count != 1 || minTimestamp != 0 || activateCutover {
return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "dedicated TSO allocator is not configured"))
}
if s.engine == nil {
return nil, errors.WithStack(status.Error(codes.Unavailable, "distribution engine is not configured"))
}
return &pb.GetTimestampResponse{Timestamp: s.engine.NextTimestamp()}, nil
return s.legacyTimestampResponse(count, minTimestamp, activateCutover, activatePhaseD)
}

reservation, err := s.allocateTimestampReservation(ctx, count, minTimestamp, activateCutover)
reservation, err := s.allocateTimestampReservation(ctx, count, minTimestamp, activateCutover, activatePhaseD)
if err != nil {
return nil, timestampRPCError(err)
}
Expand All @@ -171,22 +168,79 @@ func (s *DistributionServer) GetTimestamp(ctx context.Context, req *pb.GetTimest
Count: uint32(count), //nolint:gosec // count is bounded by MaxTSOBatchSize.
PreviousAllocationFloor: reservation.PreviousAllocationFloor,
CutoverActive: reservation.CutoverActive,
PhaseDActive: reservation.PhaseDActive,
PhaseDFloor: reservation.PhaseDFloor,
}, nil
}

func timestampActivationValues(req *pb.GetTimestampRequest) (bool, bool, error) {
activateCutover := req != nil && req.GetActivateCutover()
activatePhaseD := req != nil && req.GetActivatePhaseD()
if activatePhaseD && !activateCutover {
return false, false, errors.WithStack(status.Error(codes.InvalidArgument,
"phase D activation requires cutover activation"))
}
return activateCutover, activatePhaseD, nil
}

func (s *DistributionServer) legacyTimestampResponse(
count int,
minTimestamp uint64,
activateCutover bool,
activatePhaseD bool,
) (*pb.GetTimestampResponse, error) {
if count != 1 || minTimestamp != 0 || activateCutover || activatePhaseD {
return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "dedicated TSO allocator is not configured"))
}
if s.engine == nil {
return nil, errors.WithStack(status.Error(codes.Unavailable, "distribution engine is not configured"))
}
return &pb.GetTimestampResponse{Timestamp: s.engine.NextTimestamp()}, nil
}

// ValidateTimestamp verifies a read/start timestamp against the durable M7
// allocation range. The local allocator performs the group-0 leader fence;
// followers reject so clients re-resolve rather than validating stale state.
func (s *DistributionServer) ValidateTimestamp(
ctx context.Context,
req *pb.ValidateTimestampRequest,
) (*pb.ValidateTimestampResponse, error) {
if req == nil || req.GetTimestamp() == 0 {
return nil, errors.WithStack(status.Error(codes.InvalidArgument, "timestamp is required"))
}
validator, ok := s.timestampAllocator.(kv.DurableTimestampValidator)
if !ok {
return nil, errors.WithStack(status.Error(codes.FailedPrecondition,
"dedicated TSO timestamp validation is not configured"))
}
if err := validator.ValidateDurableTimestamp(ctx, req.GetTimestamp()); err != nil {
return nil, timestampRPCError(err)
}
resp := &pb.ValidateTimestampResponse{Valid: true, PhaseDActive: true}
if state, ok := s.timestampAllocator.(interface {
PhaseDFloor() uint64
AllocationFloor() uint64
}); ok {
resp.PhaseDFloor = state.PhaseDFloor()
resp.AllocationFloor = state.AllocationFloor()
}
return resp, nil
}

func (s *DistributionServer) allocateTimestampReservation(
ctx context.Context,
count int,
minTimestamp uint64,
activateCutover bool,
activatePhaseD bool,
) (kv.TSOReservation, error) {
if allocator, ok := s.timestampAllocator.(kv.TSOReservationAllocator); ok {
reservation, err := allocator.ReserveBatchAfter(ctx, count, minTimestamp, activateCutover)
reservation, err := allocator.ReserveBatchAfter(ctx, count, minTimestamp, activateCutover, activatePhaseD)
return reservation, errors.Wrap(err, "reserve dedicated TSO window")
}
reservation := kv.TSOReservation{Count: count}
switch {
case activateCutover:
case activateCutover || activatePhaseD:
return reservation, errors.WithStack(status.Error(codes.FailedPrecondition,
"TSO allocator does not support durable cutover"))
case minTimestamp > 0:
Expand Down Expand Up @@ -251,6 +305,12 @@ func timestampRPCError(err error) error {
return errors.WithStack(status.Error(codes.InvalidArgument, err.Error()))
case errors.Is(err, kv.ErrTSONotLeader), errors.Is(err, kv.ErrLeaderNotFound):
return errors.WithStack(status.Error(codes.FailedPrecondition, err.Error()))
case errors.Is(err, kv.ErrTSOTimestampPrePhaseD):
return errors.WithStack(status.Error(codes.OutOfRange, err.Error()))
case errors.Is(err, kv.ErrTSOTimestampInvalid):
return errors.WithStack(status.Error(codes.InvalidArgument, err.Error()))
case errors.Is(err, kv.ErrTSOPhaseDInactive):
return errors.WithStack(status.Error(codes.FailedPrecondition, err.Error()))
default:
return errors.WithStack(status.Error(codes.Unavailable, err.Error()))
}
Expand Down Expand Up @@ -280,7 +340,16 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR
return nil, err
}

snapshot, err := s.loadCatalogSnapshot(ctx)
readTimestamp, err := kv.BeginReadTimestampThrough(
ctx,
s.coordinator,
s.catalog.LatestCommitTS(),
"distribution split range: begin read timestamp",
)
if err != nil {
return nil, grpcStatusErrorf(codes.Internal, "begin catalog split snapshot: %v", err)
}
snapshot, err := s.loadCatalogSnapshotAt(ctx, readTimestamp.Timestamp())
if err != nil {
return nil, err
}
Expand All @@ -290,15 +359,8 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR
return nil, err
}

parent, found := findRouteByID(snapshot.Routes, req.GetRouteId())
if !found {
return nil, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error())
}

rawSplitKey := req.GetSplitKey()
splitKey := distribution.CloneBytes(fskeys.NormalizeSplitBoundary(kv.RouteKey(rawSplitKey)))
if err := validateSplitKey(parent, splitKey); err != nil {
s.observeFilePinnedHotspotIfNeeded(rawSplitKey, splitKey, err)
parent, splitKey, err := s.prepareSplitRange(snapshot.Routes, req)
if err != nil {
return nil, err
}

Expand Down Expand Up @@ -327,6 +389,20 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR
}, nil
}

func (s *DistributionServer) prepareSplitRange(routes []distribution.RouteDescriptor, req *pb.SplitRangeRequest) (distribution.RouteDescriptor, []byte, error) {
parent, found := findRouteByID(routes, req.GetRouteId())
if !found {
return distribution.RouteDescriptor{}, nil, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error())
}
rawSplitKey := req.GetSplitKey()
splitKey := distribution.CloneBytes(fskeys.NormalizeSplitBoundary(kv.RouteKey(rawSplitKey)))
if err := validateSplitKey(parent, splitKey); err != nil {
s.observeFilePinnedHotspotIfNeeded(rawSplitKey, splitKey, err)
return distribution.RouteDescriptor{}, nil, err
}
return parent, splitKey, nil
}

func (s *DistributionServer) pinReadTS(ts uint64) *kv.ActiveTimestampToken {
if s == nil || s.readTracker == nil {
return nil
Expand Down Expand Up @@ -479,6 +555,17 @@ func (s *DistributionServer) loadCatalogSnapshot(ctx context.Context) (distribut
return snapshot, nil
}

func (s *DistributionServer) loadCatalogSnapshotAt(ctx context.Context, readTS uint64) (distribution.CatalogSnapshot, error) {
if s.catalog == nil {
return distribution.CatalogSnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error())
}
snapshot, err := s.catalog.SnapshotAt(ctx, readTS)
if err != nil {
return distribution.CatalogSnapshot{}, grpcStatusErrorf(codes.Internal, "load route catalog: %v", err)
}
return snapshot, nil
}

func (s *DistributionServer) loadCatalogSnapshotAtVersion(
ctx context.Context,
readTS uint64,
Expand Down
Loading
Loading