diff --git a/lib/guest/clock.go b/lib/guest/clock.go new file mode 100644 index 00000000..af098e16 --- /dev/null +++ b/lib/guest/clock.go @@ -0,0 +1,106 @@ +package guest + +import ( + "context" + "fmt" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "go.opentelemetry.io/otel/attribute" + otelcodes "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// SyncClockOptions configures a guest clock sync +type SyncClockOptions struct { + WaitForAgent time.Duration // Max time to wait for agent to be ready (0 = no wait, fail immediately) +} + +// SyncClockInInstance sets the guest realtime clock to the host's current +// time. Guest clocks resume from the snapshot's saved time after a standby +// restore, so this is called post-restore to remove the accumulated skew. +func SyncClockInInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts SyncClockOptions) error { + if opts.WaitForAgent == 0 { + err := syncClockOnce(ctx, dialer) + if err != nil && isRetryableConnectionError(err) { + CloseConn(dialer.Key()) + } + return err + } + + ctx, span := guestTracer().Start(ctx, "guest.sync_clock", trace.WithAttributes( + attribute.Bool("wait_for_agent", true), + attribute.Int64("wait_for_agent_ms", opts.WaitForAgent.Milliseconds()), + )) + defer span.End() + + deadline := time.Now().Add(opts.WaitForAgent) + start := time.Now() + attempts := 0 + retryableAttempts := 0 + firstRetryableErrorType := "" + lastRetryableErrorType := "" + lastRetryInterval := time.Duration(0) + + for { + attempts++ + err := syncClockOnce(ctx, dialer) + if err == nil { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.SetStatus(otelcodes.Ok, "") + return nil + } + if !isRetryableConnectionError(err) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryableAttempts++ + errType := retryableConnectionErrorType(err) + if firstRetryableErrorType == "" { + firstRetryableErrorType = errType + } + lastRetryableErrorType = errType + CloseConn(dialer.Key()) + + if time.Now().After(deadline) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryInterval := guestExecRetryInterval(time.Since(start)) + lastRetryInterval = retryInterval + select { + case <-ctx.Done(): + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(ctx.Err()) + span.SetStatus(otelcodes.Error, ctx.Err().Error()) + return ctx.Err() + case <-time.After(retryInterval): + } + } +} + +func syncClockOnce(ctx context.Context, dialer hypervisor.VsockDialer) error { + grpcConn, err := GetOrCreateConn(ctx, dialer) + if err != nil { + return fmt.Errorf("get grpc connection: %w", err) + } + client := NewGuestServiceClient(grpcConn) + + _, span := guestTracer().Start(ctx, "guest.sync_clock.rpc") + // Capture the host time per attempt so retries after a long agent wait + // don't bake the wait into the guest clock. + _, err = client.SyncClock(ctx, &SyncClockRequest{ + UnixNanos: time.Now().UnixNano(), + }) + finishGuestExecStepSpan(span, err) + if err != nil { + return fmt.Errorf("sync clock rpc: %w", err) + } + return nil +} diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index a239fc97..96134003 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -1380,6 +1380,88 @@ func (*ReconfigureNetworkResponse) Descriptor() ([]byte, []int) { return file_lib_guest_guest_proto_rawDescGZIP(), []int{18} } +// SyncClockRequest sets the guest realtime clock to the given wall-clock time +type SyncClockRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + UnixNanos int64 `protobuf:"varint,1,opt,name=unix_nanos,json=unixNanos,proto3" json:"unix_nanos,omitempty"` // Host CLOCK_REALTIME in nanoseconds since the Unix epoch + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SyncClockRequest) Reset() { + *x = SyncClockRequest{} + mi := &file_lib_guest_guest_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SyncClockRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SyncClockRequest) ProtoMessage() {} + +func (x *SyncClockRequest) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SyncClockRequest.ProtoReflect.Descriptor instead. +func (*SyncClockRequest) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{19} +} + +func (x *SyncClockRequest) GetUnixNanos() int64 { + if x != nil { + return x.UnixNanos + } + return 0 +} + +// SyncClockResponse acknowledges the clock sync request +type SyncClockResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SyncClockResponse) Reset() { + *x = SyncClockResponse{} + mi := &file_lib_guest_guest_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SyncClockResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SyncClockResponse) ProtoMessage() {} + +func (x *SyncClockResponse) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SyncClockResponse.ProtoReflect.Descriptor instead. +func (*SyncClockResponse) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{20} +} + var File_lib_guest_guest_proto protoreflect.FileDescriptor const file_lib_guest_guest_proto_rawDesc = "" + @@ -1479,14 +1561,19 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\x04ipv4\x18\x03 \x01(\tR\x04ipv4\x12\x16\n" + "\x06prefix\x18\x04 \x01(\rR\x06prefix\x12\x18\n" + "\agateway\x18\x05 \x01(\tR\agateway\"\x1c\n" + - "\x1aReconfigureNetworkResponse2\xae\x03\n" + + "\x1aReconfigureNetworkResponse\"1\n" + + "\x10SyncClockRequest\x12\x1d\n" + + "\n" + + "unix_nanos\x18\x01 \x01(\x03R\tunixNanos\"\x13\n" + + "\x11SyncClockResponse2\xee\x03\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + "\rCopyFromGuest\x12\x1b.guest.CopyFromGuestRequest\x1a\x1c.guest.CopyFromGuestResponse0\x01\x12;\n" + "\bStatPath\x12\x16.guest.StatPathRequest\x1a\x17.guest.StatPathResponse\x12;\n" + "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponse\x12Y\n" + - "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" + "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponse\x12>\n" + + "\tSyncClock\x12\x17.guest.SyncClockRequest\x1a\x18.guest.SyncClockResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" var ( file_lib_guest_guest_proto_rawDescOnce sync.Once @@ -1500,7 +1587,7 @@ func file_lib_guest_guest_proto_rawDescGZIP() []byte { return file_lib_guest_guest_proto_rawDescData } -var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_lib_guest_guest_proto_goTypes = []any{ (*ExecRequest)(nil), // 0: guest.ExecRequest (*ExecStart)(nil), // 1: guest.ExecStart @@ -1521,12 +1608,14 @@ var file_lib_guest_guest_proto_goTypes = []any{ (*ShutdownResponse)(nil), // 16: guest.ShutdownResponse (*ReconfigureNetworkRequest)(nil), // 17: guest.ReconfigureNetworkRequest (*ReconfigureNetworkResponse)(nil), // 18: guest.ReconfigureNetworkResponse - nil, // 19: guest.ExecStart.EnvEntry + (*SyncClockRequest)(nil), // 19: guest.SyncClockRequest + (*SyncClockResponse)(nil), // 20: guest.SyncClockResponse + nil, // 21: guest.ExecStart.EnvEntry } var file_lib_guest_guest_proto_depIdxs = []int32{ 1, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart 2, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize - 19, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry + 21, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry 5, // 3: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart 6, // 4: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd 10, // 5: guest.CopyFromGuestResponse.header:type_name -> guest.CopyFromGuestHeader @@ -1538,14 +1627,16 @@ var file_lib_guest_guest_proto_depIdxs = []int32{ 13, // 11: guest.GuestService.StatPath:input_type -> guest.StatPathRequest 15, // 12: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest 17, // 13: guest.GuestService.ReconfigureNetwork:input_type -> guest.ReconfigureNetworkRequest - 3, // 14: guest.GuestService.Exec:output_type -> guest.ExecResponse - 7, // 15: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse - 9, // 16: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse - 14, // 17: guest.GuestService.StatPath:output_type -> guest.StatPathResponse - 16, // 18: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse - 18, // 19: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse - 14, // [14:20] is the sub-list for method output_type - 8, // [8:14] is the sub-list for method input_type + 19, // 14: guest.GuestService.SyncClock:input_type -> guest.SyncClockRequest + 3, // 15: guest.GuestService.Exec:output_type -> guest.ExecResponse + 7, // 16: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse + 9, // 17: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse + 14, // 18: guest.GuestService.StatPath:output_type -> guest.StatPathResponse + 16, // 19: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse + 18, // 20: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse + 20, // 21: guest.GuestService.SyncClock:output_type -> guest.SyncClockResponse + 15, // [15:22] is the sub-list for method output_type + 8, // [8:15] is the sub-list for method input_type 8, // [8:8] is the sub-list for extension type_name 8, // [8:8] is the sub-list for extension extendee 0, // [0:8] is the sub-list for field type_name @@ -1583,7 +1674,7 @@ func file_lib_guest_guest_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_lib_guest_guest_proto_rawDesc), len(file_lib_guest_guest_proto_rawDesc)), NumEnums: 0, - NumMessages: 20, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index 317c21b3..6096f62d 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -23,6 +23,9 @@ service GuestService { // ReconfigureNetwork updates the guest network identity without spawning shell commands rpc ReconfigureNetwork(ReconfigureNetworkRequest) returns (ReconfigureNetworkResponse); + + // SyncClock sets the guest realtime clock, correcting skew after snapshot restore + rpc SyncClock(SyncClockRequest) returns (SyncClockResponse); } // ExecRequest represents messages from client to server @@ -169,3 +172,11 @@ message ReconfigureNetworkRequest { // ReconfigureNetworkResponse acknowledges the network reconfiguration request message ReconfigureNetworkResponse {} + +// SyncClockRequest sets the guest realtime clock to the given wall-clock time +message SyncClockRequest { + int64 unix_nanos = 1; // Host CLOCK_REALTIME in nanoseconds since the Unix epoch +} + +// SyncClockResponse acknowledges the clock sync request +message SyncClockResponse {} diff --git a/lib/guest/guest_grpc.pb.go b/lib/guest/guest_grpc.pb.go index f93631d9..958005e7 100644 --- a/lib/guest/guest_grpc.pb.go +++ b/lib/guest/guest_grpc.pb.go @@ -25,6 +25,7 @@ const ( GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" GuestService_ReconfigureNetwork_FullMethodName = "/guest.GuestService/ReconfigureNetwork" + GuestService_SyncClock_FullMethodName = "/guest.GuestService/SyncClock" ) // GuestServiceClient is the client API for GuestService service. @@ -45,6 +46,8 @@ type GuestServiceClient interface { Shutdown(ctx context.Context, in *ShutdownRequest, opts ...grpc.CallOption) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(ctx context.Context, in *ReconfigureNetworkRequest, opts ...grpc.CallOption) (*ReconfigureNetworkResponse, error) + // SyncClock sets the guest realtime clock, correcting skew after snapshot restore + SyncClock(ctx context.Context, in *SyncClockRequest, opts ...grpc.CallOption) (*SyncClockResponse, error) } type guestServiceClient struct { @@ -130,6 +133,16 @@ func (c *guestServiceClient) ReconfigureNetwork(ctx context.Context, in *Reconfi return out, nil } +func (c *guestServiceClient) SyncClock(ctx context.Context, in *SyncClockRequest, opts ...grpc.CallOption) (*SyncClockResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SyncClockResponse) + err := c.cc.Invoke(ctx, GuestService_SyncClock_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GuestServiceServer is the server API for GuestService service. // All implementations must embed UnimplementedGuestServiceServer // for forward compatibility. @@ -148,6 +161,8 @@ type GuestServiceServer interface { Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) + // SyncClock sets the guest realtime clock, correcting skew after snapshot restore + SyncClock(context.Context, *SyncClockRequest) (*SyncClockResponse, error) mustEmbedUnimplementedGuestServiceServer() } @@ -176,6 +191,9 @@ func (UnimplementedGuestServiceServer) Shutdown(context.Context, *ShutdownReques func (UnimplementedGuestServiceServer) ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) { return nil, status.Error(codes.Unimplemented, "method ReconfigureNetwork not implemented") } +func (UnimplementedGuestServiceServer) SyncClock(context.Context, *SyncClockRequest) (*SyncClockResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SyncClock not implemented") +} func (UnimplementedGuestServiceServer) mustEmbedUnimplementedGuestServiceServer() {} func (UnimplementedGuestServiceServer) testEmbeddedByValue() {} @@ -276,6 +294,24 @@ func _GuestService_ReconfigureNetwork_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } +func _GuestService_SyncClock_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SyncClockRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GuestServiceServer).SyncClock(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GuestService_SyncClock_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GuestServiceServer).SyncClock(ctx, req.(*SyncClockRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GuestService_ServiceDesc is the grpc.ServiceDesc for GuestService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -295,6 +331,10 @@ var GuestService_ServiceDesc = grpc.ServiceDesc{ MethodName: "ReconfigureNetwork", Handler: _GuestService_ReconfigureNetwork_Handler, }, + { + MethodName: "SyncClock", + Handler: _GuestService_SyncClock_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 49a1f80b..1b4f4700 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -1551,6 +1551,13 @@ func TestStandbyAndRestore(t *testing.T) { assert.True(t, inst.HasSnapshot) t.Log("Instance in standby") + // Let real time advance while the guest is paused. The snapshot froze the + // guest wall clock, so without the post-restore SyncClock the guest would + // lag by at least this long after restore. + standbySkew := 10 * time.Second + t.Logf("Sleeping %s in standby to accumulate clock skew...", standbySkew) + time.Sleep(standbySkew) + // Verify snapshot exists p := paths.New(tmpDir) snapshotDir := p.InstanceSnapshotLatest(inst.Id) @@ -1593,6 +1600,30 @@ func TestStandbyAndRestore(t *testing.T) { } } + // The restore path syncs the guest realtime clock before returning, so the + // guest must not lag by the time spent in standby. + guestEpoch := readGuestEpoch(t, ctx, inst) + clockLag := time.Now().Unix() - guestEpoch + t.Logf("Guest clock lag after restore: %ds", clockLag) + assert.LessOrEqual(t, clockLag, int64(5), "guest clock should be resynced after restore, not lag by the standby duration") + assert.GreaterOrEqual(t, clockLag, int64(-5), "guest clock should not run ahead of the host") + + // Exercise the exec fallback used for snapshots taken by older agents: + // skew the guest clock back an hour, run the fallback, verify it recovers. + _, code, err := execInInstance(ctx, inst, "sh", "-c", fmt.Sprintf("date -u -s @%d", time.Now().Add(-time.Hour).Unix())) + require.NoError(t, err) + require.Equal(t, 0, code, "skewing guest clock via date should succeed") + + dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + require.NoError(t, err) + require.NoError(t, syncGuestClockWithExec(ctx, dialer)) + + guestEpoch = readGuestEpoch(t, ctx, inst) + clockLag = time.Now().Unix() - guestEpoch + t.Logf("Guest clock lag after exec fallback sync: %ds", clockLag) + assert.LessOrEqual(t, clockLag, int64(5), "exec fallback should resync a skewed guest clock") + assert.GreaterOrEqual(t, clockLag, int64(-5), "exec fallback should not set the guest clock ahead of the host") + // Cleanup (no sleep needed - DeleteInstance handles process cleanup) t.Log("Cleaning up...") err = manager.DeleteInstance(ctx, inst.Id) @@ -1601,6 +1632,16 @@ func TestStandbyAndRestore(t *testing.T) { t.Log("Standby/restore test complete!") } +func readGuestEpoch(t *testing.T, ctx context.Context, inst *Instance) int64 { + t.Helper() + out, code, err := execInInstance(ctx, inst, "date", "+%s") + require.NoError(t, err) + require.Equal(t, 0, code, "date should succeed in guest: %s", out) + epoch, err := strconv.ParseInt(strings.TrimSpace(out), 10, 64) + require.NoError(t, err, "parse guest epoch from %q", out) + return epoch +} + func TestCloudHypervisorSnapshotFeature(t *testing.T) { t.Parallel() if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { diff --git a/lib/instances/restore.go b/lib/instances/restore.go index f41c7725..68a038bb 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -360,6 +360,22 @@ func (m *manager) restoreInstance( } releaseRestoreSlotOnce() + // 7. Sync the guest realtime clock. It resumes from the snapshot's saved + // time, so until corrected every in-guest timestamp lags by the time spent + // in standby. Non-fatal: a skewed clock degrades timestamps, not the VM. + if !stored.SkipGuestAgent { + syncCtx, syncSpanEnd := m.startLifecycleStep(ctx, "sync_guest_clock", + attribute.String("instance_id", id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "sync_guest_clock"), + ) + err := syncGuestClock(syncCtx, stored) + syncSpanEnd(err) + if err != nil { + log.WarnContext(ctx, "failed to sync guest clock after restore", "instance_id", id, "error", err) + } + } + // 8. Delete snapshot after successful restore unless the hypervisor is keeping it // as the base for the next standby snapshot. if m.supportsSnapshotBaseReuse(stored.HypervisorType) { @@ -495,6 +511,55 @@ func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc return nil } +// syncClockAgentWait bounds how long a restore blocks on clock sync. The +// agent was running when the snapshot was taken, so it is reachable almost +// immediately after resume; clock sync is non-fatal, so an unresponsive agent +// should cost seconds of restore latency, not the 120s the (fatal, fork-only) +// network reconfigure path tolerates. +const syncClockAgentWait = 10 * time.Second + +func syncGuestClock(ctx context.Context, stored *StoredMetadata) error { + dialer, err := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) + if err != nil { + return fmt.Errorf("create vsock dialer: %w", err) + } + + err = guest.SyncClockInInstance(ctx, dialer, guest.SyncClockOptions{ + WaitForAgent: syncClockAgentWait, + }) + if err != nil { + // Instances standbyed before the agent gained SyncClock still run the + // old binary from their snapshot; set the clock via exec instead. + if status.Code(err) == codes.Unimplemented { + return syncGuestClockWithExec(ctx, dialer) + } + return fmt.Errorf("sync guest clock: %w", err) + } + + return nil +} + +func syncGuestClockWithExec(ctx context.Context, dialer hypervisor.VsockDialer) error { + var stdout, stderr bytes.Buffer + // The timestamp is baked into the command before any agent wait, but this + // path only runs right after the agent answered the SyncClock RPC with + // Unimplemented, so the wait is ~0 and staleness is bounded by + // syncClockAgentWait — negligible next to the standby skew being corrected. + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"sh", "-c", fmt.Sprintf("date -u -s @%d", time.Now().Unix())}, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: syncClockAgentWait, + }) + if err != nil { + return fmt.Errorf("exec clock sync command: %w", err) + } + if exit.Code != 0 { + return fmt.Errorf("clock sync command failed (exit=%d, stdout=%q, stderr=%q)", exit.Code, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String())) + } + return nil +} + func reconfigureGuestNetworkWithExec(ctx context.Context, dialer hypervisor.VsockDialer, alloc *network.Allocation) error { cmd, err := guestNetworkReconfigureCommand(alloc) if err != nil { diff --git a/lib/system/guest_agent/clock.go b/lib/system/guest_agent/clock.go new file mode 100644 index 00000000..6e5e9429 --- /dev/null +++ b/lib/system/guest_agent/clock.go @@ -0,0 +1,29 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + pb "github.com/kernel/hypeman/lib/guest" +) + +// SyncClock sets the guest realtime clock. The guest wall clock resumes from +// the snapshot's saved time after a standby restore, so the host calls this +// post-restore to remove the accumulated skew. +func (s *guestServer) SyncClock(_ context.Context, req *pb.SyncClockRequest) (*pb.SyncClockResponse, error) { + if req.UnixNanos <= 0 { + return nil, fmt.Errorf("invalid unix_nanos %d", req.UnixNanos) + } + + target := time.Unix(0, req.UnixNanos) + adjustment := time.Until(target) + if err := setRealtimeClock(req.UnixNanos); err != nil { + return nil, err + } + + log.Printf("[guest-agent] realtime clock set to %s (adjusted by %s)", + target.UTC().Format(time.RFC3339Nano), adjustment) + return &pb.SyncClockResponse{}, nil +} diff --git a/lib/system/guest_agent/clock_linux.go b/lib/system/guest_agent/clock_linux.go new file mode 100644 index 00000000..8f0af23e --- /dev/null +++ b/lib/system/guest_agent/clock_linux.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +func setRealtimeClock(unixNanos int64) error { + ts := unix.NsecToTimespec(unixNanos) + if err := unix.ClockSettime(unix.CLOCK_REALTIME, &ts); err != nil { + return fmt.Errorf("set realtime clock: %w", err) + } + return nil +} diff --git a/lib/system/guest_agent/clock_other.go b/lib/system/guest_agent/clock_other.go new file mode 100644 index 00000000..17659cde --- /dev/null +++ b/lib/system/guest_agent/clock_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package main + +import "errors" + +func setRealtimeClock(int64) error { + return errors.New("setting the realtime clock is only supported on linux") +} diff --git a/lib/system/guest_agent/clock_test.go b/lib/system/guest_agent/clock_test.go new file mode 100644 index 00000000..48b7a662 --- /dev/null +++ b/lib/system/guest_agent/clock_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "context" + "testing" + + pb "github.com/kernel/hypeman/lib/guest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSyncClockRejectsInvalidTime(t *testing.T) { + s := &guestServer{} + + for _, nanos := range []int64{0, -1} { + _, err := s.SyncClock(context.Background(), &pb.SyncClockRequest{UnixNanos: nanos}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid unix_nanos") + } +}