From d00132ad01795e13c4b470b1b324f27d523797ec Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:14:47 +0000 Subject: [PATCH 1/4] Sync guest realtime clock after standby restore The guest wall clock resumes from the snapshot's saved time after a restore, so every in-guest timestamp lags by the time spent in standby until corrected. Add a SyncClock guest-agent RPC that sets CLOCK_REALTIME to the host's current time, and call it from the restore path once the VM resumes. Instances whose snapshots predate the new agent binary fall back to setting the clock via exec. Co-Authored-By: Claude Opus 4.7 --- lib/guest/clock.go | 106 ++++++++++++++++++++++++ lib/guest/guest.pb.go | 119 +++++++++++++++++++++++---- lib/guest/guest.proto | 11 +++ lib/guest/guest_grpc.pb.go | 40 +++++++++ lib/instances/restore.go | 54 ++++++++++++ lib/system/guest_agent/clock.go | 31 +++++++ lib/system/guest_agent/clock_test.go | 20 +++++ 7 files changed, 367 insertions(+), 14 deletions(-) create mode 100644 lib/guest/clock.go create mode 100644 lib/system/guest_agent/clock.go create mode 100644 lib/system/guest_agent/clock_test.go diff --git a/lib/guest/clock.go b/lib/guest/clock.go new file mode 100644 index 00000000..af098e16 --- /dev/null +++ b/lib/guest/clock.go @@ -0,0 +1,106 @@ +package guest + +import ( + "context" + "fmt" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "go.opentelemetry.io/otel/attribute" + otelcodes "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// SyncClockOptions configures a guest clock sync +type SyncClockOptions struct { + WaitForAgent time.Duration // Max time to wait for agent to be ready (0 = no wait, fail immediately) +} + +// SyncClockInInstance sets the guest realtime clock to the host's current +// time. Guest clocks resume from the snapshot's saved time after a standby +// restore, so this is called post-restore to remove the accumulated skew. +func SyncClockInInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts SyncClockOptions) error { + if opts.WaitForAgent == 0 { + err := syncClockOnce(ctx, dialer) + if err != nil && isRetryableConnectionError(err) { + CloseConn(dialer.Key()) + } + return err + } + + ctx, span := guestTracer().Start(ctx, "guest.sync_clock", trace.WithAttributes( + attribute.Bool("wait_for_agent", true), + attribute.Int64("wait_for_agent_ms", opts.WaitForAgent.Milliseconds()), + )) + defer span.End() + + deadline := time.Now().Add(opts.WaitForAgent) + start := time.Now() + attempts := 0 + retryableAttempts := 0 + firstRetryableErrorType := "" + lastRetryableErrorType := "" + lastRetryInterval := time.Duration(0) + + for { + attempts++ + err := syncClockOnce(ctx, dialer) + if err == nil { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.SetStatus(otelcodes.Ok, "") + return nil + } + if !isRetryableConnectionError(err) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryableAttempts++ + errType := retryableConnectionErrorType(err) + if firstRetryableErrorType == "" { + firstRetryableErrorType = errType + } + lastRetryableErrorType = errType + CloseConn(dialer.Key()) + + if time.Now().After(deadline) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryInterval := guestExecRetryInterval(time.Since(start)) + lastRetryInterval = retryInterval + select { + case <-ctx.Done(): + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(ctx.Err()) + span.SetStatus(otelcodes.Error, ctx.Err().Error()) + return ctx.Err() + case <-time.After(retryInterval): + } + } +} + +func syncClockOnce(ctx context.Context, dialer hypervisor.VsockDialer) error { + grpcConn, err := GetOrCreateConn(ctx, dialer) + if err != nil { + return fmt.Errorf("get grpc connection: %w", err) + } + client := NewGuestServiceClient(grpcConn) + + _, span := guestTracer().Start(ctx, "guest.sync_clock.rpc") + // Capture the host time per attempt so retries after a long agent wait + // don't bake the wait into the guest clock. + _, err = client.SyncClock(ctx, &SyncClockRequest{ + UnixNanos: time.Now().UnixNano(), + }) + finishGuestExecStepSpan(span, err) + if err != nil { + return fmt.Errorf("sync clock rpc: %w", err) + } + return nil +} diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index a239fc97..96134003 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -1380,6 +1380,88 @@ func (*ReconfigureNetworkResponse) Descriptor() ([]byte, []int) { return file_lib_guest_guest_proto_rawDescGZIP(), []int{18} } +// SyncClockRequest sets the guest realtime clock to the given wall-clock time +type SyncClockRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + UnixNanos int64 `protobuf:"varint,1,opt,name=unix_nanos,json=unixNanos,proto3" json:"unix_nanos,omitempty"` // Host CLOCK_REALTIME in nanoseconds since the Unix epoch + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SyncClockRequest) Reset() { + *x = SyncClockRequest{} + mi := &file_lib_guest_guest_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SyncClockRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SyncClockRequest) ProtoMessage() {} + +func (x *SyncClockRequest) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SyncClockRequest.ProtoReflect.Descriptor instead. +func (*SyncClockRequest) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{19} +} + +func (x *SyncClockRequest) GetUnixNanos() int64 { + if x != nil { + return x.UnixNanos + } + return 0 +} + +// SyncClockResponse acknowledges the clock sync request +type SyncClockResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SyncClockResponse) Reset() { + *x = SyncClockResponse{} + mi := &file_lib_guest_guest_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SyncClockResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SyncClockResponse) ProtoMessage() {} + +func (x *SyncClockResponse) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SyncClockResponse.ProtoReflect.Descriptor instead. +func (*SyncClockResponse) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{20} +} + var File_lib_guest_guest_proto protoreflect.FileDescriptor const file_lib_guest_guest_proto_rawDesc = "" + @@ -1479,14 +1561,19 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\x04ipv4\x18\x03 \x01(\tR\x04ipv4\x12\x16\n" + "\x06prefix\x18\x04 \x01(\rR\x06prefix\x12\x18\n" + "\agateway\x18\x05 \x01(\tR\agateway\"\x1c\n" + - "\x1aReconfigureNetworkResponse2\xae\x03\n" + + "\x1aReconfigureNetworkResponse\"1\n" + + "\x10SyncClockRequest\x12\x1d\n" + + "\n" + + "unix_nanos\x18\x01 \x01(\x03R\tunixNanos\"\x13\n" + + "\x11SyncClockResponse2\xee\x03\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + "\rCopyFromGuest\x12\x1b.guest.CopyFromGuestRequest\x1a\x1c.guest.CopyFromGuestResponse0\x01\x12;\n" + "\bStatPath\x12\x16.guest.StatPathRequest\x1a\x17.guest.StatPathResponse\x12;\n" + "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponse\x12Y\n" + - "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" + "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponse\x12>\n" + + "\tSyncClock\x12\x17.guest.SyncClockRequest\x1a\x18.guest.SyncClockResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" var ( file_lib_guest_guest_proto_rawDescOnce sync.Once @@ -1500,7 +1587,7 @@ func file_lib_guest_guest_proto_rawDescGZIP() []byte { return file_lib_guest_guest_proto_rawDescData } -var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_lib_guest_guest_proto_goTypes = []any{ (*ExecRequest)(nil), // 0: guest.ExecRequest (*ExecStart)(nil), // 1: guest.ExecStart @@ -1521,12 +1608,14 @@ var file_lib_guest_guest_proto_goTypes = []any{ (*ShutdownResponse)(nil), // 16: guest.ShutdownResponse (*ReconfigureNetworkRequest)(nil), // 17: guest.ReconfigureNetworkRequest (*ReconfigureNetworkResponse)(nil), // 18: guest.ReconfigureNetworkResponse - nil, // 19: guest.ExecStart.EnvEntry + (*SyncClockRequest)(nil), // 19: guest.SyncClockRequest + (*SyncClockResponse)(nil), // 20: guest.SyncClockResponse + nil, // 21: guest.ExecStart.EnvEntry } var file_lib_guest_guest_proto_depIdxs = []int32{ 1, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart 2, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize - 19, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry + 21, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry 5, // 3: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart 6, // 4: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd 10, // 5: guest.CopyFromGuestResponse.header:type_name -> guest.CopyFromGuestHeader @@ -1538,14 +1627,16 @@ var file_lib_guest_guest_proto_depIdxs = []int32{ 13, // 11: guest.GuestService.StatPath:input_type -> guest.StatPathRequest 15, // 12: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest 17, // 13: guest.GuestService.ReconfigureNetwork:input_type -> guest.ReconfigureNetworkRequest - 3, // 14: guest.GuestService.Exec:output_type -> guest.ExecResponse - 7, // 15: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse - 9, // 16: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse - 14, // 17: guest.GuestService.StatPath:output_type -> guest.StatPathResponse - 16, // 18: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse - 18, // 19: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse - 14, // [14:20] is the sub-list for method output_type - 8, // [8:14] is the sub-list for method input_type + 19, // 14: guest.GuestService.SyncClock:input_type -> guest.SyncClockRequest + 3, // 15: guest.GuestService.Exec:output_type -> guest.ExecResponse + 7, // 16: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse + 9, // 17: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse + 14, // 18: guest.GuestService.StatPath:output_type -> guest.StatPathResponse + 16, // 19: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse + 18, // 20: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse + 20, // 21: guest.GuestService.SyncClock:output_type -> guest.SyncClockResponse + 15, // [15:22] is the sub-list for method output_type + 8, // [8:15] is the sub-list for method input_type 8, // [8:8] is the sub-list for extension type_name 8, // [8:8] is the sub-list for extension extendee 0, // [0:8] is the sub-list for field type_name @@ -1583,7 +1674,7 @@ func file_lib_guest_guest_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_lib_guest_guest_proto_rawDesc), len(file_lib_guest_guest_proto_rawDesc)), NumEnums: 0, - NumMessages: 20, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index 317c21b3..6096f62d 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -23,6 +23,9 @@ service GuestService { // ReconfigureNetwork updates the guest network identity without spawning shell commands rpc ReconfigureNetwork(ReconfigureNetworkRequest) returns (ReconfigureNetworkResponse); + + // SyncClock sets the guest realtime clock, correcting skew after snapshot restore + rpc SyncClock(SyncClockRequest) returns (SyncClockResponse); } // ExecRequest represents messages from client to server @@ -169,3 +172,11 @@ message ReconfigureNetworkRequest { // ReconfigureNetworkResponse acknowledges the network reconfiguration request message ReconfigureNetworkResponse {} + +// SyncClockRequest sets the guest realtime clock to the given wall-clock time +message SyncClockRequest { + int64 unix_nanos = 1; // Host CLOCK_REALTIME in nanoseconds since the Unix epoch +} + +// SyncClockResponse acknowledges the clock sync request +message SyncClockResponse {} diff --git a/lib/guest/guest_grpc.pb.go b/lib/guest/guest_grpc.pb.go index f93631d9..958005e7 100644 --- a/lib/guest/guest_grpc.pb.go +++ b/lib/guest/guest_grpc.pb.go @@ -25,6 +25,7 @@ const ( GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" GuestService_ReconfigureNetwork_FullMethodName = "/guest.GuestService/ReconfigureNetwork" + GuestService_SyncClock_FullMethodName = "/guest.GuestService/SyncClock" ) // GuestServiceClient is the client API for GuestService service. @@ -45,6 +46,8 @@ type GuestServiceClient interface { Shutdown(ctx context.Context, in *ShutdownRequest, opts ...grpc.CallOption) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(ctx context.Context, in *ReconfigureNetworkRequest, opts ...grpc.CallOption) (*ReconfigureNetworkResponse, error) + // SyncClock sets the guest realtime clock, correcting skew after snapshot restore + SyncClock(ctx context.Context, in *SyncClockRequest, opts ...grpc.CallOption) (*SyncClockResponse, error) } type guestServiceClient struct { @@ -130,6 +133,16 @@ func (c *guestServiceClient) ReconfigureNetwork(ctx context.Context, in *Reconfi return out, nil } +func (c *guestServiceClient) SyncClock(ctx context.Context, in *SyncClockRequest, opts ...grpc.CallOption) (*SyncClockResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SyncClockResponse) + err := c.cc.Invoke(ctx, GuestService_SyncClock_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GuestServiceServer is the server API for GuestService service. // All implementations must embed UnimplementedGuestServiceServer // for forward compatibility. @@ -148,6 +161,8 @@ type GuestServiceServer interface { Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) + // SyncClock sets the guest realtime clock, correcting skew after snapshot restore + SyncClock(context.Context, *SyncClockRequest) (*SyncClockResponse, error) mustEmbedUnimplementedGuestServiceServer() } @@ -176,6 +191,9 @@ func (UnimplementedGuestServiceServer) Shutdown(context.Context, *ShutdownReques func (UnimplementedGuestServiceServer) ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) { return nil, status.Error(codes.Unimplemented, "method ReconfigureNetwork not implemented") } +func (UnimplementedGuestServiceServer) SyncClock(context.Context, *SyncClockRequest) (*SyncClockResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SyncClock not implemented") +} func (UnimplementedGuestServiceServer) mustEmbedUnimplementedGuestServiceServer() {} func (UnimplementedGuestServiceServer) testEmbeddedByValue() {} @@ -276,6 +294,24 @@ func _GuestService_ReconfigureNetwork_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } +func _GuestService_SyncClock_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SyncClockRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GuestServiceServer).SyncClock(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GuestService_SyncClock_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GuestServiceServer).SyncClock(ctx, req.(*SyncClockRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GuestService_ServiceDesc is the grpc.ServiceDesc for GuestService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -295,6 +331,10 @@ var GuestService_ServiceDesc = grpc.ServiceDesc{ MethodName: "ReconfigureNetwork", Handler: _GuestService_ReconfigureNetwork_Handler, }, + { + MethodName: "SyncClock", + Handler: _GuestService_SyncClock_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/lib/instances/restore.go b/lib/instances/restore.go index f41c7725..6099fce7 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -360,6 +360,22 @@ func (m *manager) restoreInstance( } releaseRestoreSlotOnce() + // 7. Sync the guest realtime clock. It resumes from the snapshot's saved + // time, so until corrected every in-guest timestamp lags by the time spent + // in standby. Non-fatal: a skewed clock degrades timestamps, not the VM. + if !stored.SkipGuestAgent { + syncCtx, syncSpanEnd := m.startLifecycleStep(ctx, "sync_guest_clock", + attribute.String("instance_id", id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "sync_guest_clock"), + ) + err := syncGuestClock(syncCtx, stored) + syncSpanEnd(err) + if err != nil { + log.WarnContext(ctx, "failed to sync guest clock after restore", "instance_id", id, "error", err) + } + } + // 8. Delete snapshot after successful restore unless the hypervisor is keeping it // as the base for the next standby snapshot. if m.supportsSnapshotBaseReuse(stored.HypervisorType) { @@ -495,6 +511,44 @@ func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc return nil } +func syncGuestClock(ctx context.Context, stored *StoredMetadata) error { + dialer, err := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) + if err != nil { + return fmt.Errorf("create vsock dialer: %w", err) + } + + err = guest.SyncClockInInstance(ctx, dialer, guest.SyncClockOptions{ + WaitForAgent: 120 * time.Second, + }) + if err != nil { + // Instances standbyed before the agent gained SyncClock still run the + // old binary from their snapshot; set the clock via exec instead. + if status.Code(err) == codes.Unimplemented { + return syncGuestClockWithExec(ctx, dialer) + } + return fmt.Errorf("sync guest clock: %w", err) + } + + return nil +} + +func syncGuestClockWithExec(ctx context.Context, dialer hypervisor.VsockDialer) error { + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"sh", "-c", fmt.Sprintf("date -u -s @%d", time.Now().Unix())}, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 120 * time.Second, + }) + if err != nil { + return fmt.Errorf("exec clock sync command: %w", err) + } + if exit.Code != 0 { + return fmt.Errorf("clock sync command failed (exit=%d, stdout=%q, stderr=%q)", exit.Code, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String())) + } + return nil +} + func reconfigureGuestNetworkWithExec(ctx context.Context, dialer hypervisor.VsockDialer, alloc *network.Allocation) error { cmd, err := guestNetworkReconfigureCommand(alloc) if err != nil { diff --git a/lib/system/guest_agent/clock.go b/lib/system/guest_agent/clock.go new file mode 100644 index 00000000..2d0bcfbb --- /dev/null +++ b/lib/system/guest_agent/clock.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + pb "github.com/kernel/hypeman/lib/guest" + "golang.org/x/sys/unix" +) + +// SyncClock sets the guest realtime clock. The guest wall clock resumes from +// the snapshot's saved time after a standby restore, so the host calls this +// post-restore to remove the accumulated skew. +func (s *guestServer) SyncClock(_ context.Context, req *pb.SyncClockRequest) (*pb.SyncClockResponse, error) { + if req.UnixNanos <= 0 { + return nil, fmt.Errorf("invalid unix_nanos %d", req.UnixNanos) + } + + target := time.Unix(0, req.UnixNanos) + adjustment := time.Until(target) + ts := unix.NsecToTimespec(req.UnixNanos) + if err := unix.ClockSettime(unix.CLOCK_REALTIME, &ts); err != nil { + return nil, fmt.Errorf("set realtime clock: %w", err) + } + + log.Printf("[guest-agent] realtime clock set to %s (adjusted by %s)", + target.UTC().Format(time.RFC3339Nano), adjustment) + return &pb.SyncClockResponse{}, nil +} diff --git a/lib/system/guest_agent/clock_test.go b/lib/system/guest_agent/clock_test.go new file mode 100644 index 00000000..48b7a662 --- /dev/null +++ b/lib/system/guest_agent/clock_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "context" + "testing" + + pb "github.com/kernel/hypeman/lib/guest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSyncClockRejectsInvalidTime(t *testing.T) { + s := &guestServer{} + + for _, nanos := range []int64{0, -1} { + _, err := s.SyncClock(context.Background(), &pb.SyncClockRequest{UnixNanos: nanos}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid unix_nanos") + } +} From 8e2a101a0498c4c57c29877ed4cd9edde7e7c1a6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:19:30 +0000 Subject: [PATCH 2/4] Gate clock_settime behind a linux build tag unix.ClockSettime does not exist on darwin, which broke test-darwin compiling the guest_agent package. The shipped agent binary is linux-only; non-linux builds get an error stub. Co-Authored-By: Claude Opus 4.7 --- lib/system/guest_agent/clock.go | 6 ++---- lib/system/guest_agent/clock_linux.go | 15 +++++++++++++++ lib/system/guest_agent/clock_other.go | 9 +++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 lib/system/guest_agent/clock_linux.go create mode 100644 lib/system/guest_agent/clock_other.go diff --git a/lib/system/guest_agent/clock.go b/lib/system/guest_agent/clock.go index 2d0bcfbb..6e5e9429 100644 --- a/lib/system/guest_agent/clock.go +++ b/lib/system/guest_agent/clock.go @@ -7,7 +7,6 @@ import ( "time" pb "github.com/kernel/hypeman/lib/guest" - "golang.org/x/sys/unix" ) // SyncClock sets the guest realtime clock. The guest wall clock resumes from @@ -20,9 +19,8 @@ func (s *guestServer) SyncClock(_ context.Context, req *pb.SyncClockRequest) (*p target := time.Unix(0, req.UnixNanos) adjustment := time.Until(target) - ts := unix.NsecToTimespec(req.UnixNanos) - if err := unix.ClockSettime(unix.CLOCK_REALTIME, &ts); err != nil { - return nil, fmt.Errorf("set realtime clock: %w", err) + if err := setRealtimeClock(req.UnixNanos); err != nil { + return nil, err } log.Printf("[guest-agent] realtime clock set to %s (adjusted by %s)", diff --git a/lib/system/guest_agent/clock_linux.go b/lib/system/guest_agent/clock_linux.go new file mode 100644 index 00000000..8f0af23e --- /dev/null +++ b/lib/system/guest_agent/clock_linux.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +func setRealtimeClock(unixNanos int64) error { + ts := unix.NsecToTimespec(unixNanos) + if err := unix.ClockSettime(unix.CLOCK_REALTIME, &ts); err != nil { + return fmt.Errorf("set realtime clock: %w", err) + } + return nil +} diff --git a/lib/system/guest_agent/clock_other.go b/lib/system/guest_agent/clock_other.go new file mode 100644 index 00000000..17659cde --- /dev/null +++ b/lib/system/guest_agent/clock_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package main + +import "errors" + +func setRealtimeClock(int64) error { + return errors.New("setting the realtime clock is only supported on linux") +} From 2bd1c2b1f2aae958c3b820f3a78d7ff3f05c2dbf Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:00:14 +0000 Subject: [PATCH 3/4] Bound restore clock sync agent wait to 10s Clock sync is non-fatal and runs on every restore, so an unresponsive agent should cost seconds of restore latency, not the 120s the fatal fork-only network reconfigure path tolerates. The agent was running at snapshot time and is reachable almost immediately after resume, so the short wait covers the happy path. This also bounds how stale the timestamp baked into the exec fallback command can get. Co-Authored-By: Claude Opus 4.7 --- lib/instances/restore.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 6099fce7..68a038bb 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -511,6 +511,13 @@ func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc return nil } +// syncClockAgentWait bounds how long a restore blocks on clock sync. The +// agent was running when the snapshot was taken, so it is reachable almost +// immediately after resume; clock sync is non-fatal, so an unresponsive agent +// should cost seconds of restore latency, not the 120s the (fatal, fork-only) +// network reconfigure path tolerates. +const syncClockAgentWait = 10 * time.Second + func syncGuestClock(ctx context.Context, stored *StoredMetadata) error { dialer, err := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) if err != nil { @@ -518,7 +525,7 @@ func syncGuestClock(ctx context.Context, stored *StoredMetadata) error { } err = guest.SyncClockInInstance(ctx, dialer, guest.SyncClockOptions{ - WaitForAgent: 120 * time.Second, + WaitForAgent: syncClockAgentWait, }) if err != nil { // Instances standbyed before the agent gained SyncClock still run the @@ -534,11 +541,15 @@ func syncGuestClock(ctx context.Context, stored *StoredMetadata) error { func syncGuestClockWithExec(ctx context.Context, dialer hypervisor.VsockDialer) error { var stdout, stderr bytes.Buffer + // The timestamp is baked into the command before any agent wait, but this + // path only runs right after the agent answered the SyncClock RPC with + // Unimplemented, so the wait is ~0 and staleness is bounded by + // syncClockAgentWait — negligible next to the standby skew being corrected. exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ Command: []string{"sh", "-c", fmt.Sprintf("date -u -s @%d", time.Now().Unix())}, Stdout: &stdout, Stderr: &stderr, - WaitForAgent: 120 * time.Second, + WaitForAgent: syncClockAgentWait, }) if err != nil { return fmt.Errorf("exec clock sync command: %w", err) From b06ac63c6c9355f332598678f0171d21810ebb45 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:15:02 +0000 Subject: [PATCH 4/4] Verify guest clock resync in standby/restore test TestStandbyAndRestore now sleeps 10s in standby and asserts the guest clock does not lag after restore, and exercises the exec fallback by skewing the guest clock back an hour and syncing it via syncGuestClockWithExec. Without the restore-time SyncClock the guest lags by the full standby duration and the assertion fails. Co-Authored-By: Claude Opus 4.7 --- lib/instances/manager_test.go | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 49a1f80b..1b4f4700 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -1551,6 +1551,13 @@ func TestStandbyAndRestore(t *testing.T) { assert.True(t, inst.HasSnapshot) t.Log("Instance in standby") + // Let real time advance while the guest is paused. The snapshot froze the + // guest wall clock, so without the post-restore SyncClock the guest would + // lag by at least this long after restore. + standbySkew := 10 * time.Second + t.Logf("Sleeping %s in standby to accumulate clock skew...", standbySkew) + time.Sleep(standbySkew) + // Verify snapshot exists p := paths.New(tmpDir) snapshotDir := p.InstanceSnapshotLatest(inst.Id) @@ -1593,6 +1600,30 @@ func TestStandbyAndRestore(t *testing.T) { } } + // The restore path syncs the guest realtime clock before returning, so the + // guest must not lag by the time spent in standby. + guestEpoch := readGuestEpoch(t, ctx, inst) + clockLag := time.Now().Unix() - guestEpoch + t.Logf("Guest clock lag after restore: %ds", clockLag) + assert.LessOrEqual(t, clockLag, int64(5), "guest clock should be resynced after restore, not lag by the standby duration") + assert.GreaterOrEqual(t, clockLag, int64(-5), "guest clock should not run ahead of the host") + + // Exercise the exec fallback used for snapshots taken by older agents: + // skew the guest clock back an hour, run the fallback, verify it recovers. + _, code, err := execInInstance(ctx, inst, "sh", "-c", fmt.Sprintf("date -u -s @%d", time.Now().Add(-time.Hour).Unix())) + require.NoError(t, err) + require.Equal(t, 0, code, "skewing guest clock via date should succeed") + + dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + require.NoError(t, err) + require.NoError(t, syncGuestClockWithExec(ctx, dialer)) + + guestEpoch = readGuestEpoch(t, ctx, inst) + clockLag = time.Now().Unix() - guestEpoch + t.Logf("Guest clock lag after exec fallback sync: %ds", clockLag) + assert.LessOrEqual(t, clockLag, int64(5), "exec fallback should resync a skewed guest clock") + assert.GreaterOrEqual(t, clockLag, int64(-5), "exec fallback should not set the guest clock ahead of the host") + // Cleanup (no sleep needed - DeleteInstance handles process cleanup) t.Log("Cleaning up...") err = manager.DeleteInstance(ctx, inst.Id) @@ -1601,6 +1632,16 @@ func TestStandbyAndRestore(t *testing.T) { t.Log("Standby/restore test complete!") } +func readGuestEpoch(t *testing.T, ctx context.Context, inst *Instance) int64 { + t.Helper() + out, code, err := execInInstance(ctx, inst, "date", "+%s") + require.NoError(t, err) + require.Equal(t, 0, code, "date should succeed in guest: %s", out) + epoch, err := strconv.ParseInt(strings.TrimSpace(out), 10, 64) + require.NoError(t, err, "parse guest epoch from %q", out) + return epoch +} + func TestCloudHypervisorSnapshotFeature(t *testing.T) { t.Parallel() if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {