Skip to content

Add lifecycle tracing spans to instance delete path #299

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sjmiller609 merged 1 commit into from
Jul 8, 2026
Merged

Add lifecycle tracing spans to instance delete path #299

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 50 additions & 6 deletions lib/instances/delete.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/kernel/hypeman/lib/hypervisor"
"github.com/kernel/hypeman/lib/logger"
"github.com/kernel/hypeman/lib/network"
"go.opentelemetry.io/otel/attribute"
)

const deleteGracefulShutdownTimeout = 2
Expand All @@ -20,10 +21,16 @@ const deleteGracefulShutdownTimeout = 2
func (m *manager) deleteInstance(
ctx context.Context,
id string,
) error {
) (retErr error) {
log := logger.FromContext(ctx)
log.InfoContext(ctx, "deleting instance", "instance_id", id)

ctx, span := m.startLifecycleSpan(ctx, "instances.delete",
attribute.String("instance_id", id),
attribute.String("operation", "delete"),
)
defer func() { finishInstancesSpan(span, retErr) }()

// 1. Load instance
meta, err := m.loadMetadata(id)
if err != nil {
Expand All @@ -33,9 +40,18 @@ func (m *manager) deleteInstance(

inst := m.toInstance(ctx, meta)
stored := &meta.StoredMetadata
ctx = enrichInstancesTrace(ctx,
attribute.String("hypervisor", string(stored.HypervisorType)),
attribute.String("instance_state", string(inst.State)),
)
log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State)

target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id))
compressionCtx, compressionSpanEnd := m.startLifecycleStep(ctx, "cancel_compression_job",
attribute.String("instance_id", id),
attribute.String("operation", "cancel_compression_job"),
)
target, err := m.cancelAndWaitCompressionJob(compressionCtx, m.snapshotJobKeyForInstance(id))
compressionSpanEnd(err)
if err != nil {
return fmt.Errorf("wait for instance compression to stop: %w", err)
}
Expand Down Expand Up @@ -65,8 +81,16 @@ func (m *manager) deleteInstance(
if stopTimeout > deleteGracefulShutdownTimeout {
stopTimeout = deleteGracefulShutdownTimeout
}
gracefulShutdown = m.tryGracefulGuestShutdown(ctx, &inst, stopTimeout)
if !gracefulShutdown {
gracefulCtx, gracefulSpanEnd := m.startLifecycleStep(ctx, "graceful_guest_shutdown",
attribute.String("instance_id", id),
attribute.String("hypervisor", string(stored.HypervisorType)),
attribute.String("operation", "graceful_guest_shutdown"),
)
gracefulShutdown = m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout)
if gracefulShutdown {
gracefulSpanEnd(nil)
} else {
gracefulSpanEnd(errGracefulShutdownFailed)
log.DebugContext(ctx, "graceful shutdown before delete did not complete", "instance_id", id)
}
}
Expand All @@ -75,7 +99,14 @@ func (m *manager) deleteInstance(
// Also attempt kill for StateUnknown since we can't be sure if hypervisor is running
if !gracefulShutdown && (inst.State.RequiresVMM() || inst.State == StateUnknown) {
log.DebugContext(ctx, "stopping hypervisor", "instance_id", id, "state", inst.State)
if err := m.killHypervisor(ctx, &inst); err != nil {
killCtx, killSpanEnd := m.startLifecycleStep(ctx, "force_kill_hypervisor",
attribute.String("instance_id", id),
attribute.String("hypervisor", string(stored.HypervisorType)),
attribute.String("operation", "force_kill_hypervisor"),
)
err := m.killHypervisor(killCtx, &inst)
killSpanEnd(err)
if err != nil {
// Log error but continue with cleanup
// Best effort to clean up even if hypervisor is unresponsive
log.WarnContext(ctx, "failed to kill hypervisor, continuing with cleanup", "instance_id", id, "error", err)
Expand All @@ -87,7 +118,14 @@ func (m *manager) deleteInstance(
if inst.NetworkEnabled {
m.unregisterEgressProxyInstance(ctx, id)
log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default")
if err := m.networkManager.ReleaseAllocation(ctx, networkAlloc); err != nil {
releaseNetworkCtx, releaseNetworkSpanEnd := m.startLifecycleStep(ctx, "release_network",
attribute.String("instance_id", id),
attribute.String("hypervisor", string(stored.HypervisorType)),
attribute.String("operation", "release_network"),
)
err := m.networkManager.ReleaseAllocation(releaseNetworkCtx, networkAlloc)
releaseNetworkSpanEnd(err)
if err != nil {
// Log error but continue with cleanup
log.WarnContext(ctx, "failed to release network, continuing with cleanup", "instance_id", id, "error", err)
}
Expand Down Expand Up @@ -132,10 +170,16 @@ func (m *manager) deleteInstance(

// 8. Delete all instance data
log.DebugContext(ctx, "deleting instance data", "instance_id", id)
_, dataSpanEnd := m.startLifecycleStep(ctx, "delete_instance_data",
attribute.String("instance_id", id),
attribute.String("operation", "delete_instance_data"),
)
if err := m.deleteInstanceData(id); err != nil {
dataSpanEnd(err)
log.ErrorContext(ctx, "failed to delete instance data", "instance_id", id, "error", err)
return fmt.Errorf("delete instance data: %w", err)
}
dataSpanEnd(nil)

log.InfoContext(ctx, "instance deleted successfully", "instance_id", id)
return nil
Expand Down
Loading