From 38dea8a73c9484bb19acd7a9b49132919caad7ab Mon Sep 17 00:00:00 2001
From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com>
Date: Tue, 7 Jul 2026 16:17:53 +0000
Subject: [PATCH] Add lifecycle tracing spans to instance delete path

The delete path runs the same teardown sequence as stop (graceful guest
shutdown, hypervisor kill, network release, data removal) but emitted no
step spans, so slow deletes showed up in traces as one opaque server
span. Wrap each blocking phase in the existing lifecycle step helpers,
mirroring the instrumentation already present in stop.go.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
---
 lib/instances/delete.go | 56 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 50 insertions(+), 6 deletions(-)

diff --git a/lib/instances/delete.go b/lib/instances/delete.go
index 90236809..7e175c4f 100644
--- a/lib/instances/delete.go
+++ b/lib/instances/delete.go
@@ -12,6 +12,7 @@ import (
 	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 	"github.com/kernel/hypeman/lib/network"
+	"go.opentelemetry.io/otel/attribute"
 )
 
 const deleteGracefulShutdownTimeout = 2
@@ -20,10 +21,16 @@ const deleteGracefulShutdownTimeout = 2
 func (m *manager) deleteInstance(
 	ctx context.Context,
 	id string,
-) error {
+) (retErr error) {
 	log := logger.FromContext(ctx)
 	log.InfoContext(ctx, "deleting instance", "instance_id", id)
 
+	ctx, span := m.startLifecycleSpan(ctx, "instances.delete",
+		attribute.String("instance_id", id),
+		attribute.String("operation", "delete"),
+	)
+	defer func() { finishInstancesSpan(span, retErr) }()
+
 	// 1. Load instance
 	meta, err := m.loadMetadata(id)
 	if err != nil {
@@ -33,9 +40,18 @@ func (m *manager) deleteInstance(
 
 	inst := m.toInstance(ctx, meta)
 	stored := &meta.StoredMetadata
+	ctx = enrichInstancesTrace(ctx,
+		attribute.String("hypervisor", string(stored.HypervisorType)),
+		attribute.String("instance_state", string(inst.State)),
+	)
 	log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State)
 
-	target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id))
+	compressionCtx, compressionSpanEnd := m.startLifecycleStep(ctx, "cancel_compression_job",
+		attribute.String("instance_id", id),
+		attribute.String("operation", "cancel_compression_job"),
+	)
+	target, err := m.cancelAndWaitCompressionJob(compressionCtx, m.snapshotJobKeyForInstance(id))
+	compressionSpanEnd(err)
 	if err != nil {
 		return fmt.Errorf("wait for instance compression to stop: %w", err)
 	}
@@ -65,8 +81,16 @@ func (m *manager) deleteInstance(
 		if stopTimeout > deleteGracefulShutdownTimeout {
 			stopTimeout = deleteGracefulShutdownTimeout
 		}
-		gracefulShutdown = m.tryGracefulGuestShutdown(ctx, &inst, stopTimeout)
-		if !gracefulShutdown {
+		gracefulCtx, gracefulSpanEnd := m.startLifecycleStep(ctx, "graceful_guest_shutdown",
+			attribute.String("instance_id", id),
+			attribute.String("hypervisor", string(stored.HypervisorType)),
+			attribute.String("operation", "graceful_guest_shutdown"),
+		)
+		gracefulShutdown = m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout)
+		if gracefulShutdown {
+			gracefulSpanEnd(nil)
+		} else {
+			gracefulSpanEnd(errGracefulShutdownFailed)
 			log.DebugContext(ctx, "graceful shutdown before delete did not complete", "instance_id", id)
 		}
 	}
@@ -75,7 +99,14 @@ func (m *manager) deleteInstance(
 	// Also attempt kill for StateUnknown since we can't be sure if hypervisor is running
 	if !gracefulShutdown && (inst.State.RequiresVMM() || inst.State == StateUnknown) {
 		log.DebugContext(ctx, "stopping hypervisor", "instance_id", id, "state", inst.State)
-		if err := m.killHypervisor(ctx, &inst); err != nil {
+		killCtx, killSpanEnd := m.startLifecycleStep(ctx, "force_kill_hypervisor",
+			attribute.String("instance_id", id),
+			attribute.String("hypervisor", string(stored.HypervisorType)),
+			attribute.String("operation", "force_kill_hypervisor"),
+		)
+		err := m.killHypervisor(killCtx, &inst)
+		killSpanEnd(err)
+		if err != nil {
 			// Log error but continue with cleanup
 			// Best effort to clean up even if hypervisor is unresponsive
 			log.WarnContext(ctx, "failed to kill hypervisor, continuing with cleanup", "instance_id", id, "error", err)
@@ -87,7 +118,14 @@ func (m *manager) deleteInstance(
 	if inst.NetworkEnabled {
 		m.unregisterEgressProxyInstance(ctx, id)
 		log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default")
-		if err := m.networkManager.ReleaseAllocation(ctx, networkAlloc); err != nil {
+		releaseNetworkCtx, releaseNetworkSpanEnd := m.startLifecycleStep(ctx, "release_network",
+			attribute.String("instance_id", id),
+			attribute.String("hypervisor", string(stored.HypervisorType)),
+			attribute.String("operation", "release_network"),
+		)
+		err := m.networkManager.ReleaseAllocation(releaseNetworkCtx, networkAlloc)
+		releaseNetworkSpanEnd(err)
+		if err != nil {
 			// Log error but continue with cleanup
 			log.WarnContext(ctx, "failed to release network, continuing with cleanup", "instance_id", id, "error", err)
 		}
@@ -132,10 +170,16 @@ func (m *manager) deleteInstance(
 
 	// 8. Delete all instance data
 	log.DebugContext(ctx, "deleting instance data", "instance_id", id)
+	_, dataSpanEnd := m.startLifecycleStep(ctx, "delete_instance_data",
+		attribute.String("instance_id", id),
+		attribute.String("operation", "delete_instance_data"),
+	)
 	if err := m.deleteInstanceData(id); err != nil {
+		dataSpanEnd(err)
 		log.ErrorContext(ctx, "failed to delete instance data", "instance_id", id, "error", err)
 		return fmt.Errorf("delete instance data: %w", err)
 	}
+	dataSpanEnd(nil)
 
 	log.InfoContext(ctx, "instance deleted successfully", "instance_id", id)
 	return nil