rego: Introduce metadata rollback and enhance handling of mount failures in C-LCOW

internal/guest/storage/mount.go

@@ -127,6 +127,7 @@ func UnmountPath(ctx context.Context, target string, removeTarget bool) (err err
 
 	if _, err := osStat(target); err != nil {
 		if os.IsNotExist(err) {
+			log.G(ctx).WithField("target", target).Warnf("UnmountPath called for non-existent path")
 			return nil
 		}
 		return errors.Wrapf(err, "failed to determine if path '%s' exists", target)

internal/guest/storage/overlay/overlay.go

@@ -56,8 +56,7 @@ func processErrNoSpace(ctx context.Context, path string, err error) {
 	}).WithError(err).Warn("got ENOSPC, gathering diagnostics")
 }
 
-// MountLayer first enforces the security policy for the container's layer paths
-// and then calls Mount to mount the layer paths as an overlayfs.
+// MountLayer calls Mount to mount the layer paths as an overlayfs.
 func MountLayer(
 	ctx context.Context,
 	layerPaths []string,

internal/guest/storage/scsi/scsi.go

@@ -121,8 +121,9 @@ type Config struct {
 // Mount creates a mount from the SCSI device on `controller` index `lun` to
 // `target`
 //
-// `target` will be created. On mount failure the created `target` will be
-// automatically cleaned up.
+// `target` will be created. On mount failure the created `target`, as well as
+// any associated dm-crypt or dm-verify devices will be automatically cleaned
+// up.
 //
 // If the config has `encrypted` is set to true, the SCSI device will be
 // encrypted using dm-crypt.
@@ -200,7 +201,8 @@ func Mount(
 	var deviceFS string
 	if config.Encrypted {
 		cryptDeviceName := fmt.Sprintf(cryptDeviceFmt, controller, lun, partition)
-		encryptedSource, err := encryptDevice(spnCtx, source, cryptDeviceName)
+		var encryptedSource string
+		encryptedSource, err = encryptDevice(spnCtx, source, cryptDeviceName)
 		if err != nil {
 			// todo (maksiman): add better retry logic, similar to how SCSI device mounts are
 			// retried on unix.ENOENT and unix.ENXIO. The retry should probably be on an
@@ -211,6 +213,13 @@ func Mount(
 			}
 		}
 		source = encryptedSource
+		defer func() {
+			if err != nil {
+				if err := cleanupCryptDevice(spnCtx, cryptDeviceName); err != nil {
+					log.G(spnCtx).WithError(err).WithField("cryptDeviceName", cryptDeviceName).Debug("failed to cleanup dm-crypt device after mount failure")
+				}
+			}
+		}()
 	} else {
 		// Get the filesystem that is already on the device (if any) and use that
 		// as the mountType unless `Filesystem` was given.

internal/guest/storage/scsi/scsi_test.go

@@ -999,6 +999,12 @@ func Test_Mount_EncryptDevice_Mkfs_Error(t *testing.T) {
 		}
 		return expectedDevicePath, nil
 	}
+	cleanupCryptDevice = func(_ context.Context, dmCryptName string) error {
+		if dmCryptName != expectedCryptTarget {
+			t.Fatalf("expected cleanupCryptDevice name %q got %q", expectedCryptTarget, dmCryptName)
+		}
+		return nil
+	}
 	osStat = osStatNoop
 
 	xfsFormat = func(arg string) error {

internal/regopolicyinterpreter/regopolicyinterpreter.go

@@ -63,6 +63,9 @@ type RegoModule struct {
 
 type regoMetadata map[string]map[string]interface{}
 
+const metadataRootKey = "metadata"
+const metadataOperationsKey = "metadata"
+
 type regoMetadataAction string
 
 const (
@@ -81,6 +84,11 @@ type regoMetadataOperation struct {
 // The result from a policy query
 type RegoQueryResult map[string]interface{}
 
+// An immutable, saved copy of the metadata state.
+type SavedMetadata struct {
+	metadataRoot regoMetadata
+}
+
 // deep copy for an object
 func copyObject(data map[string]interface{}) (map[string]interface{}, error) {
 	objJSON, err := json.Marshal(data)
@@ -113,6 +121,24 @@ func copyValue(value interface{}) (interface{}, error) {
 	return valueCopy, nil
 }
 
+// deep copy for regoMetadata.
+// We cannot use copyObject for this due to the fact that map[string]interface{}
+// is a concrete type and a map of it cannot be used as a map of interface{}.
+func copyRegoMetadata(value regoMetadata) (regoMetadata, error) {
+	valueJSON, err := json.Marshal(value)
+	if err != nil {
+		return nil, err
+	}
+
+	var valueCopy regoMetadata
+	err = json.Unmarshal(valueJSON, &valueCopy)
+	if err != nil {
+		return nil, err
+	}
+
+	return valueCopy, nil
+}
+
 // NewRegoPolicyInterpreter creates a new RegoPolicyInterpreter, using the code provided.
 // inputData is the Rego data which should be used as the initial state
 // of the interpreter. A deep copy is performed on it such that it will
@@ -123,8 +149,8 @@ func NewRegoPolicyInterpreter(code string, inputData map[string]interface{}) (*R
 		return nil, fmt.Errorf("unable to copy the input data: %w", err)
 	}
 
-	if _, ok := data["metadata"]; !ok {
-		data["metadata"] = make(regoMetadata)
+	if _, ok := data[metadataRootKey]; !ok {
+		data[metadataRootKey] = make(regoMetadata)
 	}
 
 	policy := &RegoPolicyInterpreter{
@@ -207,7 +233,7 @@ func (r *RegoPolicyInterpreter) GetMetadata(name string, key string) (interface{
 	r.dataAndModulesMutex.Lock()
 	defer r.dataAndModulesMutex.Unlock()
 
-	metadataRoot, ok := r.data["metadata"].(regoMetadata)
+	metadataRoot, ok := r.data[metadataRootKey].(regoMetadata)
 	if !ok {
 		return nil, errors.New("illegal interpreter state: invalid metadata object type")
 	}
@@ -228,6 +254,32 @@ func (r *RegoPolicyInterpreter) GetMetadata(name string, key string) (interface{
 	}
 }
 
+// Saves a copy of the internal policy metadata state.
+func (r *RegoPolicyInterpreter) SaveMetadata() (s SavedMetadata, err error) {
+	r.dataAndModulesMutex.Lock()
+	defer r.dataAndModulesMutex.Unlock()
+
+	metadataRoot, ok := r.data[metadataRootKey].(regoMetadata)
+	if !ok {
+		return SavedMetadata{}, errors.New("illegal interpreter state: invalid metadata object type")
+	}
+	s.metadataRoot, err = copyRegoMetadata(metadataRoot)
+	return s, err
+}
+
+// Restores a previously saved metadata state.
+func (r *RegoPolicyInterpreter) RestoreMetadata(m SavedMetadata) error {
+	r.dataAndModulesMutex.Lock()
+	defer r.dataAndModulesMutex.Unlock()
+
+	copied, err := copyRegoMetadata(m.metadataRoot)
+	if err != nil {
+		return fmt.Errorf("unable to copy metadata: %w", err)
+	}
+	r.data[metadataRootKey] = copied
+	return nil
+}
+
 func newRegoMetadataOperation(operation interface{}) (*regoMetadataOperation, error) {
 	var metadataOp regoMetadataOperation
 
@@ -286,7 +338,7 @@ func (r *RegoPolicyInterpreter) UpdateOSType(os string) error {
 func (r *RegoPolicyInterpreter) updateMetadata(ops []*regoMetadataOperation) error {
 	// dataAndModulesMutex must be held before calling this
 
-	metadataRoot, ok := r.data["metadata"].(regoMetadata)
+	metadataRoot, ok := r.data[metadataRootKey].(regoMetadata)
 	if !ok {
 		return errors.New("illegal interpreter state: invalid metadata object type")
 	}
@@ -431,7 +483,7 @@ func (r *RegoPolicyInterpreter) logMetadata() {
 		return
 	}
 
-	contents, err := json.Marshal(r.data["metadata"])
+	contents, err := json.Marshal(r.data[metadataRootKey])
 	if err != nil {
 		r.metadataLogger.Printf("error marshaling metadata: %v\n", err.Error())
 	} else {
@@ -637,7 +689,7 @@ func (r *RegoPolicyInterpreter) Query(rule string, input map[string]interface{})
 	r.logResult(rule, resultSet)
 
 	ops := []*regoMetadataOperation{}
-	if rawMetadata, ok := resultSet["metadata"]; ok {
+	if rawMetadata, ok := resultSet[metadataOperationsKey]; ok {
 		metadata, ok := rawMetadata.([]interface{})
 		if !ok {
 			return nil, errors.New("error loading metadata array: invalid type")
@@ -660,7 +712,7 @@ func (r *RegoPolicyInterpreter) Query(rule string, input map[string]interface{})
 	}
 
 	for name, value := range resultSet {
-		if name == "metadata" {
+		if name == metadataOperationsKey {
 			continue
 		} else {
 			result[name] = value