fix(oscal-import): upsert existing documents and fix nested-dir recursion

cmd/oscal/import.go

@@ -3,6 +3,8 @@ package oscal
 import (
 	"context"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"log"
 	"os"
@@ -61,18 +63,122 @@ func importOscal(cmd *cobra.Command, args []string) {
 		panic("failed to connect database")
 	}
 
+	var errs error
 	for _, f := range files {
 		systemFile, err := os.Open(f)
 		if err != nil {
-			panic(err)
+			errs = errors.Join(errs, err)
+			sugar.Errorw("Failed to open import path", "path", f, "error", err)
+			continue
 		}
 
-		err = importFile(db, sugar, systemFile)
-		if err != nil {
-			panic(err)
+		if err := importFile(db, sugar, systemFile); err != nil {
+			errs = errors.Join(errs, err)
 		}
+	}
+	if errs != nil {
+		sugar.Fatalw("Import finished with errors", "error", errs)
+	}
+}
+
+// importResult reports what upsertDocument did with a document.
+type importResult struct {
+	created bool
+	err     error
+}
+
+// upsertDocument creates the document when its uuid is new, and otherwise
+// updates it from the file contents (FullSaveAssociations upserts every
+// nested row by primary key). Re-importing an updated file therefore
+// propagates new controls, requirements and statements instead of being
+// silently skipped.
+//
+// Update semantics are additive: struct-based Updates skips zero-value
+// fields, so DB-managed columns not represented in OSCAL (e.g. a catalog's
+// Active flag) are left untouched, and rows or values removed from the file
+// are NOT deleted or cleared.
+func upsertDocument[T any](db *gorm.DB, rawID string, def *T) importResult {
+	id, err := uuid.Parse(rawID)
+	if err != nil {
+		return importResult{err: err}
+	}
+
+	var existing T
+	err = db.Select("id").First(&existing, "id = ?", id).Error
+	if errors.Is(err, gorm.ErrRecordNotFound) {
+		return importResult{created: true, err: db.Create(def).Error}
+	}
+	if err != nil {
+		return importResult{err: err}
+	}
+	reuseSingletonAssociations(db, id, def)
+	return importResult{err: db.Session(&gorm.Session{FullSaveAssociations: true}).Updates(def).Error}
+}
+
+// reuseSingletonAssociations points has-one association rows that carry no
+// stable OSCAL identifier of their own (metadata, back-matter — their ids are
+// DB-generated) at the existing rows for this document, so an update modifies
+// them in place. Without this, every re-import would insert a duplicate
+// metadata/back-matter row alongside the old one.
+func reuseSingletonAssociations(db *gorm.DB, parentID uuid.UUID, def any) {
+	reuseMetadata := func(parentType string, meta *relational.Metadata) {
+		var existing relational.Metadata
+		if err := db.Select("id").
+			First(&existing, "parent_id = ? AND parent_type = ?", parentID.String(), parentType).
+			Error; err == nil {
+			meta.ID = existing.ID
+		}
+	}
+	reuseBackMatter := func(parentType string, bm *relational.BackMatter) {
+		if bm == nil {
+			return
+		}
+		var existing relational.BackMatter
+		if err := db.Select("id").
+			First(&existing, "parent_id = ? AND parent_type = ?", parentID.String(), parentType).
+			Error; err == nil {
+			bm.ID = existing.ID
+		}
+	}
+
+	switch d := def.(type) {
+	case *relational.Catalog:
+		reuseMetadata("catalogs", &d.Metadata)
+		reuseBackMatter("catalogs", d.BackMatter)
+	case *relational.ComponentDefinition:
+		reuseMetadata("component_definitions", &d.Metadata)
+		reuseBackMatter("component_definitions", &d.BackMatter)
+	case *relational.SystemSecurityPlan:
+		reuseMetadata("system_security_plans", &d.Metadata)
+		reuseBackMatter("system_security_plans", d.BackMatter)
+	case *relational.AssessmentPlan:
+		reuseMetadata("assessment_plans", &d.Metadata)
+		reuseBackMatter("assessment_plans", d.BackMatter)
+	case *relational.AssessmentResult:
+		reuseMetadata("assessment_results", &d.Metadata)
+		reuseBackMatter("assessment_results", d.BackMatter)
+	case *relational.Profile:
+		reuseMetadata("profiles", &d.Metadata)
+		reuseBackMatter("profiles", d.BackMatter)
+	case *relational.PlanOfActionAndMilestones:
+		reuseMetadata("plan_of_action_and_milestones", &d.Metadata)
+		reuseBackMatter("plan_of_action_and_milestones", &d.BackMatter)
+	}
+}
 
+// logImport reports the result and returns the error (if any) so callers can
+// propagate it into the process exit status.
+func logImport(sugar *zap.SugaredLogger, res importResult, kind, title, file string) error {
+	if res.err != nil {
+		sugar.Errorw("Failed to import "+kind, "title", title, "file", file, "error", res.err)
+		return res.err
 	}
+	action := "Updated"
+	if res.created {
+		action = "Created"
+	}
+	sugar.Infow("Successfully "+action+" "+kind, "title", title, "file", file)
+	return nil
 }
 
 func importFile(db *gorm.DB, sugar *zap.SugaredLogger, f *os.File) error {
@@ -88,14 +194,21 @@ func importFile(db *gorm.DB, sugar *zap.SugaredLogger, f *os.File) error {
 			return err
 		}
 
+		// Import every entry even when one fails, and report the failures
+		// together so a bad file surfaces in the exit status without
+		// aborting the rest of the directory.
+		var errs error
 		for _, dirFile := range files {
 			if dirFile.Name()[0:1] == "." {
 				continue
 			}
 
-			systemFile, err := os.Open(path.Join(info.Name(), dirFile.Name()))
+			// Join against the path the directory was opened with (f.Name()),
+			// not its base name (info.Name()), so nested directories resolve.
+			systemFile, err := os.Open(path.Join(f.Name(), dirFile.Name()))
 			if err != nil {
-				panic(err)
+				errs = errors.Join(errs, err)
+				continue
 			}
 			defer func() {
 				err := systemFile.Close()
@@ -104,16 +217,31 @@ func importFile(db *gorm.DB, sugar *zap.SugaredLogger, f *os.File) error {
 				}
 			}()
 
-			err = importFile(db, sugar, systemFile)
-			if err != nil {
-				panic(err)
+			if err := importFile(db, sugar, systemFile); err != nil {
+				errs = errors.Join(errs, err)
 			}
 		}
 
-		return nil
+		return errs
 	}
 
-	sugar.Infow("Importing file", "file", info.Name())
+	return importDocuments(db, sugar, f)
+}
+
+// importDocuments imports the OSCAL documents in a single file. A panic while
+// unmarshalling or persisting (e.g. a malformed uuid in the file — the
+// relational layer uses uuid.MustParse) is converted into an error so one bad
+// file fails the import's exit status without aborting the rest of the batch.
+func importDocuments(db *gorm.DB, sugar *zap.SugaredLogger, f *os.File) (errs error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err := fmt.Errorf("importing %s panicked: %v", f.Name(), r)
+			sugar.Errorw("Failed to import file", "file", f.Name(), "error", err)
+			errs = errors.Join(errs, err)
+		}
+	}()
+
+	sugar.Infow("Importing file", "file", f.Name())
 
 	input := &struct {
 		ComponentDefinition       *oscalTypes_1_1_3.ComponentDefinition       `json:"component-definition"`
@@ -125,8 +253,7 @@ func importFile(db *gorm.DB, sugar *zap.SugaredLogger, f *os.File) error {
 		PlanOfActionAndMilestones *oscalTypes_1_1_3.PlanOfActionAndMilestones `json:"plan-of-action-and-milestones"`
 	}{}
 
-	err = json.NewDecoder(f).Decode(input)
-	if err != nil {
+	if err := json.NewDecoder(f).Decode(input); err != nil {
 		sugar.Error(err)
 	}
 
@@ -135,74 +262,49 @@ func importFile(db *gorm.DB, sugar *zap.SugaredLogger, f *os.File) error {
 	if input.Catalog != nil {
 		def := &relational.Catalog{}
 		def.UnmarshalOscal(*input.Catalog)
-		out := db.FirstOrCreate(def)
-		if out.Error != nil {
-			sugar.Error(out.Error)
-		}
-		sugar.Infow("Successfully Created Catalog", "title", def.Metadata.Title, "file", f.Name())
+		errs = errors.Join(errs, logImport(sugar, upsertDocument(db, input.Catalog.UUID, def), "Catalog", def.Metadata.Title, f.Name()))
 		imported = true
 	}
 
 	if input.ComponentDefinition != nil {
 		def := &relational.ComponentDefinition{}
 		def.UnmarshalOscal(*input.ComponentDefinition)
-		out := db.FirstOrCreate(def)
-		if out.Error != nil {
-			sugar.Error(out.Error)
-		}
-		sugar.Infow("Successfully Created Component Definition", "title", def.Metadata.Title, "file", f.Name())
+		errs = errors.Join(errs, logImport(sugar, upsertDocument(db, input.ComponentDefinition.UUID, def), "Component Definition", def.Metadata.Title, f.Name()))
 		imported = true
 	}
 
 	if input.SystemSecurityPlan != nil {
 		def := &relational.SystemSecurityPlan{}
 		def.UnmarshalOscal(*input.SystemSecurityPlan)
-		out := db.FirstOrCreate(def)
-		if out.Error != nil {
-			sugar.Error(out.Error)
-		}
-		sugar.Infow("Successfully Created System Security Plan", "title", def.Metadata.Title, "file", f.Name())
+		errs = errors.Join(errs, logImport(sugar, upsertDocument(db, input.SystemSecurityPlan.UUID, def), "System Security Plan", def.Metadata.Title, f.Name()))
 		imported = true
 	}
 
 	if input.AssessmentPlan != nil {
 		def := &relational.AssessmentPlan{}
 		def.UnmarshalOscal(*input.AssessmentPlan)
-		out := db.FirstOrCreate(def)
-		if out.Error != nil {
-			panic(out.Error)
-		}
-		sugar.Infow("Successfully Created Assessment Plan", "title", def.Metadata.Title, "file", f.Name())
+		errs = errors.Join(errs, logImport(sugar, upsertDocument(db, input.AssessmentPlan.UUID, def), "Assessment Plan", def.Metadata.Title, f.Name()))
 		imported = true
 	}
 
 	if input.AssessmentResult != nil {
 		def := &relational.AssessmentResult{}
 		def.UnmarshalOscal(*input.AssessmentResult)
-		out := db.FirstOrCreate(def)
-		if out.Error != nil {
-			panic(out.Error)
-		}
-		sugar.Infow("Successfully Created Assessment Result", "title", def.Metadata.Title, "file", f.Name())
+		errs = errors.Join(errs, logImport(sugar, upsertDocument(db, input.AssessmentResult.UUID, def), "Assessment Result", def.Metadata.Title, f.Name()))
 		imported = true
 	}
 
 	if input.Profile != nil {
 		def := &relational.Profile{}
 		def.UnmarshalOscal(*input.Profile)
-		out := db.FirstOrCreate(def)
-		if out.Error != nil {
-			panic(out.Error)
-		}
+		errs = errors.Join(errs, logImport(sugar, upsertDocument(db, input.Profile.UUID, def), "Profile", def.Metadata.Title, f.Name()))
 
 		// Sync ProfileControl pivot table synchronously so errors can be reported
-		_, err := oscal.SyncProfileControls(db, uuid.MustParse(input.Profile.UUID))
-		if err != nil {
+		if _, err := oscal.SyncProfileControls(db, uuid.MustParse(input.Profile.UUID)); err != nil {
 			sugar.Errorw("Failed to sync profile controls", "error", err)
-			return err
+			errs = errors.Join(errs, err)
 		}
 
-		sugar.Infow("Successfully Created Profile", "title", def.Metadata.Title, "file", f.Name())
 		imported = true
 	}
 
@@ -214,30 +316,21 @@ func importFile(db *gorm.DB, sugar *zap.SugaredLogger, f *os.File) error {
 		sugar.Infof("Importing POAM with %d risks, %d observations, %d findings",
 			len(def.Risks), len(def.Observations), len(def.Findings))
 
-		// Create with polymorphic entities
-		out := db.FirstOrCreate(def)
-		if out.Error != nil {
-			sugar.Errorf("Error creating POAM: %v", out.Error)
-			return err
-		}
-		sugar.Infow("Successfully Created Plan of Action and Milestones", "title", def.Metadata.Title, "file", f.Name())
+		errs = errors.Join(errs, logImport(sugar, upsertDocument(db, input.PlanOfActionAndMilestones.UUID, def), "Plan of Action and Milestones", def.Metadata.Title, f.Name()))
 		imported = true
 	}
 
 	if imported {
-		return nil
+		return errs
 	}
 
 	// Reset the file to the beginning. We'll read it again.
-	_, err = f.Seek(0, io.SeekStart)
-	if err != nil {
+	if _, err := f.Seek(0, io.SeekStart); err != nil {
 		return err
 	}
 
 	output := &map[string]any{}
-	decoder := json.NewDecoder(f)
-	err = decoder.Decode(output)
-	if err != nil {
+	if err := json.NewDecoder(f).Decode(output); err != nil {
 		sugar.Error(err)
 		return err
 	}