Feat/exclude drift (#2650)

* feat: include and exclude driff patterns

* chore: dep

* fix: truncate chars over gh limit

* docs: update digger reference to include new drift includes

* docs: phrasing

Author: s1ntaxe770r

backend/go.sum

@@ -1,7 +1,5 @@
 ariga.io/atlas-go-sdk v0.7.2 h1:pvS8tKVeRQuqdETBqj5qAQtVbQE88Gya6bOfY8YF3vU=
 ariga.io/atlas-go-sdk v0.7.2/go.mod h1:cFq7bnvHgKTWHCsU46mtkGxdl41rx2o7SjaLoh6cO8M=
-ariga.io/atlas-provider-gorm v0.5.0 h1:DqYNWroKUiXmx2N6nf/I9lIWu6fpgB6OQx/JoelCTes=
-ariga.io/atlas-provider-gorm v0.5.0/go.mod h1:8m6+N6+IgWMzPcR63c9sNOBoxfNk6yV6txBZBrgLg1o=
 ariga.io/atlas-provider-gorm v0.5.4 h1:64xboUDrP+JHdZOy4juPydHT5UP1kY152b5Gh/xNzmM=
 ariga.io/atlas-provider-gorm v0.5.4/go.mod h1:cXt4kxq8KIldPXHoWXC0HvSr8dVI0dIykZt3MZ4AmqE=
 c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805 h1:u2qwJeEvnypw+OCPUHmoZE3IqwfuN5kgDfo5MLzpNM0=
@@ -759,10 +757,6 @@ github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm
 github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
-github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
-github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
-github.com/alecthomas/kong v1.9.0 h1:Wgg0ll5Ys7xDnpgYBuBn/wPeLGAuK0NvYmEcisJgrIs=
-github.com/alecthomas/kong v1.9.0/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -2847,7 +2841,6 @@ gorm.io/gorm v1.23.7/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 gorm.io/gorm v1.23.10/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
 gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
-gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
 gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
 gorm.io/gorm v1.30.1 h1:lSHg33jJTBxs2mgJRfRZeLDG+WZaHYCk3Wtfl6Ngzo4=
 gorm.io/gorm v1.30.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=

cli/pkg/drift/github_issue.go

@@ -16,6 +16,11 @@ func (ghi *GithubIssueNotification) SendNotificationForProject(projectName strin
     log.Printf("Info: Sending drift notification regarding project: %v", projectName)
     title := fmt.Sprintf("Drift detected in project: %v", projectName)
     message := fmt.Sprintf(":bangbang: Drift detected in digger project %v details below: \n\n```\n%v\n```", projectName, plan)
+    const maxLen = 65536
+    const truncMsg = "\n\n> ⚠️ Output truncated: plan exceeds GitHub's 65536 character limit. See job logs for full output."
+    if len(message) > maxLen {
+        message = message[:maxLen-len(truncMsg)] + truncMsg
+    }
     existingIssues, err := (*ghi.GithubService).ListIssues()
     if err != nil {
         log.Printf("failed to retrieve issues: %v", err)

cli/pkg/github/github.go

@@ -215,6 +215,12 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
 					continue
 				}
 			}
+			if len(diggerConfig.DriftIncludePatterns) > 0 || len(diggerConfig.DriftExcludePatterns) > 0 {
+				if !digger_config.MatchIncludeExcludePatternsToFile(projectConfig.Dir, diggerConfig.DriftIncludePatterns, diggerConfig.DriftExcludePatterns) {
+					slog.Info("Project excluded by drift patterns, skipping", "project", projectConfig.Name, "dir", projectConfig.Dir)
+					continue
+				}
+			}
 			workflow := diggerConfig.Workflows[projectConfig.Workflow]
 
 			stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true)

docs/ce/drift/backendless-scoping-projects.mdx

@@ -1,11 +1,16 @@
 ---
 title: "Backendless: Scope Drift to Specific Projects"
-description: "Scope backendless drift checks to selected projects using dedicated config files"
+description: "Scope backendless drift checks to selected projects using dedicated config file or patterns"
 ---
 
-In Backendless mode, scope drift by pointing your scheduled workflow at a dedicated config file.
 
-## Approach
+
+
+In Backendless mode, digger provides two means scoping drift detection. 
+
+
+
+## Using dedicated config 
 
 - Create a dedicated `digger.yml` that lists only the projects or blocks you want scanned.
 - Point your drift workflow to that file using the `digger-filename` input.
@@ -57,7 +62,31 @@ jobs:
           digger-filename: digger-drift-dev.yml
 ```
 
+## Using drift patterns
+
+Add `drift_include_patterns` and `drift_exclude_patterns` under `generate_projects` in your main `digger.yml`:
+
+```yaml
+generate_projects:
+  blocks:
+    - block_name: infra
+      root_dir: "infra/"
+      workflow: default
+      include: "**"
+  drift_include_patterns:
+    - "infra/prod/**"
+    - "infra/staging/**"
+  drift_exclude_patterns:
+    - "infra/_global/**"
+```
+
+Only projects whose `dir` matches an include pattern and does not match an exclude pattern  will run drift detection. Exclude patterns are evaluated after include patterns.
+
 ## Notes
 
-- There is no per-project drift filter in the action; scoping via a dedicated config file is the recommended approach.
-- You can also mark projects with `drift_detection: false` in your main config to disable drift checks for them.
+- Patterns use [doublestar](https://github.com/bmatcuk/doublestar) glob matching against the project directory path.
+- Both fields default to `[]`. If `drift_include_patterns` is empty, all projects are included.
+
+## Related
+
+- [Backendless Drift via GitHub Actions](/ce/drift/backendless-github-actions)

docs/ce/reference/digger.yml.mdx

@@ -227,29 +227,17 @@ workflows:
   Workflows and configurations to run on events. See [Workflow Configuration](#workflow-configuration).
 </ParamField>
 
-<AccordionGroup>
-  <Accordion title="Reporting Configuration">
-    Configure reporting options using the `reporting` key.
-
-    <ParamField path="reporting.ai_summary" type="boolean" default="false">
-      Enable AI-generated summaries of plan output. See [AI Summaries](/ce/features/ai-summaries).
-    </ParamField>
-
-    <ParamField path="reporting.comments_enabled" type="boolean" default="true">
-      Enable posting plan/apply results as PR comments.
-    </ParamField>
-  </Accordion>
+<ParamField path="reporting.ai_summary" type="boolean" default="false">
+  Enable AI-generated summaries of plan output. See [AI Summaries](/ce/features/ai-summaries).
+</ParamField>
 
-  <Accordion title="Dependency Configuration">
-    Configure dependency handling using the `dependency_configuration` key.
+<ParamField path="reporting.comments_enabled" type="boolean" default="true">
+  Enable posting plan/apply results as PR comments.
+</ParamField>
 
-    <ParamField path="dependency_configuration.mode" type="string" default="hard">
-      Dependency execution mode:
-      - `hard` - Execute dependency projects even if they weren't changed
-      - `soft` - Skip dependency projects if they weren't changed
-    </ParamField>
-  </Accordion>
-</AccordionGroup>
+<ParamField path="dependency_configuration.mode" type="string" default="hard">
+  Dependency execution mode: `hard` executes dependency projects even if unchanged, `soft` skips them if unchanged.
+</ParamField>
 
 ---
 
@@ -389,6 +377,14 @@ Automatically generate projects from directory structure using the `generate_pro
   Terragrunt-specific parsing configuration. See [Terragrunt Parsing](/ce/reference/terragrunt-parsing) for all options.
 </ParamField>
 
+<ParamField path="drift_include_patterns" type="array" default="[]">
+  Glob patterns matched against each project's `dir`. Only matching projects run drift detection. If empty, all projects are included. See [Backendless: Scope Drift to Specific Projects](/ce/drift/backendless-scoping-projects).
+</ParamField>
+
+<ParamField path="drift_exclude_patterns" type="array" default="[]">
+  Glob patterns matched against each project's `dir`. Matching projects are skipped during drift detection. Evaluated after `drift_include_patterns`.
+</ParamField>
+
 <ParamField path="aws_role_to_assume" type="object">
   Default AWS role configuration for all generated projects. See [AWS Role Configuration](#aws-role-configuration).
 </ParamField>
@@ -512,7 +508,7 @@ Define custom workflows using the `workflows` map. Each workflow can have its ow
     Configure plan and apply stages.
 
     <ParamField path="filter_regex" type="string">
-      Regular expression to filter which files trigger this stage.
+      Regular expression to mask sensitive values from plan output and PR comments. Matches are replaced with `<REDACTED>`. See [Masking sensitive values](/ce/howto/masking-sensitive-values).
     </ParamField>
 
     <ParamField path="steps" type="array" default="[]">

drift/controllers/drift.go

@@ -82,6 +82,16 @@ func (mc MainController) TriggerDriftRunForProject(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("could not find project %v in digger.yml", theProject)})
 		return
 	}
+
+	// Apply drift include/exclude patterns from generate_projects config
+	if len(config.DriftIncludePatterns) > 0 || len(config.DriftExcludePatterns) > 0 {
+		if !dg_configuration.MatchIncludeExcludePatternsToFile(theProject.Dir, config.DriftIncludePatterns, config.DriftExcludePatterns) {
+			log.Printf("Project %v dir %v excluded by drift patterns, skipping", project.Name, theProject.Dir)
+			c.String(http.StatusOK, "project excluded by drift patterns")
+			return
+		}
+	}
+
 	projects := []dg_configuration.Project{*theProject}
 
 	jobsForImpactedProjects, err := generic.CreateJobsForProjects(projects, command, "drift", repoFullName, "digger", config.Workflows, &issueNumber, nil, branch, branch, false)

libs/digger_config/config.go

@@ -31,6 +31,8 @@ type DiggerConfig struct {
 	TraverseToNestedProjects      bool
 	Reporting                     ReporterConfig
 	ReportTerraformOutputs        bool
+	DriftExcludePatterns          []string
+	DriftIncludePatterns          []string
 }
 
 type ReporterConfig struct {

libs/digger_config/converters.go

@@ -292,6 +292,11 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml) (*DiggerConfig, gra
 		diggerConfig.MentionDriftedProjectsInPR = false
 	}
 
+	if diggerYaml.GenerateProjectsConfig != nil {
+		diggerConfig.DriftExcludePatterns = diggerYaml.GenerateProjectsConfig.DriftExcludePatterns
+		diggerConfig.DriftIncludePatterns = diggerYaml.GenerateProjectsConfig.DriftIncludePatterns
+	}
+
 	if diggerYaml.PrLocks != nil {
 		diggerConfig.PrLocks = *diggerYaml.PrLocks
 	} else {

libs/digger_config/yaml.go

@@ -159,6 +159,8 @@ type GenerateProjectsConfigYaml struct {
 	TerragruntParsingConfig *TerragruntParsingConfig    `yaml:"terragrunt_parsing,omitempty"`
 	AwsRoleToAssume         *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
 	AwsCognitoOidcConfig    *AwsCognitoOidcConfig       `yaml:"aws_cognito_oidc,omitempty"`
+	DriftExcludePatterns    []string                    `yaml:"drift_exclude_patterns,omitempty"`
+	DriftIncludePatterns    []string                    `yaml:"drift_include_patterns,omitempty"`
 }
 
 type TerragruntParsingConfig struct {