refactor(guardrails): single Presidio image, native VIN, per-rule redaction language

- collapse the analyzer/anonymizer URLs into one PRESIDIO_URL (combined image
  serves /analyze + /anonymize)
- remove the TS VIN recognizer (vin.ts, recognizers.ts) — VIN is now native +
  multi-language in the image; validate_pii is a thin analyze→anonymize client
- trim KR_RRN/TH_TNIN from the catalog (no Korean/Thai model in the image)
- add per-rule redaction language: PII_LANGUAGES catalog drives the contract enum,
  the Data Retention rule modal, and the guardrails block dropdown; resolver +
  logger thread it through to maskPIIBatch (default en), so non-English entity
  rules (e.g. ES_NIF) actually fire instead of silently no-op'ing under en

Author: TheodoreSpeaks

apps/sim/app/api/organizations/[id]/data-retention/route.ts

@@ -16,6 +16,14 @@ import { isOrganizationOnEnterprisePlan } from '@/lib/billing/core/subscription'
 import { isBillingEnabled } from '@/lib/core/config/env-flags'
 import { isFeatureEnabled } from '@/lib/core/config/feature-flags'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { PII_LANGUAGE_CODES, type PIILanguage } from '@/lib/guardrails/pii-entities'
+
+/** Narrow a stored (loosely-typed) language to the supported set; unknown ⇒ undefined (defaults to en). */
+function coercePiiLanguage(value: string | undefined): PIILanguage | undefined {
+  return value && (PII_LANGUAGE_CODES as readonly string[]).includes(value)
+    ? (value as PIILanguage)
+    : undefined
+}
 
 const logger = createLogger('DataRetentionAPI')
 
@@ -35,7 +43,14 @@ function normalizeConfigured(
     logRetentionHours: settings?.logRetentionHours ?? null,
     softDeleteRetentionHours: settings?.softDeleteRetentionHours ?? null,
     taskCleanupHours: settings?.taskCleanupHours ?? null,
-    piiRedaction: settings?.piiRedaction?.rules ? { rules: settings.piiRedaction.rules } : null,
+    piiRedaction: settings?.piiRedaction?.rules
+      ? {
+          rules: settings.piiRedaction.rules.map((rule) => ({
+            ...rule,
+            language: coercePiiLanguage(rule.language),
+          })),
+        }
+      : null,
   }
 }
 

apps/sim/blocks/blocks/guardrails.ts

@@ -1,5 +1,5 @@
 import { ShieldCheckIcon } from '@/components/icons'
-import { PII_ENTITY_GROUPS } from '@/lib/guardrails/pii-entities'
+import { PII_ENTITY_GROUPS, PII_LANGUAGES } from '@/lib/guardrails/pii-entities'
 import type { BlockConfig } from '@/blocks/types'
 import {
   getModelOptions,
@@ -206,13 +206,7 @@ Return ONLY the regex pattern - no explanations, no quotes, no forward slashes,
       id: 'piiLanguage',
       title: 'Language',
       type: 'dropdown',
-      options: [
-        { label: 'English', id: 'en' },
-        { label: 'Spanish', id: 'es' },
-        { label: 'Italian', id: 'it' },
-        { label: 'Polish', id: 'pl' },
-        { label: 'Finnish', id: 'fi' },
-      ],
+      options: PII_LANGUAGES.map((language) => ({ label: language.label, id: language.value })),
       defaultValue: 'en',
       condition: {
         field: 'validationType',

apps/sim/ee/data-retention/components/data-retention-settings.tsx

@@ -21,7 +21,13 @@ import {
 } from '@/components/emcn'
 import { useSession } from '@/lib/auth/auth-client'
 import { isBillingEnabled } from '@/lib/core/config/env-flags'
-import { PII_ENTITY_GROUPS, SUPPORTED_PII_ENTITIES } from '@/lib/guardrails/pii-entities'
+import {
+  DEFAULT_PII_LANGUAGE,
+  PII_ENTITY_GROUPS,
+  PII_LANGUAGES,
+  type PIILanguage,
+  SUPPORTED_PII_ENTITIES,
+} from '@/lib/guardrails/pii-entities'
 import { getUserRole } from '@/lib/workspaces/organization/utils'
 import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section'
 import { InfoNote } from '@/ee/components/info-note'
@@ -59,6 +65,7 @@ interface RuleDraft {
   id: string
   entityTypes: string[]
   workspaceId: string | null
+  language: PIILanguage
 }
 
 function hoursToDisplayDays(hours: number | null): string {
@@ -75,6 +82,7 @@ function normalizeRule(rule: RuleDraft): string {
   return JSON.stringify({
     entityTypes: [...rule.entityTypes].sort(),
     workspaceId: rule.workspaceId,
+    language: rule.language,
   })
 }
 
@@ -227,6 +235,18 @@ function RuleModal({
             onChange={(entityTypes) => onChange({ ...draft, entityTypes })}
           />
         </ChipModalField>
+        <ChipModalField
+          type='custom'
+          title='Language'
+          hint='Detection runs with this language’s recognizers — match it to your log content.'
+        >
+          <ChipSelect
+            value={draft.language}
+            onChange={(language) => onChange({ ...draft, language: language as PIILanguage })}
+            options={PII_LANGUAGES.map((l) => ({ value: l.value, label: l.label }))}
+            align='start'
+          />
+        </ChipModalField>
       </ChipModalBody>
       <ChipModalFooter
         onCancel={onClose}
@@ -291,6 +311,7 @@ export function DataRetentionSettings() {
         id: r.id,
         entityTypes: r.entityTypes,
         workspaceId: r.workspaceId,
+        language: r.language ?? DEFAULT_PII_LANGUAGE,
       }))
     )
     hydratedOrgRef.current = orgId
@@ -327,6 +348,7 @@ export function DataRetentionSettings() {
             id: r.id,
             entityTypes: r.entityTypes,
             workspaceId: r.workspaceId,
+            language: r.language,
           })),
         },
       },
@@ -335,7 +357,12 @@ export function DataRetentionSettings() {
   }
 
   function openEditDefault() {
-    const rule: RuleDraft = defaultRule ?? { id: generateId(), entityTypes: [], workspaceId: null }
+    const rule: RuleDraft = defaultRule ?? {
+      id: generateId(),
+      entityTypes: [],
+      workspaceId: null,
+      language: DEFAULT_PII_LANGUAGE,
+    }
     setModalIsNew(defaultRule === null)
     setModalOriginal(rule)
     setModalDraft({ ...rule })
@@ -344,7 +371,12 @@ export function DataRetentionSettings() {
   function openAddOverride() {
     const workspaceId = freeWorkspaces[0]?.value
     if (!workspaceId) return
-    const blank: RuleDraft = { id: generateId(), entityTypes: [], workspaceId }
+    const blank: RuleDraft = {
+      id: generateId(),
+      entityTypes: [],
+      workspaceId,
+      language: DEFAULT_PII_LANGUAGE,
+    }
     setModalIsNew(true)
     setModalOriginal(blank)
     setModalDraft(blank)

apps/sim/lib/api/contracts/primitives.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod'
+import { PII_LANGUAGE_CODES } from '@/lib/guardrails/pii-entities'
 
 export const unknownRecordSchema = z.record(z.string(), z.unknown())
 
@@ -93,6 +94,8 @@ export const piiRedactionRuleSchema = z.object({
   entityTypes: z.array(z.string().min(1, 'Entity type cannot be empty')).max(100),
   /** null = all workspaces; otherwise the single targeted workspace. */
   workspaceId: z.string().min(1).nullable(),
+  /** Language whose Presidio recognizers apply; defaults to English. */
+  language: z.enum(PII_LANGUAGE_CODES).optional(),
 })
 
 export type PiiRedactionRule = z.output<typeof piiRedactionRuleSchema>

apps/sim/lib/billing/retention.test.ts

@@ -21,40 +21,55 @@ describe('resolveEffectivePiiRedaction', () => {
       orgSettings: settings([allRule]),
       workspaceId: 'ws-1',
     })
-    expect(result).toEqual({ enabled: true, entityTypes: ['EMAIL_ADDRESS', 'PHONE_NUMBER'] })
+    expect(result).toEqual({
+      enabled: true,
+      entityTypes: ['EMAIL_ADDRESS', 'PHONE_NUMBER'],
+      language: 'en',
+    })
   })
 
   it('lets a workspace-specific rule override the all rule', () => {
     const result = resolveEffectivePiiRedaction({
       orgSettings: settings([allRule, { id: 'r-1', entityTypes: ['US_SSN'], workspaceId: 'ws-1' }]),
       workspaceId: 'ws-1',
     })
-    expect(result).toEqual({ enabled: true, entityTypes: ['US_SSN'] })
+    expect(result).toEqual({ enabled: true, entityTypes: ['US_SSN'], language: 'en' })
+  })
+
+  it('carries the rule language through (defaults to en)', () => {
+    const result = resolveEffectivePiiRedaction({
+      orgSettings: settings([
+        { id: 'r-es', entityTypes: ['ES_NIF'], workspaceId: 'ws-1', language: 'es' },
+      ]),
+      workspaceId: 'ws-1',
+    })
+    expect(result).toEqual({ enabled: true, entityTypes: ['ES_NIF'], language: 'es' })
   })
 
   it('exempts a workspace when its specific rule has no entity types', () => {
     const result = resolveEffectivePiiRedaction({
       orgSettings: settings([allRule, { id: 'r-1', entityTypes: [], workspaceId: 'ws-1' }]),
       workspaceId: 'ws-1',
     })
-    expect(result).toEqual({ enabled: false, entityTypes: [] })
+    expect(result).toEqual({ enabled: false, entityTypes: [], language: 'en' })
   })
 
   it('is disabled when no rule matches and there is no all rule', () => {
     const result = resolveEffectivePiiRedaction({
       orgSettings: settings([{ id: 'r-1', entityTypes: ['US_SSN'], workspaceId: 'ws-2' }]),
       workspaceId: 'ws-1',
     })
-    expect(result).toEqual({ enabled: false, entityTypes: [] })
+    expect(result).toEqual({ enabled: false, entityTypes: [], language: 'en' })
   })
 
   it('is disabled when there are no rules', () => {
     expect(
       resolveEffectivePiiRedaction({ orgSettings: settings([]), workspaceId: 'ws-1' })
-    ).toEqual({ enabled: false, entityTypes: [] })
+    ).toEqual({ enabled: false, entityTypes: [], language: 'en' })
     expect(resolveEffectivePiiRedaction({ orgSettings: null, workspaceId: 'ws-1' })).toEqual({
       enabled: false,
       entityTypes: [],
+      language: 'en',
     })
   })
 })

apps/sim/lib/billing/retention.ts

@@ -1,14 +1,18 @@
 import type { DataRetentionSettings } from '@sim/db/schema'
+import { DEFAULT_PII_LANGUAGE } from '@/lib/guardrails/pii-entities'
 
 export interface EffectivePiiRedaction {
   enabled: boolean
   /** Presidio entity types to mask. Empty = redact all detected PII. */
   entityTypes: string[]
+  /** Language whose Presidio recognizers apply when masking. */
+  language: string
 }
 
 export const DEFAULT_PII_REDACTION: EffectivePiiRedaction = {
   enabled: false,
   entityTypes: [],
+  language: DEFAULT_PII_LANGUAGE,
 }
 
 /**
@@ -34,5 +38,6 @@ export function resolveEffectivePiiRedaction(params: {
     ? rule.entityTypes.filter((t): t is string => typeof t === 'string')
     : []
   if (types.length === 0) return DEFAULT_PII_REDACTION
-  return { enabled: true, entityTypes: types }
+  const language = typeof rule?.language === 'string' ? rule.language : DEFAULT_PII_LANGUAGE
+  return { enabled: true, entityTypes: types, language }
 }

apps/sim/lib/core/config/env.ts

@@ -311,8 +311,7 @@ export const env = createEnv({
     PORT:                                  z.number().optional(),                  // Main application port
     INTERNAL_API_BASE_URL:                 z.string().optional(),                  // Optional internal base URL for server-side self-calls; must include protocol if set (e.g., http://sim-app.namespace.svc.cluster.local:3000)
     ALLOWED_ORIGINS:                       z.string().optional(),                  // CORS allowed origins
-    PRESIDIO_ANALYZER_URL:                 z.string().optional(),                  // Presidio analyzer sidecar base URL for PII detection (default http://localhost:5002)
-    PRESIDIO_ANONYMIZER_URL:               z.string().optional(),                  // Presidio anonymizer sidecar base URL for PII masking (default http://localhost:5001)
+    PRESIDIO_URL:                          z.string().optional(),                  // Presidio sidecar base URL serving /analyze + /anonymize (default http://localhost:5002)
 
     // OAuth Integration Credentials - All optional, enables third-party integrations
     GOOGLE_CLIENT_ID:                      z.string().optional(),                  // Google OAuth client ID for Google services

apps/sim/lib/guardrails/pii-entities.ts

@@ -51,8 +51,6 @@ export const SUPPORTED_PII_ENTITIES = {
   IN_VOTER: 'Indian voter ID',
   IN_PASSPORT: 'Indian passport',
   FI_PERSONAL_IDENTITY_CODE: 'Finnish Personal Identity Code',
-  KR_RRN: 'Korean Resident Registration Number',
-  TH_TNIN: 'Thai National ID Number',
 } as const
 
 export type PIIEntityType = keyof typeof SUPPORTED_PII_ENTITIES
@@ -115,8 +113,6 @@ export const PII_ENTITY_GROUPS: ReadonlyArray<{
       'IN_VOTER',
       'IN_PASSPORT',
       'FI_PERSONAL_IDENTITY_CODE',
-      'KR_RRN',
-      'TH_TNIN',
     ],
   },
 ].map((group) => ({
@@ -126,3 +122,26 @@ export const PII_ENTITY_GROUPS: ReadonlyArray<{
     label: SUPPORTED_PII_ENTITIES[value as PIIEntityType],
   })),
 }))
+
+/**
+ * Languages the Presidio image has NLP models for. The analyzer only recognizes a
+ * language's entities when its model is loaded, so this set must match the image.
+ */
+export const PII_LANGUAGES = [
+  { value: 'en', label: 'English' },
+  { value: 'es', label: 'Spanish' },
+  { value: 'it', label: 'Italian' },
+  { value: 'pl', label: 'Polish' },
+  { value: 'fi', label: 'Finnish' },
+] as const
+
+export type PIILanguage = (typeof PII_LANGUAGES)[number]['value']
+
+/** Non-empty tuple of language codes for schema/enum use. */
+export const PII_LANGUAGE_CODES = PII_LANGUAGES.map((l) => l.value) as [
+  PIILanguage,
+  ...PIILanguage[],
+]
+
+/** Default redaction language when a rule doesn't set one. */
+export const DEFAULT_PII_LANGUAGE: PIILanguage = 'en'