Feat: Add configurable discovery

CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## v1.5.0
+
+### Added
+- Support for datamasque-python 1.2.1.
+  - `dm discover schema-results` handles matches with no label.
+  - `dm rulesets validate` and `dm libraries validate` print validation
+    errors for invalid YAML.
+- Support for Configurable Discovery:
+  - `dm discover configs` — list, get, defaults, create, delete, validate,
+    and status for discovery configs (`database` or `file`).
+  - `dm discover libraries` — list, get, create, delete, validate, and status
+    for discovery config libraries (untyped; shared by both config types).
+  - `dm discover schema --config <name>` and `dm discover file
+    [--config <name>]` start discovery runs with or without a specific config.
+  - `dm discover config-snapshot <run-id>` downloads the discovery config a run
+    actually used.
+- `dm rulesets status` and `dm libraries status` — show a stored ruleset's or
+  library's validation state and errors.
+- `dm rulesets validate` and `dm discover configs validate` refuse YAML of
+  60 KiB or larger, which the server validates asynchronously; create it and
+  poll `status` instead.
+- Safe Data Preview: `dm discover schema-results` and `dm discover file-report`
+  include `safe_data_preview` in their `--json` output.
+- `dm rulesets generate`, `dm connections update --password`, and the
+  deprecated `dm system import` no longer fail.
+
 ## v1.4.0
 
 ### Added

README.md

@@ -148,7 +148,8 @@ dm rulesets create --name <n> --file rules.yaml --type file  # Force a type
 dm rulesets delete <name> [--type file|database]  # Delete a ruleset
 dm rulesets generate --file request.json          # Generate from schema
 dm rulesets generate --file req.json -o out.yaml  # Generate to file
-dm rulesets validate --file rules.yaml            # Validate against server
+dm rulesets validate --file rules.yaml            # Validate against server (YAML under 60 KiB)
+dm rulesets status <name>                         # Validation status; poll after creating YAML of 60 KiB+
 dm rulesets export-bundle -o bundle.zip           # Export rulesets + libraries + seeds
 dm rulesets import-bundle --file bundle.zip       # Import a previously exported bundle
 dm rulesets import-bundle -f bundle.zip --overwrite-rulesets --overwrite-libraries  # Replace existing entries
@@ -164,6 +165,7 @@ dm libraries create --name <n> --file lib.yaml    # Create/update from file
 dm libraries create --name <n> --file lib.yaml --namespace pii  # With namespace
 dm libraries delete <name>                        # Delete a library
 dm libraries validate <name>                      # Re-validate against current server schema
+dm libraries status <name>                        # Validation status; poll after creating YAML of 60 KiB+
 dm libraries usage <name>                         # Show rulesets using it
 ```
 
@@ -216,11 +218,40 @@ dm users delete <username>                      # Delete a user
 ### Discovery
 
 ```console
-dm discover schema <connection>                 # Start a schema-discovery run
-dm discover schema-results <run-id>             # List schema-discovery results once the run finishes
-dm discover sdd-report <run-id>                 # Sensitive data discovery report
-dm discover db-report <run-id>                  # Database discovery CSV
-dm discover file-report <run-id>                # File discovery report
+dm discover schema <connection>                      # Schema discovery (built-in keyword-driven)
+dm discover schema <connection> --config <name>      # Schema discovery from a saved database config
+dm discover schema-results <run-id>                  # List schema-discovery results once the run finishes
+dm discover file <connection>                        # File data discovery (built-in keyword-driven)
+dm discover file <connection> --config <name>        # File data discovery from a saved file config
+dm discover sdd-report <run-id>                      # Sensitive data discovery report
+dm discover db-report <run-id>                       # Database discovery CSV
+dm discover file-report <run-id>                     # File discovery report
+dm discover config-snapshot <run-id> -o used.yaml    # Download the discovery config a run actually used
+```
+
+#### Discovery configs
+
+```console
+dm discover configs list [--type database|file]                    # List configs
+dm discover configs get <name> [--type database] [--yaml]          # Show details or raw YAML
+dm discover configs defaults [--type database|file] -o cfg.yaml    # Built-in default as a starting point
+dm discover configs create --name <n> --type database -f cfg.yaml  # Create/update from YAML
+dm discover configs delete <name> [--type database]                # Delete a config
+dm discover configs validate -f cfg.yaml --type database           # Validate against server (YAML under 60 KiB)
+dm discover configs status <name> [--type database]                # Validation status; poll after creating YAML of 60 KiB+
+```
+
+#### Discovery config libraries
+
+Libraries are untyped — the same library can be imported by both database and file discovery configs.
+
+```console
+dm discover libraries list
+dm discover libraries get <name> [--namespace org] [--yaml]
+dm discover libraries create --name <n> --namespace org -f lib.yaml
+dm discover libraries delete <name> [--namespace org] [--force]    # --force if imported by configs
+dm discover libraries validate -f lib.yaml
+dm discover libraries status <name> [--namespace org]
 ```
 
 ### Seeds

claude-skills/datamasque-cli/skills/datamasque-cli/SKILL.md

@@ -89,6 +89,14 @@ Pass repeated `--options key=value` for server-side knobs
   then fetch results with `dm discover schema-results <id>` /
   `sdd-report` / `db-report` / `file-report`.
 
+- **Configurable discovery and Safe Data Preview.** Save a discovery config
+  with `dm discover configs create` (start from `dm discover configs defaults`),
+  then run `dm discover schema <connection> --config <name>`. When the config
+  enables in-data discovery with safe data preview, `dm discover schema-results
+  <id> --json` carries a `safe_data_preview` per column — value distributions,
+  patterns, and cardinality worth reading before choosing masks. It is JSON-only; 
+  `file-report --json` exposes the same per locator.
+
 - **`dm rulesets validate --file <file> --type <type>`** runs server-side
   validation without committing the ruleset. Use this before `create`
   when you want a clean failure mode for bad YAML.

pyproject.toml

@@ -12,7 +12,7 @@ requires-python = ">=3.11"
 dependencies = [
     "typer>=0.15.0",
     "tomli-w>=1.0.0",
-    "datamasque-python>=1.0.0,<2",
+    "datamasque-python>=1.1.8,<2",
 ]
 classifiers = [
     "Development Status :: 4 - Beta",

src/datamasque_cli/commands/connections.py

@@ -347,7 +347,11 @@ def update_connection(
     if not updates:
         abort("Pass at least one field to update (e.g. --password, --host).", code=ErrorCode.INVALID_INPUT)
 
-    client.make_request("PATCH", f"/api/connections/{match.id}/", data=updates)
+    payload = dict(updates)
+    if "password" in payload:
+        payload["dbpassword"] = payload.pop("password")
+
+    client.make_request("PATCH", f"/api/connections/{match.id}/", data=payload)
     print_success(f"Connection '{match.name}' updated: {', '.join(updates)}.")
 
 

src/datamasque_cli/commands/discovery.py

@@ -8,12 +8,21 @@
 import typer
 from datamasque.client import DataMasqueClient, RunId
 from datamasque.client.models.connection import ConnectionId
-from datamasque.client.models.discovery import SchemaDiscoveryRequest
+from datamasque.client.models.discovery import (
+    FileDataDiscoveryFromConfigRequest,
+    FileDataDiscoveryRequest,
+    SchemaDiscoveryFromConfigRequest,
+    SchemaDiscoveryRequest,
+)
+from datamasque.client.models.discovery_config import DiscoveryConfigId, DiscoveryConfigType
 
 from datamasque_cli.client import get_client
+from datamasque_cli.commands import discovery_config_libraries, discovery_configs
 from datamasque_cli.output import ErrorCode, abort, print_json, print_success, render_output, should_emit_json
 
 app = typer.Typer(help="Data discovery operations.", no_args_is_help=True)
+app.add_typer(discovery_configs.app, name="configs")
+app.add_typer(discovery_config_libraries.app, name="libraries")
 
 
 def _write_or_echo(content: str, output: Path | None, success_label: str) -> None:
@@ -33,9 +42,40 @@ def _resolve_connection_id(client: DataMasqueClient, name_or_id: str) -> str:
     return str(match.id)
 
 
+def _resolve_discovery_config_id(
+    client: DataMasqueClient, name: str, expected_type: DiscoveryConfigType
+) -> DiscoveryConfigId:
+    """Resolve a discovery config name to its UUID, requiring it to be of `expected_type`."""
+    named = [c for c in client.list_discovery_configs() if c.name == name]
+    matches = [c for c in named if c.config_type is expected_type]
+
+    if not matches:
+        if named:
+            existing = ", ".join(c.config_type.value for c in named)
+            abort(
+                f"Discovery config '{name}' exists as {existing}, "
+                f"but {expected_type.value} discovery needs a {expected_type.value} config.",
+                code=ErrorCode.INVALID_INPUT,
+            )
+        abort(f"Discovery config '{name}' not found.", code=ErrorCode.NOT_FOUND)
+    if len(matches) > 1:
+        options = "\n  ".join(f"id={c.id}" for c in matches)
+        abort(
+            f"Multiple {expected_type.value} discovery configs named '{name}':\n  {options}",
+            code=ErrorCode.AMBIGUOUS,
+        )
+
+    config_id = matches[0].id
+    assert config_id is not None
+    return config_id
+
+
 @app.command("schema")
 def schema_discovery(
     connection: str = typer.Argument(help="Connection name or ID"),
+    config: str | None = typer.Option(
+        None, "--config", "-c", help="Run with a saved database discovery config (configurable discovery)"
+    ),
     profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"),
 ) -> None:
     """Start a schema-discovery run on a connection.
@@ -47,14 +87,54 @@ def schema_discovery(
     client = get_client(profile)
     conn_id = _resolve_connection_id(client, connection)
 
-    request = SchemaDiscoveryRequest(connection=ConnectionId(conn_id))
-    run_id = client.start_schema_discovery_run(request)
+    if config is not None:
+        config_id = _resolve_discovery_config_id(client, config, DiscoveryConfigType.database)
+        from_config = SchemaDiscoveryFromConfigRequest(connection=ConnectionId(conn_id), discovery_config=config_id)
+        run_id = client.start_schema_discovery_run_from_config(from_config)
+        source = f"config '{config}'"
+    else:
+        request = SchemaDiscoveryRequest(connection=ConnectionId(conn_id))
+        run_id = client.start_schema_discovery_run(request)
+        source = "default discovery"
+
     print_success(
-        f"Schema discovery run {run_id} started for connection '{connection}'. "
+        f"Schema discovery run {run_id} started for connection '{connection}' ({source}). "
         f"Once finished, list results with: dm discover schema-results {run_id}"
     )
 
 
+@app.command("file")
+def file_discovery(
+    connection: str = typer.Argument(help="Connection name or ID"),
+    config: str | None = typer.Option(
+        None, "--config", "-c", help="Run with a saved file discovery config (configurable discovery)"
+    ),
+    profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"),
+) -> None:
+    """Start a file-data-discovery run on a file connection.
+
+    Once finished, download the report with `dm discover file-report <run-id>`
+    (poll with `dm run status <run-id>`).
+    """
+    client = get_client(profile)
+    conn_id = _resolve_connection_id(client, connection)
+
+    if config is not None:
+        config_id = _resolve_discovery_config_id(client, config, DiscoveryConfigType.file)
+        from_config = FileDataDiscoveryFromConfigRequest(connection=ConnectionId(conn_id), discovery_config=config_id)
+        run_id = client.start_file_data_discovery_run_from_config(from_config)
+        source = f"config '{config}'"
+    else:
+        request = FileDataDiscoveryRequest(connection=ConnectionId(conn_id))
+        run_id = client.start_file_data_discovery_run(request)
+        source = "default discovery"
+
+    print_success(
+        f"File data discovery run {run_id} started for connection '{connection}' ({source}). "
+        f"Once finished, download the report with: dm discover file-report {run_id}"
+    )
+
+
 @app.command("schema-results")
 def schema_results(
     run_id: int = typer.Argument(help="Schema discovery run ID"),
@@ -77,8 +157,11 @@ def schema_results(
             "table": r.table,
             "column": r.column,
             "data_type": r.data.data_type or "",
-            "matches": ", ".join(m.label for m in r.data.discovery_matches) or "-",
+            "matches": ", ".join(m.label for m in r.data.discovery_matches if m.label) or "-",
             "constraint": r.data.constraint or "",
+            "safe_data_preview": (
+                r.data.safe_data_preview.model_dump(mode="json") if r.data.safe_data_preview else None
+            ),
         }
         for r in results
     ]
@@ -142,13 +225,43 @@ def file_discovery_report(
     """Download file discovery report for a run."""
     client = get_client(profile)
     report = client.get_file_data_discovery_report(RunId(run_id))
+    full = [result.model_dump(mode="json") for result in report]
 
     if output is not None:
-        output.write_text(json.dumps(report, indent=2, default=str))
+        output.write_text(json.dumps(full, indent=2, default=str))
         print_success(f"File discovery report written to {output}")
         return
 
     if should_emit_json(is_json):
-        print_json(report)
-    else:
-        render_output(report, is_json=False, title=f"File Discovery: Run {run_id}")
+        print_json(full)
+        return
+
+    rows = [
+        {
+            "id": result.id,
+            "files": ", ".join(f.path for f in result.files),
+            "locator": locator.locator,
+            "matches": ", ".join(m.label for m in locator.matches if m.label) or "-",
+            "data_types": ", ".join(locator.data_types) or "-",
+        }
+        for result in report
+        for locator in result.results
+    ]
+    render_output(
+        rows,
+        is_json=False,
+        columns=["id", "files", "locator", "matches", "data_types"],
+        title=f"File Discovery: Run {run_id}",
+    )
+
+
+@app.command("config-snapshot")
+def config_snapshot(
+    run_id: int = typer.Argument(help="Discovery run ID"),
+    output: Path | None = typer.Option(None, "--output", "-o", help="Write YAML to this path"),
+    profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"),
+) -> None:
+    """Download the discovery config a run used (the run's snapshot)."""
+    client = get_client(profile)
+    snapshot = client.get_discovery_run_config_snapshot_yaml(RunId(run_id))
+    _write_or_echo(snapshot, output, "Discovery config snapshot")