From 9749ab1b45079058ada03e5b390123f8e5fed24c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:26:48 +0000 Subject: [PATCH 01/17] Add audit-logs download command with resumable chunked export Wraps GET /audit-logs/export/chunk (kernel-go-sdk v0.75.0). Chunks are verified against X-Content-Sha256 before being appended to the output file, and a sidecar state file persists the cursor and committed byte offset so an interrupted download resumes without duplicating rows. Ranges over the API's 30-day cap fail fast with ready-to-run per-window commands. Co-Authored-By: Claude Fable 5 --- cmd/audit_logs.go | 2 + cmd/audit_logs_download.go | 401 ++++++++++++++++++++++++++++++++ cmd/audit_logs_download_test.go | 393 +++++++++++++++++++++++++++++++ cmd/audit_logs_test.go | 8 + go.mod | 2 +- go.sum | 4 +- 6 files changed, 807 insertions(+), 3 deletions(-) create mode 100644 cmd/audit_logs_download.go create mode 100644 cmd/audit_logs_download_test.go diff --git a/cmd/audit_logs.go b/cmd/audit_logs.go index 23230ee6..da60f527 100644 --- a/cmd/audit_logs.go +++ b/cmd/audit_logs.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "net/http" "strconv" "strings" "time" @@ -17,6 +18,7 @@ import ( type AuditLogsService interface { ListAutoPaging(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] + ExportChunk(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) } type AuditLogsCmd struct { diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go new file mode 100644 index 00000000..751468d3 --- /dev/null +++ b/cmd/audit_logs_download.go @@ -0,0 +1,401 @@ +package cmd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/kernel/cli/pkg/util" + "github.com/kernel/kernel-go-sdk" + "github.com/pterm/pterm" + "github.com/spf13/cobra" +) + +type AuditLogsDownloadInput struct { + Start string + End string + Search string + Method string + ExcludeMethod string + Service string + AuthStrategy string + UserIDs []string + To string + Format string + Force bool +} + +const ( + // auditLogsDownloadMaxRange mirrors the API's max export window. + auditLogsDownloadMaxRange = 30 * 24 * time.Hour + // auditLogsDownloadShaRetries is the number of extra attempts for a chunk + // whose body fails checksum verification. + auditLogsDownloadShaRetries = 2 +) + +// auditLogsDownloadState is the sidecar file that makes a download resumable. +// It is written atomically after every committed chunk and removed on +// completion. Version guards future format changes (e.g. multi-window splits). +type auditLogsDownloadState struct { + Version int `json:"version"` + Params string `json:"params"` + Cursor string `json:"cursor"` + BytesWritten int64 `json:"bytes_written"` + Chunks int `json:"chunks"` + Rows int64 `json:"rows"` +} + +const auditLogsDownloadStateVersion = 1 + +func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) error { + params, err := buildAuditLogsDownloadParams(in) + if err != nil { + return err + } + + format := in.Format + if format == "" { + format = string(kernel.AuditLogExportChunkParamsFormatJSONLGz) + } + switch format { + case string(kernel.AuditLogExportChunkParamsFormatJSONL), string(kernel.AuditLogExportChunkParamsFormatJSONLGz): + params.Format = kernel.AuditLogExportChunkParamsFormat(format) + default: + return fmt.Errorf("--format must be jsonl or jsonl.gz") + } + + outPath := in.To + if outPath == "" { + outPath = defaultAuditLogsDownloadPath(params.Start, params.End, format) + } + statePath := outPath + ".state.json" + fingerprint, err := auditLogsDownloadFingerprint(params) + if err != nil { + return err + } + + state, err := loadAuditLogsDownloadState(statePath, outPath, fingerprint, in.Force) + if err != nil { + return err + } + // A state file with chunks but no cursor means the download finished and + // only the cleanup was interrupted. + if state.Chunks > 0 && state.Cursor == "" { + if err := os.Remove(statePath); err != nil { + return fmt.Errorf("remove state file: %w", err) + } + pterm.Success.Printf("Download already complete: %d rows (%s) in %s\n", state.Rows, util.FormatBytes(state.BytesWritten), outPath) + return nil + } + + out, err := openAuditLogsDownloadOutput(outPath, state.BytesWritten) + if err != nil { + return err + } + defer out.Close() + + if state.Chunks > 0 { + pterm.Info.Printf("Resuming download at chunk %d (%d rows so far)\n", state.Chunks+1, state.Rows) + } + + for { + if state.Cursor != "" { + params.Cursor = kernel.String(state.Cursor) + } + body, header, err := c.fetchAuditLogsChunk(ctx, params) + if err != nil { + if state.Chunks > 0 { + pterm.Info.Println("Progress saved; rerun the same command to resume") + } + return err + } + + if _, err := out.Write(body); err != nil { + return fmt.Errorf("write %s: %w", outPath, err) + } + if err := out.Sync(); err != nil { + return fmt.Errorf("sync %s: %w", outPath, err) + } + + rows, _ := strconv.ParseInt(header.Get("X-Row-Count"), 10, 64) + state.Chunks++ + state.Rows += rows + state.BytesWritten += int64(len(body)) + + hasMore, _ := strconv.ParseBool(header.Get("X-Has-More")) + nextCursor := header.Get("X-Next-Cursor") + if hasMore { + // Guard against a server bug looping the download forever. + if nextCursor == "" { + return fmt.Errorf("server reported more records but returned no cursor; retry, and report this if it persists") + } + if nextCursor == state.Cursor { + return fmt.Errorf("server returned an unchanged cursor; retry, and report this if it persists") + } + } + state.Cursor = nextCursor + state.Params = fingerprint + if err := saveAuditLogsDownloadState(statePath, state); err != nil { + return err + } + + pterm.Info.Printf("Chunk %d: %d rows (%d total, %s)\n", state.Chunks, rows, state.Rows, util.FormatBytes(state.BytesWritten)) + if !hasMore { + break + } + } + + if err := os.Remove(statePath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove state file: %w", err) + } + pterm.Success.Printf("Downloaded %d rows (%s) to %s\n", state.Rows, util.FormatBytes(state.BytesWritten), outPath) + return nil +} + +// fetchAuditLogsChunk downloads one chunk and verifies it against the +// X-Content-Sha256 trailer before returning it, retrying on mismatch. Nothing +// is written to disk until a chunk verifies. +func (c AuditLogsCmd) fetchAuditLogsChunk(ctx context.Context, params kernel.AuditLogExportChunkParams) ([]byte, http.Header, error) { + var lastErr error + for attempt := 0; attempt <= auditLogsDownloadShaRetries; attempt++ { + res, err := c.auditLogs.ExportChunk(ctx, params) + if err != nil { + return nil, nil, util.CleanedUpSdkError{Err: err} + } + body, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + lastErr = fmt.Errorf("read chunk body: %w", err) + continue + } + if want := res.Header.Get("X-Content-Sha256"); want != "" { + sum := sha256.Sum256(body) + if got := hex.EncodeToString(sum[:]); got != want { + lastErr = fmt.Errorf("chunk checksum mismatch (got %s, want %s)", got, want) + continue + } + } + return body, res.Header, nil + } + return nil, nil, lastErr +} + +func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExportChunkParams, error) { + var params kernel.AuditLogExportChunkParams + var err error + start := time.Now().UTC().Add(-24 * time.Hour) + if in.Start != "" { + start, _, err = parseAuditLogTime(in.Start) + if err != nil { + return params, fmt.Errorf("--start: %w", err) + } + } + end := time.Now().UTC() + if in.End != "" { + var dateOnly bool + end, dateOnly, err = parseAuditLogTime(in.End) + if err != nil { + return params, fmt.Errorf("--end: %w", err) + } + if dateOnly { + end = end.Add(24 * time.Hour) + } + } + if !start.Before(end) { + return params, fmt.Errorf("--start must be before --end") + } + if end.Sub(start) > auditLogsDownloadMaxRange { + // The window list goes through pterm because the error renderer + // collapses newlines, which would mangle the copy-pasteable flags. + pterm.Info.Printf("Run one download per window:\n%s", suggestAuditLogsDownloadWindows(start, end)) + return params, fmt.Errorf("range is %d days; the API allows at most 30 days per download", int(end.Sub(start).Hours()/24)) + } + + params.Start = start + params.End = end + if in.Search != "" { + params.Search = kernel.String(in.Search) + } + if in.Method != "" { + params.Method = kernel.String(in.Method) + } + if in.ExcludeMethod != "" { + params.ExcludeMethod = kernel.String(in.ExcludeMethod) + } + if in.Service != "" { + params.Service = kernel.String(in.Service) + } + if in.AuthStrategy != "" { + params.AuthStrategy = kernel.String(in.AuthStrategy) + } + if len(in.UserIDs) > 0 { + params.SearchUserID = in.UserIDs + } + return params, nil +} + +// suggestAuditLogsDownloadWindows renders ready-to-run 30-day windows covering +// [start, end), newest first to match the export's record order. +func suggestAuditLogsDownloadWindows(start, end time.Time) string { + var out string + for windowEnd := end; windowEnd.After(start); { + windowStart := windowEnd.Add(-auditLogsDownloadMaxRange) + if windowStart.Before(start) { + windowStart = start + } + out += fmt.Sprintf(" --start %s --end %s\n", windowStart.Format(time.RFC3339), windowEnd.Format(time.RFC3339)) + windowEnd = windowStart + } + return out +} + +// auditLogsDownloadFingerprint identifies the query a state file belongs to, +// so a resume never mixes records from different queries in one output file. +// It must be computed before the cursor is set on params. +func auditLogsDownloadFingerprint(params kernel.AuditLogExportChunkParams) (string, error) { + q, err := params.URLQuery() + if err != nil { + return "", fmt.Errorf("fingerprint params: %w", err) + } + return q.Encode(), nil +} + +func defaultAuditLogsDownloadPath(start, end time.Time, format string) string { + const stamp = "20060102T150405Z" + return fmt.Sprintf("audit-logs-%s-%s.%s", start.UTC().Format(stamp), end.UTC().Format(stamp), format) +} + +// loadAuditLogsDownloadState decides how a download starts: fresh, resumed +// from a matching state file, or rejected because the output would be +// clobbered or the state belongs to a different query. +func loadAuditLogsDownloadState(statePath, outPath, fingerprint string, force bool) (auditLogsDownloadState, error) { + fresh := auditLogsDownloadState{Version: auditLogsDownloadStateVersion, Params: fingerprint} + raw, err := os.ReadFile(statePath) + if os.IsNotExist(err) { + if !force { + if _, statErr := os.Stat(outPath); statErr == nil { + return fresh, fmt.Errorf("%s already exists; pass --force to overwrite or -o to pick another path", outPath) + } + } + return fresh, nil + } + if err != nil { + return fresh, fmt.Errorf("read state file: %w", err) + } + if force { + return fresh, nil + } + var state auditLogsDownloadState + if err := json.Unmarshal(raw, &state); err != nil { + return fresh, fmt.Errorf("state file %s is corrupt; pass --force to start over", statePath) + } + if state.Version != auditLogsDownloadStateVersion { + return fresh, fmt.Errorf("state file %s was written by an incompatible CLI version; pass --force to start over", statePath) + } + if state.Params != fingerprint { + return fresh, fmt.Errorf("state file %s belongs to a download with different parameters; pass --force to start over or -o to pick another path", statePath) + } + return state, nil +} + +func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) error { + raw, err := json.Marshal(state) + if err != nil { + return fmt.Errorf("encode state: %w", err) + } + tmp := statePath + ".tmp" + if err := os.WriteFile(tmp, raw, 0o644); err != nil { + return fmt.Errorf("write state file: %w", err) + } + if err := os.Rename(tmp, statePath); err != nil { + return fmt.Errorf("commit state file: %w", err) + } + return nil +} + +// openAuditLogsDownloadOutput opens the output for appending, truncated to the +// last committed offset so a chunk interrupted mid-write is dropped rather +// than duplicated on resume. +func openAuditLogsDownloadOutput(outPath string, committed int64) (*os.File, error) { + if dir := filepath.Dir(outPath); dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create output directory: %w", err) + } + } + out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE, 0o644) + if err != nil { + return nil, fmt.Errorf("open %s: %w", outPath, err) + } + if err := out.Truncate(committed); err != nil { + out.Close() + return nil, fmt.Errorf("truncate %s to last committed chunk: %w", outPath, err) + } + if _, err := out.Seek(committed, io.SeekStart); err != nil { + out.Close() + return nil, fmt.Errorf("seek %s: %w", outPath, err) + } + return out, nil +} + +func runAuditLogsDownload(cmd *cobra.Command, args []string) error { + c := getAuditLogsHandler(cmd) + start, _ := cmd.Flags().GetString("start") + end, _ := cmd.Flags().GetString("end") + search, _ := cmd.Flags().GetString("search") + method, _ := cmd.Flags().GetString("method") + excludeMethod, _ := cmd.Flags().GetString("exclude-method") + service, _ := cmd.Flags().GetString("service") + authStrategy, _ := cmd.Flags().GetString("auth-strategy") + userIDs, _ := cmd.Flags().GetStringArray("user-id") + to, _ := cmd.Flags().GetString("to") + format, _ := cmd.Flags().GetString("format") + force, _ := cmd.Flags().GetBool("force") + + return c.Download(cmd.Context(), AuditLogsDownloadInput{ + Start: start, + End: end, + Search: search, + Method: method, + ExcludeMethod: excludeMethod, + Service: service, + AuthStrategy: authStrategy, + UserIDs: userIDs, + To: to, + Format: format, + Force: force, + }) +} + +var auditLogsDownloadCmd = &cobra.Command{ + Use: "download", + Short: "Download audit logs for a time range to a file", + Long: "Download an organization's audit log records for a time range as a file, for archival, compliance, or offline analysis.\n\n" + + "Records are fetched in chunks and appended after each chunk is verified, newest first. If a download is interrupted, rerunning the same command resumes where it left off.\n\n" + + "The API allows at most 30 days per download; for longer ranges run one download per window. Unlike search, no records are excluded by default.", + Args: cobra.NoArgs, + RunE: runAuditLogsDownload, +} + +func init() { + auditLogsDownloadCmd.Flags().String("start", "", "Start of the export window, RFC3339 or YYYY-MM-DD (default: 24 hours ago)") + auditLogsDownloadCmd.Flags().String("end", "", "End of the export window, RFC3339 or YYYY-MM-DD inclusive (default: now)") + auditLogsDownloadCmd.Flags().String("search", "", "Free-text search") + auditLogsDownloadCmd.Flags().String("method", "", "Filter by HTTP method (e.g. GET)") + auditLogsDownloadCmd.Flags().String("exclude-method", "", "Exclude an HTTP method") + auditLogsDownloadCmd.Flags().String("service", "", "Filter by service") + auditLogsDownloadCmd.Flags().String("auth-strategy", "", "Filter by authentication strategy") + auditLogsDownloadCmd.Flags().StringArray("user-id", nil, "Filter by user ID (repeatable)") + auditLogsDownloadCmd.Flags().String("to", "", "Output file path (default: audit-logs--.)") + auditLogsDownloadCmd.Flags().String("format", "jsonl.gz", "Output format: jsonl or jsonl.gz") + auditLogsDownloadCmd.Flags().Bool("force", false, "Overwrite the output file and ignore saved progress") + + auditLogsCmd.AddCommand(auditLogsDownloadCmd) +} diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go new file mode 100644 index 00000000..cc7c9553 --- /dev/null +++ b/cmd/audit_logs_download_test.go @@ -0,0 +1,393 @@ +package cmd + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/kernel/kernel-go-sdk" + "github.com/kernel/kernel-go-sdk/option" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type exportChunk struct { + body []byte + rowCount int + hasMore bool + nextCursor string + sha string // overrides the computed body sha when set +} + +func exportChunkResponse(c exportChunk) *http.Response { + sha := c.sha + if sha == "" { + sum := sha256.Sum256(c.body) + sha = hex.EncodeToString(sum[:]) + } + header := http.Header{} + header.Set("X-Has-More", strconv.FormatBool(c.hasMore)) + header.Set("X-Row-Count", strconv.Itoa(c.rowCount)) + header.Set("X-Content-Sha256", sha) + if c.nextCursor != "" { + header.Set("X-Next-Cursor", c.nextCursor) + } + return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(bytes.NewReader(c.body))} +} + +func gzipMember(t *testing.T, lines string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, err := gz.Write([]byte(lines)) + require.NoError(t, err) + require.NoError(t, gz.Close()) + return buf.Bytes() +} + +func gunzipAll(t *testing.T, data []byte) string { + t.Helper() + r, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + out, err := io.ReadAll(r) + require.NoError(t, err) + return string(out) +} + +// chunkServer serves a scripted sequence of responses keyed by call order and +// records the cursor each call carried. +func chunkServer(t *testing.T, responses []func() (*http.Response, error)) (*FakeAuditLogsService, *[]string) { + t.Helper() + var cursors []string + calls := 0 + fake := &FakeAuditLogsService{ + ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { + cursors = append(cursors, query.Cursor.Value) + require.Less(t, calls, len(responses), "more ExportChunk calls than scripted responses") + res, err := responses[calls]() + calls++ + return res, err + }, + } + return fake, &cursors +} + +func downloadInput(outPath string) AuditLogsDownloadInput { + return AuditLogsDownloadInput{ + Start: "2026-06-01", + End: "2026-06-28", + To: outPath, + Format: "jsonl.gz", + } +} + +func TestAuditLogsDownloadMultiChunk(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk1 := gzipMember(t, "{\"n\":1}\n{\"n\":2}\n") + chunk2 := gzipMember(t, "{\"n\":3}\n") + chunk3 := gzipMember(t, "{\"n\":4}\n{\"n\":5}\n") + fake, cursors := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk1, rowCount: 2, hasMore: true, nextCursor: "c1"}), nil + }, + // A short chunk with hasMore=true must not end the download. + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk2, rowCount: 1, hasMore: true, nextCursor: "c2"}), nil + }, + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk3, rowCount: 2, hasMore: false}), nil + }, + }) + + c := AuditLogsCmd{auditLogs: fake} + require.NoError(t, c.Download(context.Background(), downloadInput(outPath))) + + assert.Equal(t, []string{"", "c1", "c2"}, *cursors) + data, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n{\"n\":4}\n{\"n\":5}\n", gunzipAll(t, data)) + _, err = os.Stat(outPath + ".state.json") + assert.True(t, os.IsNotExist(err), "state file should be removed on completion") +} + +func TestAuditLogsDownloadChecksumMismatchRetriesThenSucceeds(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, sha: "deadbeef"}), nil + }, + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1}), nil + }, + }) + + c := AuditLogsCmd{auditLogs: fake} + require.NoError(t, c.Download(context.Background(), downloadInput(outPath))) + + data, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "{\"n\":1}\n", gunzipAll(t, data), "corrupt chunk must not be written") +} + +func TestAuditLogsDownloadChecksumMismatchExhaustsRetries(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk := gzipMember(t, "{\"n\":1}\n") + bad := func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, sha: "deadbeef"}), nil + } + fake, _ := chunkServer(t, []func() (*http.Response, error){bad, bad, bad}) + + c := AuditLogsCmd{auditLogs: fake} + err := c.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "checksum mismatch") + + data, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Empty(t, data, "no bytes may be written for a chunk that never verified") +} + +func TestAuditLogsDownloadResumesAfterFailure(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk1 := gzipMember(t, "{\"n\":1}\n") + chunk2 := gzipMember(t, "{\"n\":2}\n") + + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + c := AuditLogsCmd{auditLogs: fake} + require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + + fake2, cursors := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk2, rowCount: 1, hasMore: false}), nil + }, + }) + c2 := AuditLogsCmd{auditLogs: fake2} + require.NoError(t, c2.Download(context.Background(), downloadInput(outPath))) + + assert.Equal(t, []string{"c1"}, *cursors, "resume must continue from the saved cursor") + data, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", gunzipAll(t, data), "resumed file must have no duplicate or missing rows") + _, err = os.Stat(outPath + ".state.json") + assert.True(t, os.IsNotExist(err)) +} + +func TestAuditLogsDownloadResumeTruncatesTornWrite(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk1 := gzipMember(t, "{\"n\":1}\n") + chunk2 := gzipMember(t, "{\"n\":2}\n") + + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + c := AuditLogsCmd{auditLogs: fake} + require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + + // Simulate a crash mid-append: garbage past the committed offset. + f, err := os.OpenFile(outPath, os.O_WRONLY|os.O_APPEND, 0o644) + require.NoError(t, err) + _, err = f.Write([]byte("torn partial chunk")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + fake2, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk2, rowCount: 1, hasMore: false}), nil + }, + }) + c2 := AuditLogsCmd{auditLogs: fake2} + require.NoError(t, c2.Download(context.Background(), downloadInput(outPath))) + + data, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", gunzipAll(t, data)) +} + +func TestAuditLogsDownloadRejectsMismatchedState(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk1 := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + c := AuditLogsCmd{auditLogs: fake} + require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + + in := downloadInput(outPath) + in.Service = "api" + err := c.Download(context.Background(), in) + require.ErrorContains(t, err, "different parameters") +} + +func TestAuditLogsDownloadForceRestarts(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk1 := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + c := AuditLogsCmd{auditLogs: fake} + require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + + chunk := gzipMember(t, "{\"n\":9}\n") + fake2, cursors := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: false}), nil + }, + }) + in := downloadInput(outPath) + in.Force = true + c2 := AuditLogsCmd{auditLogs: fake2} + require.NoError(t, c2.Download(context.Background(), in)) + + assert.Equal(t, []string{""}, *cursors, "--force must restart from the beginning") + data, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "{\"n\":9}\n", gunzipAll(t, data)) +} + +func TestAuditLogsDownloadRefusesExistingOutputWithoutState(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + require.NoError(t, os.WriteFile(outPath, []byte("existing"), 0o644)) + + c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + err := c.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "already exists") +} + +func TestAuditLogsDownloadFinishesCompletedLeftoverState(t *testing.T) { + buf := capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk1 := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: false}), nil + }, + }) + c := AuditLogsCmd{auditLogs: fake} + require.NoError(t, c.Download(context.Background(), downloadInput(outPath))) + + // Simulate a crash after the final state save but before cleanup by + // re-creating the completed state file. + statePath := outPath + ".state.json" + require.NoError(t, os.WriteFile(statePath, []byte(`{"version":1,"params":"`+mustFingerprint(t)+`","cursor":"","bytes_written":`+strconv.Itoa(len(chunk1))+`,"chunks":1,"rows":1}`), 0o644)) + + c2 := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + require.NoError(t, c2.Download(context.Background(), downloadInput(outPath))) + assert.Contains(t, buf.String(), "already complete") + _, err := os.Stat(statePath) + assert.True(t, os.IsNotExist(err)) +} + +func mustFingerprint(t *testing.T) string { + t.Helper() + params, err := buildAuditLogsDownloadParams(downloadInput("")) + require.NoError(t, err) + params.Format = kernel.AuditLogExportChunkParamsFormatJSONLGz + fp, err := auditLogsDownloadFingerprint(params) + require.NoError(t, err) + return fp +} + +func TestAuditLogsDownloadRangeOver30DaysSuggestsWindows(t *testing.T) { + buf := capturePtermOutput(t) + c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + err := c.Download(context.Background(), AuditLogsDownloadInput{Start: "2026-04-01", End: "2026-06-30"}) + require.ErrorContains(t, err, "at most 30 days") + assert.Contains(t, buf.String(), "--start 2026-06-01T00:00:00Z --end 2026-07-01T00:00:00Z") + assert.Contains(t, buf.String(), "--start 2026-04-01T00:00:00Z --end 2026-04-02T00:00:00Z") +} + +func TestAuditLogsDownloadServerCursorBugs(t *testing.T) { + capturePtermOutput(t) + chunk := gzipMember(t, "{\"n\":1}\n") + + t.Run("has more without cursor", func(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true}), nil + }, + }) + c := AuditLogsCmd{auditLogs: fake} + err := c.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "no cursor") + }) + + t.Run("unchanged cursor", func(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + repeat := func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + } + fake, _ := chunkServer(t, []func() (*http.Response, error){repeat, repeat}) + c := AuditLogsCmd{auditLogs: fake} + err := c.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "unchanged cursor") + }) +} + +func TestAuditLogsDownloadJSONLFormat(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl") + var gotFormat kernel.AuditLogExportChunkParamsFormat + fake := &FakeAuditLogsService{ + ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { + gotFormat = query.Format + return exportChunkResponse(exportChunk{body: []byte("{\"n\":1}\n"), rowCount: 1}), nil + }, + } + in := downloadInput(outPath) + in.Format = "jsonl" + c := AuditLogsCmd{auditLogs: fake} + require.NoError(t, c.Download(context.Background(), in)) + + assert.Equal(t, kernel.AuditLogExportChunkParamsFormatJSONL, gotFormat) + data, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "{\"n\":1}\n", string(data)) +} + +func TestAuditLogsDownloadRejectsInvalidFormat(t *testing.T) { + c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + in := downloadInput("out.csv") + in.Format = "csv" + err := c.Download(context.Background(), in) + require.ErrorContains(t, err, "--format must be jsonl or jsonl.gz") +} diff --git a/cmd/audit_logs_test.go b/cmd/audit_logs_test.go index 5c813e5a..1efd5e81 100644 --- a/cmd/audit_logs_test.go +++ b/cmd/audit_logs_test.go @@ -17,6 +17,7 @@ import ( type FakeAuditLogsService struct { ListAutoPagingFunc func(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] + ExportChunkFunc func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) } func (f *FakeAuditLogsService) ListAutoPaging(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] { @@ -26,6 +27,13 @@ func (f *FakeAuditLogsService) ListAutoPaging(ctx context.Context, query kernel. return auditLogPager() } +func (f *FakeAuditLogsService) ExportChunk(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { + if f.ExportChunkFunc != nil { + return f.ExportChunkFunc(ctx, query, opts...) + } + return nil, errors.New("ExportChunk not implemented") +} + func auditLogPager(entries ...kernel.AuditLogEntry) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] { page := &pagination.PageTokenPagination[kernel.AuditLogEntry]{Items: entries} page.SetPageConfig(nil, &http.Response{Header: http.Header{}}) diff --git a/go.mod b/go.mod index 4c5dcfb7..b3939d33 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss/v2 v2.0.0-beta.1 github.com/golang-jwt/jwt/v5 v5.2.2 github.com/joho/godotenv v1.5.1 - github.com/kernel/kernel-go-sdk v0.72.0 + github.com/kernel/kernel-go-sdk v0.75.0 github.com/klauspost/compress v1.18.5 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/pterm/pterm v0.12.80 diff --git a/go.sum b/go.sum index 14e37c5c..a182f786 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/kernel/kernel-go-sdk v0.72.0 h1:QyT5v2PMJjp9GQCGV4fi1IWcCF/fcsFcz1MTXPA/WZU= -github.com/kernel/kernel-go-sdk v0.72.0/go.mod h1:EeZzSuHZVeHKxKCPUzxou2bovNGhXaz0RXrSqKNf1AQ= +github.com/kernel/kernel-go-sdk v0.75.0 h1:UTlBLOVGQIFa0Vcn9DrTbhR44IQRrcZuhaKcMTwHvms= +github.com/kernel/kernel-go-sdk v0.75.0/go.mod h1:EeZzSuHZVeHKxKCPUzxou2bovNGhXaz0RXrSqKNf1AQ= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= From e719bd263b8aaeee3bbc8317ce2a85ec1eb294c0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:42:52 +0000 Subject: [PATCH 02/17] Exclude GET requests from downloads by default, matching search --include-get lifts the exclusion. Because chunks are appended verbatim, a second exclusion can't stack client-side the way search filters, so --exclude-method with the GET default active is rejected with guidance rather than silently replacing it. Co-Authored-By: Claude Fable 5 --- cmd/audit_logs_download.go | 24 +++++++++++++++++++++--- cmd/audit_logs_download_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 751468d3..e4152912 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "time" "github.com/kernel/cli/pkg/util" @@ -25,6 +26,7 @@ type AuditLogsDownloadInput struct { Search string Method string ExcludeMethod string + IncludeGet bool Service string AuthStrategy string UserIDs []string @@ -227,8 +229,20 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp if in.Method != "" { params.Method = kernel.String(in.Method) } - if in.ExcludeMethod != "" { - params.ExcludeMethod = kernel.String(in.ExcludeMethod) + // GETs are excluded by default, like search. The API accepts a single + // exclude_method, and chunks are appended verbatim, so a second exclusion + // can't be filtered client-side the way search does; require --include-get + // so the default is never dropped silently. + excludeMethod := in.ExcludeMethod + if in.Method == "" && !in.IncludeGet { + if excludeMethod == "" { + excludeMethod = "GET" + } else if !strings.EqualFold(excludeMethod, "GET") { + return params, fmt.Errorf("--exclude-method %s would replace the default GET exclusion; add --include-get to confirm GET requests should be included", in.ExcludeMethod) + } + } + if excludeMethod != "" { + params.ExcludeMethod = kernel.String(excludeMethod) } if in.Service != "" { params.Service = kernel.String(in.Service) @@ -352,6 +366,7 @@ func runAuditLogsDownload(cmd *cobra.Command, args []string) error { search, _ := cmd.Flags().GetString("search") method, _ := cmd.Flags().GetString("method") excludeMethod, _ := cmd.Flags().GetString("exclude-method") + includeGet, _ := cmd.Flags().GetBool("include-get") service, _ := cmd.Flags().GetString("service") authStrategy, _ := cmd.Flags().GetString("auth-strategy") userIDs, _ := cmd.Flags().GetStringArray("user-id") @@ -365,6 +380,7 @@ func runAuditLogsDownload(cmd *cobra.Command, args []string) error { Search: search, Method: method, ExcludeMethod: excludeMethod, + IncludeGet: includeGet, Service: service, AuthStrategy: authStrategy, UserIDs: userIDs, @@ -379,7 +395,8 @@ var auditLogsDownloadCmd = &cobra.Command{ Short: "Download audit logs for a time range to a file", Long: "Download an organization's audit log records for a time range as a file, for archival, compliance, or offline analysis.\n\n" + "Records are fetched in chunks and appended after each chunk is verified, newest first. If a download is interrupted, rerunning the same command resumes where it left off.\n\n" + - "The API allows at most 30 days per download; for longer ranges run one download per window. Unlike search, no records are excluded by default.", + "GET requests are excluded by default; pass --include-get to include them, or --method GET to see only them.\n\n" + + "The API allows at most 30 days per download; for longer ranges run one download per window.", Args: cobra.NoArgs, RunE: runAuditLogsDownload, } @@ -390,6 +407,7 @@ func init() { auditLogsDownloadCmd.Flags().String("search", "", "Free-text search") auditLogsDownloadCmd.Flags().String("method", "", "Filter by HTTP method (e.g. GET)") auditLogsDownloadCmd.Flags().String("exclude-method", "", "Exclude an HTTP method") + auditLogsDownloadCmd.Flags().Bool("include-get", false, "Include GET requests, which are excluded by default") auditLogsDownloadCmd.Flags().String("service", "", "Filter by service") auditLogsDownloadCmd.Flags().String("auth-strategy", "", "Filter by authentication strategy") auditLogsDownloadCmd.Flags().StringArray("user-id", nil, "Filter by user ID (repeatable)") diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index cc7c9553..236ae60f 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -384,6 +384,39 @@ func TestAuditLogsDownloadJSONLFormat(t *testing.T) { assert.Equal(t, "{\"n\":1}\n", string(data)) } +func TestAuditLogsDownloadExcludesGetByDefault(t *testing.T) { + cases := []struct { + name string + mutate func(*AuditLogsDownloadInput) + wantExclude string + wantErr string + }{ + {name: "default excludes GET", mutate: func(in *AuditLogsDownloadInput) {}, wantExclude: "GET"}, + {name: "include-get lifts the default", mutate: func(in *AuditLogsDownloadInput) { in.IncludeGet = true }, wantExclude: ""}, + {name: "method filter lifts the default", mutate: func(in *AuditLogsDownloadInput) { in.Method = "POST" }, wantExclude: ""}, + {name: "explicit GET exclusion", mutate: func(in *AuditLogsDownloadInput) { in.ExcludeMethod = "get" }, wantExclude: "get"}, + {name: "other exclusion with include-get", mutate: func(in *AuditLogsDownloadInput) { + in.ExcludeMethod = "OPTIONS" + in.IncludeGet = true + }, wantExclude: "OPTIONS"}, + {name: "other exclusion without include-get is rejected", mutate: func(in *AuditLogsDownloadInput) { in.ExcludeMethod = "OPTIONS" }, + wantErr: "add --include-get"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + in := downloadInput("") + tc.mutate(&in) + params, err := buildAuditLogsDownloadParams(in) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantExclude, params.ExcludeMethod.Value) + }) + } +} + func TestAuditLogsDownloadRejectsInvalidFormat(t *testing.T) { c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} in := downloadInput("out.csv") From eebeed410c8a11cd1c7d451cfb9dd9c6b59c2666 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:33:04 +0000 Subject: [PATCH 03/17] Harden download resume state against review findings - Bind resume state to a credential identity (hashed API key or OAuth org ID) so switching credentials can't mix organizations in one archive - Validate the output file against recorded progress on resume; a missing or shortened file now fails instead of being zero-extended and reported complete - Fail closed on missing X-Has-More and X-Content-Sha256 headers, and validate pagination headers before any bytes are written - Commit initial state before the first fetch so an early failure retries without --force; --force removes stale state immediately - Write state via os.CreateTemp and use 0600/0700 modes for outputs - Cap buffered chunk responses at 256 MiB - Fix stale -o references in errors; document that resume needs explicit --start/--end Co-Authored-By: Claude Fable 5 --- cmd/audit_logs.go | 5 +- cmd/audit_logs_download.go | 150 +++++++++++++++++----- cmd/audit_logs_download_test.go | 214 ++++++++++++++++++++++++++++++-- 3 files changed, 327 insertions(+), 42 deletions(-) diff --git a/cmd/audit_logs.go b/cmd/audit_logs.go index da60f527..9497ec4e 100644 --- a/cmd/audit_logs.go +++ b/cmd/audit_logs.go @@ -23,6 +23,9 @@ type AuditLogsService interface { type AuditLogsCmd struct { auditLogs AuditLogsService + // identity distinguishes the credential in use; download resume state is + // bound to it so archives never mix organizations. + identity string } type AuditLogsSearchInput struct { @@ -188,7 +191,7 @@ func formatAuditLogUser(entry kernel.AuditLogEntry) string { func getAuditLogsHandler(cmd *cobra.Command) AuditLogsCmd { client := getKernelClient(cmd) - return AuditLogsCmd{auditLogs: &client.AuditLogs} + return AuditLogsCmd{auditLogs: &client.AuditLogs, identity: auditLogsCredentialIdentity()} } func runAuditLogsSearch(cmd *cobra.Command, args []string) error { diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index e4152912..d74f15b1 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/kernel/cli/pkg/auth" "github.com/kernel/cli/pkg/util" "github.com/kernel/kernel-go-sdk" "github.com/pterm/pterm" @@ -43,12 +44,19 @@ const ( auditLogsDownloadShaRetries = 2 ) +// auditLogsDownloadMaxChunkBytes bounds how much of a chunk response is +// buffered. The server caps chunks at 50k rows, so a legitimate response is +// far smaller; this guards against a broken server streaming without end. +// Variable so tests can lower it. +var auditLogsDownloadMaxChunkBytes int64 = 256 << 20 + // auditLogsDownloadState is the sidecar file that makes a download resumable. // It is written atomically after every committed chunk and removed on // completion. Version guards future format changes (e.g. multi-window splits). type auditLogsDownloadState struct { Version int `json:"version"` Params string `json:"params"` + Identity string `json:"identity"` Cursor string `json:"cursor"` BytesWritten int64 `json:"bytes_written"` Chunks int `json:"chunks"` @@ -84,10 +92,17 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e return err } - state, err := loadAuditLogsDownloadState(statePath, outPath, fingerprint, in.Force) + state, err := loadAuditLogsDownloadState(statePath, outPath, fingerprint, c.identity, in.Force) if err != nil { return err } + // Recorded progress is only trusted when the output still matches it; + // otherwise resuming would zero-fill or extend a replaced file. + if state.Chunks > 0 { + if err := validateAuditLogsDownloadOutput(outPath, state.BytesWritten); err != nil { + return err + } + } // A state file with chunks but no cursor means the download finished and // only the cleanup was interrupted. if state.Chunks > 0 && state.Cursor == "" { @@ -104,6 +119,12 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e } defer out.Close() + // Commit state before the first fetch so an early failure retries cleanly + // instead of tripping the exists-without-state check on the next run. + if err := saveAuditLogsDownloadState(statePath, state); err != nil { + return err + } + if state.Chunks > 0 { pterm.Info.Printf("Resuming download at chunk %d (%d rows so far)\n", state.Chunks+1, state.Rows) } @@ -120,19 +141,13 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e return err } - if _, err := out.Write(body); err != nil { - return fmt.Errorf("write %s: %w", outPath, err) - } - if err := out.Sync(); err != nil { - return fmt.Errorf("sync %s: %w", outPath, err) + // A missing pagination header must not end the download as a success: + // a stripped or renamed header would silently truncate the export. + // Validated before the write so a bad response never touches the file. + hasMore, err := strconv.ParseBool(header.Get("X-Has-More")) + if err != nil { + return fmt.Errorf("response missing or invalid X-Has-More header %q", header.Get("X-Has-More")) } - - rows, _ := strconv.ParseInt(header.Get("X-Row-Count"), 10, 64) - state.Chunks++ - state.Rows += rows - state.BytesWritten += int64(len(body)) - - hasMore, _ := strconv.ParseBool(header.Get("X-Has-More")) nextCursor := header.Get("X-Next-Cursor") if hasMore { // Guard against a server bug looping the download forever. @@ -143,8 +158,19 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e return fmt.Errorf("server returned an unchanged cursor; retry, and report this if it persists") } } + + if _, err := out.Write(body); err != nil { + return fmt.Errorf("write %s: %w", outPath, err) + } + if err := out.Sync(); err != nil { + return fmt.Errorf("sync %s: %w", outPath, err) + } + + rows, _ := strconv.ParseInt(header.Get("X-Row-Count"), 10, 64) + state.Chunks++ + state.Rows += rows + state.BytesWritten += int64(len(body)) state.Cursor = nextCursor - state.Params = fingerprint if err := saveAuditLogsDownloadState(statePath, state); err != nil { return err } @@ -163,8 +189,9 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e } // fetchAuditLogsChunk downloads one chunk and verifies it against the -// X-Content-Sha256 trailer before returning it, retrying on mismatch. Nothing -// is written to disk until a chunk verifies. +// X-Content-Sha256 header before returning it, retrying on mismatch. Nothing +// is written to disk until a chunk verifies; a response without a checksum is +// rejected rather than trusted. func (c AuditLogsCmd) fetchAuditLogsChunk(ctx context.Context, params kernel.AuditLogExportChunkParams) ([]byte, http.Header, error) { var lastErr error for attempt := 0; attempt <= auditLogsDownloadShaRetries; attempt++ { @@ -172,18 +199,23 @@ func (c AuditLogsCmd) fetchAuditLogsChunk(ctx context.Context, params kernel.Aud if err != nil { return nil, nil, util.CleanedUpSdkError{Err: err} } - body, err := io.ReadAll(res.Body) + body, err := io.ReadAll(io.LimitReader(res.Body, auditLogsDownloadMaxChunkBytes+1)) res.Body.Close() if err != nil { lastErr = fmt.Errorf("read chunk body: %w", err) continue } - if want := res.Header.Get("X-Content-Sha256"); want != "" { - sum := sha256.Sum256(body) - if got := hex.EncodeToString(sum[:]); got != want { - lastErr = fmt.Errorf("chunk checksum mismatch (got %s, want %s)", got, want) - continue - } + if int64(len(body)) > auditLogsDownloadMaxChunkBytes { + return nil, nil, fmt.Errorf("chunk response exceeds %s; refusing to buffer it", util.FormatBytes(auditLogsDownloadMaxChunkBytes)) + } + want := res.Header.Get("X-Content-Sha256") + if want == "" { + return nil, nil, fmt.Errorf("response missing X-Content-Sha256 header; refusing to write unverified data") + } + sum := sha256.Sum256(body) + if got := hex.EncodeToString(sum[:]); got != want { + lastErr = fmt.Errorf("chunk checksum mismatch (got %s, want %s)", got, want) + continue } return body, res.Header, nil } @@ -289,14 +321,14 @@ func defaultAuditLogsDownloadPath(start, end time.Time, format string) string { // loadAuditLogsDownloadState decides how a download starts: fresh, resumed // from a matching state file, or rejected because the output would be -// clobbered or the state belongs to a different query. -func loadAuditLogsDownloadState(statePath, outPath, fingerprint string, force bool) (auditLogsDownloadState, error) { - fresh := auditLogsDownloadState{Version: auditLogsDownloadStateVersion, Params: fingerprint} +// clobbered or the state belongs to a different query or credential. +func loadAuditLogsDownloadState(statePath, outPath, fingerprint, identity string, force bool) (auditLogsDownloadState, error) { + fresh := auditLogsDownloadState{Version: auditLogsDownloadStateVersion, Params: fingerprint, Identity: identity} raw, err := os.ReadFile(statePath) if os.IsNotExist(err) { if !force { if _, statErr := os.Stat(outPath); statErr == nil { - return fresh, fmt.Errorf("%s already exists; pass --force to overwrite or -o to pick another path", outPath) + return fresh, fmt.Errorf("%s already exists; pass --force to overwrite or --to to pick another path", outPath) } } return fresh, nil @@ -305,6 +337,11 @@ func loadAuditLogsDownloadState(statePath, outPath, fingerprint string, force bo return fresh, fmt.Errorf("read state file: %w", err) } if force { + // Drop the old state now, not lazily: if this run dies before its + // first chunk commits, stale progress must not survive to the next. + if err := os.Remove(statePath); err != nil { + return fresh, fmt.Errorf("remove state file: %w", err) + } return fresh, nil } var state auditLogsDownloadState @@ -315,21 +352,50 @@ func loadAuditLogsDownloadState(statePath, outPath, fingerprint string, force bo return fresh, fmt.Errorf("state file %s was written by an incompatible CLI version; pass --force to start over", statePath) } if state.Params != fingerprint { - return fresh, fmt.Errorf("state file %s belongs to a download with different parameters; pass --force to start over or -o to pick another path", statePath) + return fresh, fmt.Errorf("state file %s belongs to a download with different parameters; pass --force to start over or --to to pick another path", statePath) + } + if state.Identity != identity { + return fresh, fmt.Errorf("state file %s was written with different credentials; resuming would mix organizations in one archive — pass --force to start over", statePath) } return state, nil } +// validateAuditLogsDownloadOutput checks that recorded progress still matches +// the output file. Truncate would silently zero-fill a file shorter than the +// committed offset, so a missing or shortened output must fail instead. +func validateAuditLogsDownloadOutput(outPath string, committed int64) error { + info, err := os.Stat(outPath) + if err != nil { + return fmt.Errorf("state file records progress but %s is missing or unreadable; pass --force to start over", outPath) + } + if info.Size() < committed { + return fmt.Errorf("%s is shorter than the recorded progress; it was modified or replaced — pass --force to start over", outPath) + } + return nil +} + func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) error { raw, err := json.Marshal(state) if err != nil { return fmt.Errorf("encode state: %w", err) } - tmp := statePath + ".tmp" - if err := os.WriteFile(tmp, raw, 0o644); err != nil { + // CreateTemp gives an unpredictable 0600 file, so a symlink planted at a + // guessable name in a shared directory can't redirect the write. + tmp, err := os.CreateTemp(filepath.Dir(statePath), filepath.Base(statePath)+".*") + if err != nil { + return fmt.Errorf("write state file: %w", err) + } + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + os.Remove(tmp.Name()) return fmt.Errorf("write state file: %w", err) } - if err := os.Rename(tmp, statePath); err != nil { + if err := tmp.Close(); err != nil { + os.Remove(tmp.Name()) + return fmt.Errorf("write state file: %w", err) + } + if err := os.Rename(tmp.Name(), statePath); err != nil { + os.Remove(tmp.Name()) return fmt.Errorf("commit state file: %w", err) } return nil @@ -340,11 +406,12 @@ func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) // than duplicated on resume. func openAuditLogsDownloadOutput(outPath string, committed int64) (*os.File, error) { if dir := filepath.Dir(outPath); dir != "." { - if err := os.MkdirAll(dir, 0o755); err != nil { + if err := os.MkdirAll(dir, 0o700); err != nil { return nil, fmt.Errorf("create output directory: %w", err) } } - out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE, 0o644) + // 0600: audit logs carry user emails and client IPs. + out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE, 0o600) if err != nil { return nil, fmt.Errorf("open %s: %w", outPath, err) } @@ -359,6 +426,21 @@ func openAuditLogsDownloadOutput(outPath string, committed int64) (*os.File, err return out, nil } +// auditLogsCredentialIdentity identifies the credential a download runs under +// so resume state is never applied across organizations. API keys are hashed +// rather than stored; OAuth sessions use the org ID, which unlike the access +// token is stable across refreshes. +func auditLogsCredentialIdentity() string { + if key := os.Getenv("KERNEL_API_KEY"); key != "" { + sum := sha256.Sum256([]byte("kernel-cli-audit-logs-download:" + key)) + return "key:" + hex.EncodeToString(sum[:]) + } + if tokens, err := auth.LoadTokens(); err == nil && tokens.OrgID != "" { + return "org:" + tokens.OrgID + } + return "" +} + func runAuditLogsDownload(cmd *cobra.Command, args []string) error { c := getAuditLogsHandler(cmd) start, _ := cmd.Flags().GetString("start") @@ -394,7 +476,7 @@ var auditLogsDownloadCmd = &cobra.Command{ Use: "download", Short: "Download audit logs for a time range to a file", Long: "Download an organization's audit log records for a time range as a file, for archival, compliance, or offline analysis.\n\n" + - "Records are fetched in chunks and appended after each chunk is verified, newest first. If a download is interrupted, rerunning the same command resumes where it left off.\n\n" + + "Records are fetched in chunks and appended after each chunk is verified, newest first. If a download is interrupted, rerunning the same command resumes where it left off. Resuming requires explicit --start and --end, since the default bounds move with the current time.\n\n" + "GET requests are excluded by default; pass --include-get to include them, or --method GET to see only them.\n\n" + "The API allows at most 30 days per download; for longer ranges run one download per window.", Args: cobra.NoArgs, diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index 236ae60f..ffa4c858 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -21,11 +21,13 @@ import ( ) type exportChunk struct { - body []byte - rowCount int - hasMore bool - nextCursor string - sha string // overrides the computed body sha when set + body []byte + rowCount int + hasMore bool + nextCursor string + sha string // overrides the computed body sha when set + omitSha bool + omitHasMore bool } func exportChunkResponse(c exportChunk) *http.Response { @@ -35,9 +37,13 @@ func exportChunkResponse(c exportChunk) *http.Response { sha = hex.EncodeToString(sum[:]) } header := http.Header{} - header.Set("X-Has-More", strconv.FormatBool(c.hasMore)) + if !c.omitHasMore { + header.Set("X-Has-More", strconv.FormatBool(c.hasMore)) + } header.Set("X-Row-Count", strconv.Itoa(c.rowCount)) - header.Set("X-Content-Sha256", sha) + if !c.omitSha { + header.Set("X-Content-Sha256", sha) + } if c.nextCursor != "" { header.Set("X-Next-Cursor", c.nextCursor) } @@ -417,6 +423,200 @@ func TestAuditLogsDownloadExcludesGetByDefault(t *testing.T) { } } +func TestAuditLogsDownloadRejectsDifferentCredentials(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk1 := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + c := AuditLogsCmd{auditLogs: fake, identity: "org:org-a"} + require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + + c2 := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}, identity: "org:org-b"} + err := c2.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "different credentials") +} + +func TestAuditLogsDownloadRejectsMissingOrShortenedOutput(t *testing.T) { + capturePtermOutput(t) + interrupted := func(t *testing.T) (string, AuditLogsCmd) { + t.Helper() + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk1 := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + c := AuditLogsCmd{auditLogs: fake} + require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + return outPath, AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + } + + t.Run("missing output", func(t *testing.T) { + outPath, c := interrupted(t) + require.NoError(t, os.Remove(outPath)) + err := c.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "missing or unreadable") + }) + + t.Run("shortened output", func(t *testing.T) { + outPath, c := interrupted(t) + require.NoError(t, os.Truncate(outPath, 1)) + err := c.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "shorter than the recorded progress") + }) +} + +func TestAuditLogsDownloadFirstChunkFailureRetriesWithoutForce(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + failing, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + c := AuditLogsCmd{auditLogs: failing} + require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + + chunk := gzipMember(t, "{\"n\":1}\n") + fake, cursors := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: false}), nil + }, + }) + c2 := AuditLogsCmd{auditLogs: fake} + require.NoError(t, c2.Download(context.Background(), downloadInput(outPath)), "a failed first chunk must not require --force to retry") + + assert.Equal(t, []string{""}, *cursors) + data, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "{\"n\":1}\n", gunzipAll(t, data)) +} + +func TestAuditLogsDownloadFailedForceLeavesNoStaleProgress(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk1 := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + c := AuditLogsCmd{auditLogs: fake} + require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + + // --force run that dies before its first chunk. + failing, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + in := downloadInput(outPath) + in.Force = true + c2 := AuditLogsCmd{auditLogs: failing} + require.Error(t, c2.Download(context.Background(), in)) + + // The plain rerun must start from scratch, not resume chunk 1's stale cursor. + chunk := gzipMember(t, "{\"n\":9}\n") + fake3, cursors := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: false}), nil + }, + }) + c3 := AuditLogsCmd{auditLogs: fake3} + require.NoError(t, c3.Download(context.Background(), downloadInput(outPath))) + + assert.Equal(t, []string{""}, *cursors) + data, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "{\"n\":9}\n", gunzipAll(t, data)) +} + +func TestAuditLogsDownloadFailsClosedOnMissingHeaders(t *testing.T) { + capturePtermOutput(t) + chunk := gzipMember(t, "{\"n\":1}\n") + + t.Run("missing X-Has-More", func(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, omitHasMore: true}), nil + }, + }) + c := AuditLogsCmd{auditLogs: fake} + err := c.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "X-Has-More") + data, readErr := os.ReadFile(outPath) + require.NoError(t, readErr) + assert.Empty(t, data, "an unvalidated response must not be written") + }) + + t.Run("missing X-Content-Sha256", func(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, omitSha: true}), nil + }, + }) + c := AuditLogsCmd{auditLogs: fake} + err := c.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "X-Content-Sha256") + }) +} + +func TestAuditLogsDownloadRejectsOversizedChunk(t *testing.T) { + capturePtermOutput(t) + prev := auditLogsDownloadMaxChunkBytes + auditLogsDownloadMaxChunkBytes = 8 + t.Cleanup(func() { auditLogsDownloadMaxChunkBytes = prev }) + + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: []byte("well over eight bytes"), rowCount: 1}), nil + }, + }) + c := AuditLogsCmd{auditLogs: fake} + err := c.Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "refusing to buffer") +} + +func TestAuditLogsDownloadFilePermissions(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + c := AuditLogsCmd{auditLogs: fake} + require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + + outInfo, err := os.Stat(outPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), outInfo.Mode().Perm()) + stateInfo, err := os.Stat(outPath + ".state.json") + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), stateInfo.Mode().Perm()) +} + func TestAuditLogsDownloadRejectsInvalidFormat(t *testing.T) { c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} in := downloadInput("out.csv") From eea29a97a13ac8a63ba2779383b5737d1222e901 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:01:26 +0000 Subject: [PATCH 04/17] Harden audit log download recovery --- cmd/audit_logs.go | 5 +- cmd/audit_logs_download.go | 451 ++++++++++++++++++++++------- cmd/audit_logs_download_test.go | 267 ++++++++++++++--- cmd/audit_logs_filelock_other.go | 16 + cmd/audit_logs_filelock_unix.go | 25 ++ cmd/audit_logs_filelock_windows.go | 32 ++ go.mod | 2 +- 7 files changed, 644 insertions(+), 154 deletions(-) create mode 100644 cmd/audit_logs_filelock_other.go create mode 100644 cmd/audit_logs_filelock_unix.go create mode 100644 cmd/audit_logs_filelock_windows.go diff --git a/cmd/audit_logs.go b/cmd/audit_logs.go index 9497ec4e..b6ae9eef 100644 --- a/cmd/audit_logs.go +++ b/cmd/audit_logs.go @@ -23,8 +23,7 @@ type AuditLogsService interface { type AuditLogsCmd struct { auditLogs AuditLogsService - // identity distinguishes the credential in use; download resume state is - // bound to it so archives never mix organizations. + // identity binds download resume state to an API origin and credential. identity string } @@ -191,7 +190,7 @@ func formatAuditLogUser(entry kernel.AuditLogEntry) string { func getAuditLogsHandler(cmd *cobra.Command) AuditLogsCmd { client := getKernelClient(cmd) - return AuditLogsCmd{auditLogs: &client.AuditLogs, identity: auditLogsCredentialIdentity()} + return AuditLogsCmd{auditLogs: &client.AuditLogs} } func runAuditLogsSearch(cmd *cobra.Command, args []string) error { diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index d74f15b1..0448bc3d 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -6,8 +6,11 @@ import ( "encoding/hex" "encoding/json" "fmt" + "hash" "io" + "math" "net/http" + "net/url" "os" "path/filepath" "strconv" @@ -42,6 +45,11 @@ const ( // auditLogsDownloadShaRetries is the number of extra attempts for a chunk // whose body fails checksum verification. auditLogsDownloadShaRetries = 2 + // auditLogsDownloadMaxRowsPerChunk mirrors the export endpoint's limit. + auditLogsDownloadMaxRowsPerChunk = 50_000 + // auditLogsDownloadMaxStateBytes bounds the local checkpoint before it is + // decoded. Valid state files are under a few kilobytes. + auditLogsDownloadMaxStateBytes = 64 << 10 ) // auditLogsDownloadMaxChunkBytes bounds how much of a chunk response is @@ -54,18 +62,25 @@ var auditLogsDownloadMaxChunkBytes int64 = 256 << 20 // It is written atomically after every committed chunk and removed on // completion. Version guards future format changes (e.g. multi-window splits). type auditLogsDownloadState struct { - Version int `json:"version"` - Params string `json:"params"` - Identity string `json:"identity"` - Cursor string `json:"cursor"` - BytesWritten int64 `json:"bytes_written"` - Chunks int `json:"chunks"` - Rows int64 `json:"rows"` + Version int `json:"version"` + Params string `json:"params"` + Identity string `json:"identity"` + Cursor string `json:"cursor"` + BytesWritten int64 `json:"bytes_written"` + CommittedSHA256 string `json:"committed_sha256"` + Chunks int `json:"chunks"` + Rows int64 `json:"rows"` } -const auditLogsDownloadStateVersion = 1 +const auditLogsDownloadStateVersion = 2 + +const auditLogsDownloadEmptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) error { + if c.identity == "" { + return fmt.Errorf("cannot safely resume audit-log download without an API origin and credential identity") + } + params, err := buildAuditLogsDownloadParams(in) if err != nil { return err @@ -92,37 +107,50 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e return err } - state, err := loadAuditLogsDownloadState(statePath, outPath, fingerprint, c.identity, in.Force) + out, created, err := openAndLockAuditLogsDownloadOutput(outPath) if err != nil { return err } - // Recorded progress is only trusted when the output still matches it; - // otherwise resuming would zero-fill or extend a replaced file. - if state.Chunks > 0 { - if err := validateAuditLogsDownloadOutput(outPath, state.BytesWritten); err != nil { - return err + // Keep a newly created output only after it has a valid matching state. + removeCreatedOutput := created + defer func() { + _ = out.Close() + if removeCreatedOutput { + _ = os.Remove(outPath) } + }() + + state, stateExists, err := loadAuditLogsDownloadState(statePath, fingerprint, c.identity, in.Force) + if err != nil { + return err } - // A state file with chunks but no cursor means the download finished and - // only the cleanup was interrupted. - if state.Chunks > 0 && state.Cursor == "" { - if err := os.Remove(statePath); err != nil { - return fmt.Errorf("remove state file: %w", err) + if !stateExists && !in.Force && !created { + return fmt.Errorf("%s already exists; pass --force to overwrite or --to to pick another path", outPath) + } + if stateExists && state.Chunks > 0 && created { + return fmt.Errorf("state file records progress but %s is missing; pass --force to start over", outPath) + } + if !stateExists { + if err := saveAuditLogsDownloadState(statePath, state); err != nil { + return err } - pterm.Success.Printf("Download already complete: %d rows (%s) in %s\n", state.Rows, util.FormatBytes(state.BytesWritten), outPath) - return nil } - out, err := openAuditLogsDownloadOutput(outPath, state.BytesWritten) + completed := state.Chunks > 0 && state.Cursor == "" + committedHash, err := prepareAuditLogsDownloadOutput(out.file, outPath, state, completed) if err != nil { return err } - defer out.Close() + removeCreatedOutput = false - // Commit state before the first fetch so an early failure retries cleanly - // instead of tripping the exists-without-state check on the next run. - if err := saveAuditLogsDownloadState(statePath, state); err != nil { - return err + // A state file with chunks but no cursor means the download finished and + // only the cleanup was interrupted. + if completed { + if err := removeAuditLogsDownloadState(statePath); err != nil { + return err + } + pterm.Success.Printf("Download already complete: %d rows (%s) in %s\n", state.Rows, util.FormatBytes(state.BytesWritten), outPath) + return nil } if state.Chunks > 0 { @@ -148,6 +176,11 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if err != nil { return fmt.Errorf("response missing or invalid X-Has-More header %q", header.Get("X-Has-More")) } + rowsHeader := header.Get("X-Row-Count") + rows, err := strconv.ParseInt(rowsHeader, 10, 64) + if err != nil || rows < 0 || rows > auditLogsDownloadMaxRowsPerChunk { + return fmt.Errorf("response missing or invalid X-Row-Count header %q", rowsHeader) + } nextCursor := header.Get("X-Next-Cursor") if hasMore { // Guard against a server bug looping the download forever. @@ -157,19 +190,32 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if nextCursor == state.Cursor { return fmt.Errorf("server returned an unchanged cursor; retry, and report this if it persists") } + } else if nextCursor != "" { + return fmt.Errorf("server returned a cursor after reporting no more records; retry, and report this if it persists") + } + if state.Rows > math.MaxInt64-rows { + return fmt.Errorf("row count exceeds the supported range") + } + bodyBytes := int64(len(body)) + if state.BytesWritten > math.MaxInt64-bodyBytes { + return fmt.Errorf("download size exceeds the supported range") + } + if state.Chunks == math.MaxInt { + return fmt.Errorf("chunk count exceeds the supported range") } - if _, err := out.Write(body); err != nil { + if _, err := out.file.Write(body); err != nil { return fmt.Errorf("write %s: %w", outPath, err) } - if err := out.Sync(); err != nil { + if err := out.file.Sync(); err != nil { return fmt.Errorf("sync %s: %w", outPath, err) } + _, _ = committedHash.Write(body) - rows, _ := strconv.ParseInt(header.Get("X-Row-Count"), 10, 64) state.Chunks++ state.Rows += rows - state.BytesWritten += int64(len(body)) + state.BytesWritten += bodyBytes + state.CommittedSHA256 = hex.EncodeToString(committedHash.Sum(nil)) state.Cursor = nextCursor if err := saveAuditLogsDownloadState(statePath, state); err != nil { return err @@ -181,8 +227,8 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e } } - if err := os.Remove(statePath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("remove state file: %w", err) + if err := removeAuditLogsDownloadState(statePath); err != nil { + return err } pterm.Success.Printf("Downloaded %d rows (%s) to %s\n", state.Rows, util.FormatBytes(state.BytesWritten), outPath) return nil @@ -224,24 +270,20 @@ func (c AuditLogsCmd) fetchAuditLogsChunk(ctx context.Context, params kernel.Aud func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExportChunkParams, error) { var params kernel.AuditLogExportChunkParams - var err error - start := time.Now().UTC().Add(-24 * time.Hour) - if in.Start != "" { - start, _, err = parseAuditLogTime(in.Start) - if err != nil { - return params, fmt.Errorf("--start: %w", err) - } + if in.Start == "" || in.End == "" { + return params, fmt.Errorf("--start and --end are required so interrupted downloads can resume with the same time range") } - end := time.Now().UTC() - if in.End != "" { - var dateOnly bool - end, dateOnly, err = parseAuditLogTime(in.End) - if err != nil { - return params, fmt.Errorf("--end: %w", err) - } - if dateOnly { - end = end.Add(24 * time.Hour) - } + + start, _, err := parseAuditLogTime(in.Start) + if err != nil { + return params, fmt.Errorf("--start: %w", err) + } + end, dateOnly, err := parseAuditLogTime(in.End) + if err != nil { + return params, fmt.Errorf("--end: %w", err) + } + if dateOnly { + end = end.Add(24 * time.Hour) } if !start.Before(end) { return params, fmt.Errorf("--start must be before --end") @@ -319,57 +361,115 @@ func defaultAuditLogsDownloadPath(start, end time.Time, format string) string { return fmt.Sprintf("audit-logs-%s-%s.%s", start.UTC().Format(stamp), end.UTC().Format(stamp), format) } -// loadAuditLogsDownloadState decides how a download starts: fresh, resumed -// from a matching state file, or rejected because the output would be -// clobbered or the state belongs to a different query or credential. -func loadAuditLogsDownloadState(statePath, outPath, fingerprint, identity string, force bool) (auditLogsDownloadState, error) { - fresh := auditLogsDownloadState{Version: auditLogsDownloadStateVersion, Params: fingerprint, Identity: identity} - raw, err := os.ReadFile(statePath) - if os.IsNotExist(err) { - if !force { - if _, statErr := os.Stat(outPath); statErr == nil { - return fresh, fmt.Errorf("%s already exists; pass --force to overwrite or --to to pick another path", outPath) - } +// loadAuditLogsDownloadState decides whether a download starts fresh or resumes +// from a checkpoint bound to the same query, API origin, and credential. +func loadAuditLogsDownloadState(statePath, fingerprint, identity string, force bool) (auditLogsDownloadState, bool, error) { + fresh := auditLogsDownloadState{ + Version: auditLogsDownloadStateVersion, + Params: fingerprint, + Identity: identity, + CommittedSHA256: auditLogsDownloadEmptySHA256, + } + if force { + if err := removeAuditLogsDownloadState(statePath); err != nil { + return fresh, false, err } - return fresh, nil + return fresh, false, nil } + + raw, exists, err := readAuditLogsDownloadState(statePath) if err != nil { - return fresh, fmt.Errorf("read state file: %w", err) + return fresh, false, err } - if force { - // Drop the old state now, not lazily: if this run dies before its - // first chunk commits, stale progress must not survive to the next. - if err := os.Remove(statePath); err != nil { - return fresh, fmt.Errorf("remove state file: %w", err) - } - return fresh, nil + if !exists { + return fresh, false, nil } + var state auditLogsDownloadState if err := json.Unmarshal(raw, &state); err != nil { - return fresh, fmt.Errorf("state file %s is corrupt; pass --force to start over", statePath) + return fresh, false, fmt.Errorf("state file %s is corrupt; pass --force to start over", statePath) } if state.Version != auditLogsDownloadStateVersion { - return fresh, fmt.Errorf("state file %s was written by an incompatible CLI version; pass --force to start over", statePath) + return fresh, false, fmt.Errorf("state file %s was written by an incompatible CLI version; pass --force to start over", statePath) } if state.Params != fingerprint { - return fresh, fmt.Errorf("state file %s belongs to a download with different parameters; pass --force to start over or --to to pick another path", statePath) + return fresh, false, fmt.Errorf("state file %s belongs to a download with different parameters; pass --force to start over or --to to pick another path", statePath) } if state.Identity != identity { - return fresh, fmt.Errorf("state file %s was written with different credentials; resuming would mix organizations in one archive — pass --force to start over", statePath) + return fresh, false, fmt.Errorf("state file %s was written for a different API origin or credential; pass --force to start over", statePath) + } + if err := validateAuditLogsDownloadState(state); err != nil { + return fresh, false, fmt.Errorf("state file %s is inconsistent (%v); pass --force to start over", statePath, err) + } + return state, true, nil +} + +func readAuditLogsDownloadState(statePath string) ([]byte, bool, error) { + info, err := os.Lstat(statePath) + if os.IsNotExist(err) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("read state file: %w", err) + } + if !info.Mode().IsRegular() { + return nil, false, fmt.Errorf("state file %s is not a regular file; pass --force to start over", statePath) + } + if info.Size() > auditLogsDownloadMaxStateBytes { + return nil, false, fmt.Errorf("state file %s is too large; pass --force to start over", statePath) + } + + f, err := os.Open(statePath) + if err != nil { + return nil, false, fmt.Errorf("read state file: %w", err) + } + defer f.Close() + openedInfo, err := f.Stat() + if err != nil { + return nil, false, fmt.Errorf("read state file: %w", err) } - return state, nil + if !openedInfo.Mode().IsRegular() || !os.SameFile(info, openedInfo) { + return nil, false, fmt.Errorf("state file %s changed while it was being opened; retry", statePath) + } + raw, err := io.ReadAll(io.LimitReader(f, auditLogsDownloadMaxStateBytes+1)) + if err != nil { + return nil, false, fmt.Errorf("read state file: %w", err) + } + if len(raw) > auditLogsDownloadMaxStateBytes { + return nil, false, fmt.Errorf("state file %s is too large; pass --force to start over", statePath) + } + return raw, true, nil } -// validateAuditLogsDownloadOutput checks that recorded progress still matches -// the output file. Truncate would silently zero-fill a file shorter than the -// committed offset, so a missing or shortened output must fail instead. -func validateAuditLogsDownloadOutput(outPath string, committed int64) error { - info, err := os.Stat(outPath) +func removeAuditLogsDownloadState(statePath string) error { + info, err := os.Lstat(statePath) + if os.IsNotExist(err) { + return nil + } if err != nil { - return fmt.Errorf("state file records progress but %s is missing or unreadable; pass --force to start over", outPath) + return fmt.Errorf("remove state file: %w", err) } - if info.Size() < committed { - return fmt.Errorf("%s is shorter than the recorded progress; it was modified or replaced — pass --force to start over", outPath) + if info.IsDir() { + return fmt.Errorf("state path %s is a directory; remove it before using --force", statePath) + } + if err := os.Remove(statePath); err != nil { + return fmt.Errorf("remove state file: %w", err) + } + return nil +} + +func validateAuditLogsDownloadState(state auditLogsDownloadState) error { + if state.BytesWritten < 0 || state.Chunks < 0 || state.Rows < 0 { + return fmt.Errorf("negative progress counters") + } + digest, err := hex.DecodeString(state.CommittedSHA256) + if err != nil || len(digest) != sha256.Size { + return fmt.Errorf("invalid committed checksum") + } + if state.Chunks == 0 { + if state.Cursor != "" || state.BytesWritten != 0 || state.Rows != 0 || state.CommittedSHA256 != auditLogsDownloadEmptySHA256 { + return fmt.Errorf("zero-chunk state contains committed progress") + } } return nil } @@ -379,6 +479,9 @@ func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) if err != nil { return fmt.Errorf("encode state: %w", err) } + if len(raw) > auditLogsDownloadMaxStateBytes { + return fmt.Errorf("state file would exceed %s; narrow the download filters", util.FormatBytes(auditLogsDownloadMaxStateBytes)) + } // CreateTemp gives an unpredictable 0600 file, so a symlink planted at a // guessable name in a shared directory can't redirect the write. tmp, err := os.CreateTemp(filepath.Dir(statePath), filepath.Base(statePath)+".*") @@ -390,6 +493,11 @@ func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) os.Remove(tmp.Name()) return fmt.Errorf("write state file: %w", err) } + if err := tmp.Sync(); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return fmt.Errorf("sync state file: %w", err) + } if err := tmp.Close(); err != nil { os.Remove(tmp.Name()) return fmt.Errorf("write state file: %w", err) @@ -401,48 +509,171 @@ func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) return nil } -// openAuditLogsDownloadOutput opens the output for appending, truncated to the -// last committed offset so a chunk interrupted mid-write is dropped rather -// than duplicated on resume. -func openAuditLogsDownloadOutput(outPath string, committed int64) (*os.File, error) { +type lockedAuditLogsDownloadOutput struct { + file *os.File +} + +func (o *lockedAuditLogsDownloadOutput) Close() error { + unlockErr := unlockAuditLogsDownloadFile(o.file) + closeErr := o.file.Close() + if unlockErr != nil { + return unlockErr + } + return closeErr +} + +// openAndLockAuditLogsDownloadOutput securely opens a regular output file and +// locks it before any state is read or output is modified. +func openAndLockAuditLogsDownloadOutput(outPath string) (*lockedAuditLogsDownloadOutput, bool, error) { if dir := filepath.Dir(outPath); dir != "." { if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, fmt.Errorf("create output directory: %w", err) + return nil, false, fmt.Errorf("create output directory: %w", err) } } - // 0600: audit logs carry user emails and client IPs. - out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE, 0o600) + + for attempt := 0; attempt < 2; attempt++ { + info, err := os.Lstat(outPath) + created := false + var out *os.File + switch { + case os.IsNotExist(err): + out, err = os.OpenFile(outPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if os.IsExist(err) { + continue + } + created = err == nil + case err != nil: + return nil, false, fmt.Errorf("inspect %s: %w", outPath, err) + case !info.Mode().IsRegular(): + return nil, false, fmt.Errorf("%s must be a regular file; refusing to follow a symlink or special file", outPath) + default: + out, err = os.OpenFile(outPath, os.O_RDWR, 0) + if err == nil { + openedInfo, statErr := out.Stat() + if statErr != nil { + out.Close() + return nil, false, fmt.Errorf("inspect %s: %w", outPath, statErr) + } + if !openedInfo.Mode().IsRegular() || !os.SameFile(info, openedInfo) { + out.Close() + return nil, false, fmt.Errorf("%s changed while it was being opened; retry", outPath) + } + } + } + if err != nil { + return nil, false, fmt.Errorf("open %s: %w", outPath, err) + } + locked, err := tryLockAuditLogsDownloadFile(out) + // A second process can open and briefly lock a just-created file before + // its creator reaches the lock call. The creator retries long enough for + // that process to observe the missing state and release its lock. + for retries := 0; created && !locked && err == nil && retries < 25; retries++ { + time.Sleep(10 * time.Millisecond) + locked, err = tryLockAuditLogsDownloadFile(out) + } + if err != nil { + out.Close() + if created { + _ = os.Remove(outPath) + } + return nil, false, fmt.Errorf("lock %s: %w", outPath, err) + } + if !locked { + out.Close() + return nil, false, fmt.Errorf("another audit-log download is already using %s", outPath) + } + return &lockedAuditLogsDownloadOutput{file: out}, created, nil + } + return nil, false, fmt.Errorf("%s changed while it was being opened; retry", outPath) +} + +// prepareAuditLogsDownloadOutput verifies the committed prefix before +// truncating an interrupted append and positioning the file for the next chunk. +func prepareAuditLogsDownloadOutput(out *os.File, outPath string, state auditLogsDownloadState, completed bool) (hash.Hash, error) { + if err := out.Chmod(0o600); err != nil { + return nil, fmt.Errorf("secure permissions on %s: %w", outPath, err) + } + info, err := out.Stat() if err != nil { - return nil, fmt.Errorf("open %s: %w", outPath, err) + return nil, fmt.Errorf("inspect %s: %w", outPath, err) + } + if info.Size() < state.BytesWritten { + return nil, fmt.Errorf("%s is shorter than the recorded progress; it was modified or replaced — pass --force to start over", outPath) + } + if completed && info.Size() != state.BytesWritten { + return nil, fmt.Errorf("%s does not match the completed download size; pass --force to start over", outPath) + } + if _, err := out.Seek(0, io.SeekStart); err != nil { + return nil, fmt.Errorf("seek %s: %w", outPath, err) + } + committedHash := sha256.New() + if _, err := io.CopyN(committedHash, out, state.BytesWritten); err != nil { + return nil, fmt.Errorf("verify %s: %w", outPath, err) } - if err := out.Truncate(committed); err != nil { - out.Close() - return nil, fmt.Errorf("truncate %s to last committed chunk: %w", outPath, err) + got := hex.EncodeToString(committedHash.Sum(nil)) + if got != state.CommittedSHA256 { + return nil, fmt.Errorf("%s does not match the recorded checksum; it was modified or replaced — pass --force to start over", outPath) } - if _, err := out.Seek(committed, io.SeekStart); err != nil { - out.Close() + if !completed { + if err := out.Truncate(state.BytesWritten); err != nil { + return nil, fmt.Errorf("truncate %s to last committed chunk: %w", outPath, err) + } + } + if _, err := out.Seek(state.BytesWritten, io.SeekStart); err != nil { return nil, fmt.Errorf("seek %s: %w", outPath, err) } - return out, nil + return committedHash, nil } -// auditLogsCredentialIdentity identifies the credential a download runs under -// so resume state is never applied across organizations. API keys are hashed -// rather than stored; OAuth sessions use the org ID, which unlike the access -// token is stable across refreshes. -func auditLogsCredentialIdentity() string { +// auditLogsCredentialIdentity binds resume state to both the effective API +// origin and the authenticated organization or API key. +func auditLogsCredentialIdentity() (string, error) { + baseURL, err := normalizeAuditLogsAPIBaseURL(util.GetBaseURL()) + if err != nil { + return "", err + } if key := os.Getenv("KERNEL_API_KEY"); key != "" { sum := sha256.Sum256([]byte("kernel-cli-audit-logs-download:" + key)) - return "key:" + hex.EncodeToString(sum[:]) + return baseURL + "|key:" + hex.EncodeToString(sum[:]), nil } - if tokens, err := auth.LoadTokens(); err == nil && tokens.OrgID != "" { - return "org:" + tokens.OrgID + tokens, err := auth.LoadTokens() + if err != nil { + return "", fmt.Errorf("determine OAuth organization for resumable download: %w", err) + } + if tokens.OrgID == "" { + return "", fmt.Errorf("cannot determine OAuth organization for resumable download; run 'kernel login --force' and retry") } - return "" + return baseURL + "|org:" + tokens.OrgID, nil +} + +func normalizeAuditLogsAPIBaseURL(raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Scheme == "" || u.Host == "" { + return "", fmt.Errorf("invalid KERNEL_BASE_URL %q", raw) + } + if u.Scheme != "http" && u.Scheme != "https" { + return "", fmt.Errorf("invalid KERNEL_BASE_URL scheme %q", u.Scheme) + } + if u.User != nil { + return "", fmt.Errorf("KERNEL_BASE_URL must not contain user information") + } + if u.RawQuery != "" || u.Fragment != "" { + return "", fmt.Errorf("KERNEL_BASE_URL must not contain a query or fragment") + } + u.Scheme = strings.ToLower(u.Scheme) + u.Host = strings.ToLower(u.Host) + u.Path = strings.TrimRight(u.Path, "/") + u.RawPath = strings.TrimRight(u.RawPath, "/") + return u.String(), nil } func runAuditLogsDownload(cmd *cobra.Command, args []string) error { c := getAuditLogsHandler(cmd) + identity, err := auditLogsCredentialIdentity() + if err != nil { + return err + } + c.identity = identity start, _ := cmd.Flags().GetString("start") end, _ := cmd.Flags().GetString("end") search, _ := cmd.Flags().GetString("search") @@ -476,7 +707,7 @@ var auditLogsDownloadCmd = &cobra.Command{ Use: "download", Short: "Download audit logs for a time range to a file", Long: "Download an organization's audit log records for a time range as a file, for archival, compliance, or offline analysis.\n\n" + - "Records are fetched in chunks and appended after each chunk is verified, newest first. If a download is interrupted, rerunning the same command resumes where it left off. Resuming requires explicit --start and --end, since the default bounds move with the current time.\n\n" + + "Records are fetched in chunks and appended after each chunk is verified, newest first. If a download is interrupted, rerunning the same command resumes where it left off.\n\n" + "GET requests are excluded by default; pass --include-get to include them, or --method GET to see only them.\n\n" + "The API allows at most 30 days per download; for longer ranges run one download per window.", Args: cobra.NoArgs, @@ -484,8 +715,8 @@ var auditLogsDownloadCmd = &cobra.Command{ } func init() { - auditLogsDownloadCmd.Flags().String("start", "", "Start of the export window, RFC3339 or YYYY-MM-DD (default: 24 hours ago)") - auditLogsDownloadCmd.Flags().String("end", "", "End of the export window, RFC3339 or YYYY-MM-DD inclusive (default: now)") + auditLogsDownloadCmd.Flags().String("start", "", "Start of the export window, RFC3339 or YYYY-MM-DD (required)") + auditLogsDownloadCmd.Flags().String("end", "", "End of the export window, RFC3339 or YYYY-MM-DD inclusive (required)") auditLogsDownloadCmd.Flags().String("search", "", "Free-text search") auditLogsDownloadCmd.Flags().String("method", "", "Filter by HTTP method (e.g. GET)") auditLogsDownloadCmd.Flags().String("exclude-method", "", "Exclude an HTTP method") @@ -496,6 +727,8 @@ func init() { auditLogsDownloadCmd.Flags().String("to", "", "Output file path (default: audit-logs--.)") auditLogsDownloadCmd.Flags().String("format", "jsonl.gz", "Output format: jsonl or jsonl.gz") auditLogsDownloadCmd.Flags().Bool("force", false, "Overwrite the output file and ignore saved progress") + _ = auditLogsDownloadCmd.MarkFlagRequired("start") + _ = auditLogsDownloadCmd.MarkFlagRequired("end") auditLogsCmd.AddCommand(auditLogsDownloadCmd) } diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index ffa4c858..14dd7625 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -21,13 +21,15 @@ import ( ) type exportChunk struct { - body []byte - rowCount int - hasMore bool - nextCursor string - sha string // overrides the computed body sha when set - omitSha bool - omitHasMore bool + body []byte + rowCount int + rowCountRaw string + hasMore bool + nextCursor string + sha string // overrides the computed body sha when set + omitSha bool + omitHasMore bool + omitRowCount bool } func exportChunkResponse(c exportChunk) *http.Response { @@ -40,7 +42,13 @@ func exportChunkResponse(c exportChunk) *http.Response { if !c.omitHasMore { header.Set("X-Has-More", strconv.FormatBool(c.hasMore)) } - header.Set("X-Row-Count", strconv.Itoa(c.rowCount)) + if !c.omitRowCount { + rowCount := c.rowCountRaw + if rowCount == "" { + rowCount = strconv.Itoa(c.rowCount) + } + header.Set("X-Row-Count", rowCount) + } if !c.omitSha { header.Set("X-Content-Sha256", sha) } @@ -96,6 +104,12 @@ func downloadInput(outPath string) AuditLogsDownloadInput { } } +const auditLogsDownloadTestIdentity = "https://api.test|org:org-test" + +func auditLogsDownloadTestCmd(service AuditLogsService) AuditLogsCmd { + return AuditLogsCmd{auditLogs: service, identity: auditLogsDownloadTestIdentity} +} + func TestAuditLogsDownloadMultiChunk(t *testing.T) { capturePtermOutput(t) outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") @@ -115,7 +129,7 @@ func TestAuditLogsDownloadMultiChunk(t *testing.T) { }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.NoError(t, c.Download(context.Background(), downloadInput(outPath))) assert.Equal(t, []string{"", "c1", "c2"}, *cursors) @@ -139,7 +153,7 @@ func TestAuditLogsDownloadChecksumMismatchRetriesThenSucceeds(t *testing.T) { }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.NoError(t, c.Download(context.Background(), downloadInput(outPath))) data, err := os.ReadFile(outPath) @@ -156,7 +170,7 @@ func TestAuditLogsDownloadChecksumMismatchExhaustsRetries(t *testing.T) { } fake, _ := chunkServer(t, []func() (*http.Response, error){bad, bad, bad}) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) err := c.Download(context.Background(), downloadInput(outPath)) require.ErrorContains(t, err, "checksum mismatch") @@ -179,7 +193,7 @@ func TestAuditLogsDownloadResumesAfterFailure(t *testing.T) { func() (*http.Response, error) { return nil, errors.New("boom") }, func() (*http.Response, error) { return nil, errors.New("boom") }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.Error(t, c.Download(context.Background(), downloadInput(outPath))) fake2, cursors := chunkServer(t, []func() (*http.Response, error){ @@ -187,7 +201,7 @@ func TestAuditLogsDownloadResumesAfterFailure(t *testing.T) { return exportChunkResponse(exportChunk{body: chunk2, rowCount: 1, hasMore: false}), nil }, }) - c2 := AuditLogsCmd{auditLogs: fake2} + c2 := auditLogsDownloadTestCmd(fake2) require.NoError(t, c2.Download(context.Background(), downloadInput(outPath))) assert.Equal(t, []string{"c1"}, *cursors, "resume must continue from the saved cursor") @@ -212,7 +226,7 @@ func TestAuditLogsDownloadResumeTruncatesTornWrite(t *testing.T) { func() (*http.Response, error) { return nil, errors.New("boom") }, func() (*http.Response, error) { return nil, errors.New("boom") }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.Error(t, c.Download(context.Background(), downloadInput(outPath))) // Simulate a crash mid-append: garbage past the committed offset. @@ -227,7 +241,7 @@ func TestAuditLogsDownloadResumeTruncatesTornWrite(t *testing.T) { return exportChunkResponse(exportChunk{body: chunk2, rowCount: 1, hasMore: false}), nil }, }) - c2 := AuditLogsCmd{auditLogs: fake2} + c2 := auditLogsDownloadTestCmd(fake2) require.NoError(t, c2.Download(context.Background(), downloadInput(outPath))) data, err := os.ReadFile(outPath) @@ -247,7 +261,7 @@ func TestAuditLogsDownloadRejectsMismatchedState(t *testing.T) { func() (*http.Response, error) { return nil, errors.New("boom") }, func() (*http.Response, error) { return nil, errors.New("boom") }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.Error(t, c.Download(context.Background(), downloadInput(outPath))) in := downloadInput(outPath) @@ -268,7 +282,7 @@ func TestAuditLogsDownloadForceRestarts(t *testing.T) { func() (*http.Response, error) { return nil, errors.New("boom") }, func() (*http.Response, error) { return nil, errors.New("boom") }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.Error(t, c.Download(context.Background(), downloadInput(outPath))) chunk := gzipMember(t, "{\"n\":9}\n") @@ -279,7 +293,7 @@ func TestAuditLogsDownloadForceRestarts(t *testing.T) { }) in := downloadInput(outPath) in.Force = true - c2 := AuditLogsCmd{auditLogs: fake2} + c2 := auditLogsDownloadTestCmd(fake2) require.NoError(t, c2.Download(context.Background(), in)) assert.Equal(t, []string{""}, *cursors, "--force must restart from the beginning") @@ -293,7 +307,7 @@ func TestAuditLogsDownloadRefusesExistingOutputWithoutState(t *testing.T) { outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") require.NoError(t, os.WriteFile(outPath, []byte("existing"), 0o644)) - c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + c := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) err := c.Download(context.Background(), downloadInput(outPath)) require.ErrorContains(t, err, "already exists") } @@ -307,15 +321,24 @@ func TestAuditLogsDownloadFinishesCompletedLeftoverState(t *testing.T) { return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: false}), nil }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.NoError(t, c.Download(context.Background(), downloadInput(outPath))) // Simulate a crash after the final state save but before cleanup by // re-creating the completed state file. statePath := outPath + ".state.json" - require.NoError(t, os.WriteFile(statePath, []byte(`{"version":1,"params":"`+mustFingerprint(t)+`","cursor":"","bytes_written":`+strconv.Itoa(len(chunk1))+`,"chunks":1,"rows":1}`), 0o644)) - - c2 := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + sum := sha256.Sum256(chunk1) + require.NoError(t, saveAuditLogsDownloadState(statePath, auditLogsDownloadState{ + Version: auditLogsDownloadStateVersion, + Params: mustFingerprint(t), + Identity: auditLogsDownloadTestIdentity, + BytesWritten: int64(len(chunk1)), + CommittedSHA256: hex.EncodeToString(sum[:]), + Chunks: 1, + Rows: 1, + })) + + c2 := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) require.NoError(t, c2.Download(context.Background(), downloadInput(outPath))) assert.Contains(t, buf.String(), "already complete") _, err := os.Stat(statePath) @@ -334,7 +357,7 @@ func mustFingerprint(t *testing.T) string { func TestAuditLogsDownloadRangeOver30DaysSuggestsWindows(t *testing.T) { buf := capturePtermOutput(t) - c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + c := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) err := c.Download(context.Background(), AuditLogsDownloadInput{Start: "2026-04-01", End: "2026-06-30"}) require.ErrorContains(t, err, "at most 30 days") assert.Contains(t, buf.String(), "--start 2026-06-01T00:00:00Z --end 2026-07-01T00:00:00Z") @@ -352,7 +375,7 @@ func TestAuditLogsDownloadServerCursorBugs(t *testing.T) { return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true}), nil }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) err := c.Download(context.Background(), downloadInput(outPath)) require.ErrorContains(t, err, "no cursor") }) @@ -363,7 +386,7 @@ func TestAuditLogsDownloadServerCursorBugs(t *testing.T) { return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil } fake, _ := chunkServer(t, []func() (*http.Response, error){repeat, repeat}) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) err := c.Download(context.Background(), downloadInput(outPath)) require.ErrorContains(t, err, "unchanged cursor") }) @@ -381,7 +404,7 @@ func TestAuditLogsDownloadJSONLFormat(t *testing.T) { } in := downloadInput(outPath) in.Format = "jsonl" - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.NoError(t, c.Download(context.Background(), in)) assert.Equal(t, kernel.AuditLogExportChunkParamsFormatJSONL, gotFormat) @@ -440,7 +463,7 @@ func TestAuditLogsDownloadRejectsDifferentCredentials(t *testing.T) { c2 := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}, identity: "org:org-b"} err := c2.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "different credentials") + require.ErrorContains(t, err, "different API origin or credential") } func TestAuditLogsDownloadRejectsMissingOrShortenedOutput(t *testing.T) { @@ -457,16 +480,16 @@ func TestAuditLogsDownloadRejectsMissingOrShortenedOutput(t *testing.T) { func() (*http.Response, error) { return nil, errors.New("boom") }, func() (*http.Response, error) { return nil, errors.New("boom") }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.Error(t, c.Download(context.Background(), downloadInput(outPath))) - return outPath, AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + return outPath, auditLogsDownloadTestCmd(&FakeAuditLogsService{}) } t.Run("missing output", func(t *testing.T) { outPath, c := interrupted(t) require.NoError(t, os.Remove(outPath)) err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "missing or unreadable") + require.ErrorContains(t, err, "is missing") }) t.Run("shortened output", func(t *testing.T) { @@ -485,7 +508,7 @@ func TestAuditLogsDownloadFirstChunkFailureRetriesWithoutForce(t *testing.T) { func() (*http.Response, error) { return nil, errors.New("boom") }, func() (*http.Response, error) { return nil, errors.New("boom") }, }) - c := AuditLogsCmd{auditLogs: failing} + c := auditLogsDownloadTestCmd(failing) require.Error(t, c.Download(context.Background(), downloadInput(outPath))) chunk := gzipMember(t, "{\"n\":1}\n") @@ -494,7 +517,7 @@ func TestAuditLogsDownloadFirstChunkFailureRetriesWithoutForce(t *testing.T) { return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: false}), nil }, }) - c2 := AuditLogsCmd{auditLogs: fake} + c2 := auditLogsDownloadTestCmd(fake) require.NoError(t, c2.Download(context.Background(), downloadInput(outPath)), "a failed first chunk must not require --force to retry") assert.Equal(t, []string{""}, *cursors) @@ -515,7 +538,7 @@ func TestAuditLogsDownloadFailedForceLeavesNoStaleProgress(t *testing.T) { func() (*http.Response, error) { return nil, errors.New("boom") }, func() (*http.Response, error) { return nil, errors.New("boom") }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.Error(t, c.Download(context.Background(), downloadInput(outPath))) // --force run that dies before its first chunk. @@ -526,7 +549,7 @@ func TestAuditLogsDownloadFailedForceLeavesNoStaleProgress(t *testing.T) { }) in := downloadInput(outPath) in.Force = true - c2 := AuditLogsCmd{auditLogs: failing} + c2 := auditLogsDownloadTestCmd(failing) require.Error(t, c2.Download(context.Background(), in)) // The plain rerun must start from scratch, not resume chunk 1's stale cursor. @@ -536,7 +559,7 @@ func TestAuditLogsDownloadFailedForceLeavesNoStaleProgress(t *testing.T) { return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: false}), nil }, }) - c3 := AuditLogsCmd{auditLogs: fake3} + c3 := auditLogsDownloadTestCmd(fake3) require.NoError(t, c3.Download(context.Background(), downloadInput(outPath))) assert.Equal(t, []string{""}, *cursors) @@ -556,7 +579,7 @@ func TestAuditLogsDownloadFailsClosedOnMissingHeaders(t *testing.T) { return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, omitHasMore: true}), nil }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) err := c.Download(context.Background(), downloadInput(outPath)) require.ErrorContains(t, err, "X-Has-More") data, readErr := os.ReadFile(outPath) @@ -571,7 +594,7 @@ func TestAuditLogsDownloadFailsClosedOnMissingHeaders(t *testing.T) { return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, omitSha: true}), nil }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) err := c.Download(context.Background(), downloadInput(outPath)) require.ErrorContains(t, err, "X-Content-Sha256") }) @@ -589,7 +612,7 @@ func TestAuditLogsDownloadRejectsOversizedChunk(t *testing.T) { return exportChunkResponse(exportChunk{body: []byte("well over eight bytes"), rowCount: 1}), nil }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) err := c.Download(context.Background(), downloadInput(outPath)) require.ErrorContains(t, err, "refusing to buffer") } @@ -606,7 +629,7 @@ func TestAuditLogsDownloadFilePermissions(t *testing.T) { func() (*http.Response, error) { return nil, errors.New("boom") }, func() (*http.Response, error) { return nil, errors.New("boom") }, }) - c := AuditLogsCmd{auditLogs: fake} + c := auditLogsDownloadTestCmd(fake) require.Error(t, c.Download(context.Background(), downloadInput(outPath))) outInfo, err := os.Stat(outPath) @@ -617,8 +640,170 @@ func TestAuditLogsDownloadFilePermissions(t *testing.T) { assert.Equal(t, os.FileMode(0o600), stateInfo.Mode().Perm()) } -func TestAuditLogsDownloadRejectsInvalidFormat(t *testing.T) { +func TestAuditLogsDownloadRequiresStableTimeBounds(t *testing.T) { + c := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) + for _, in := range []AuditLogsDownloadInput{ + {End: "2026-06-28"}, + {Start: "2026-06-01"}, + {}, + } { + err := c.Download(context.Background(), in) + require.ErrorContains(t, err, "--start and --end are required") + } +} + +func TestAuditLogsCredentialIdentityIncludesAPIOrigin(t *testing.T) { + t.Setenv("KERNEL_API_KEY", "test-key") + t.Setenv("KERNEL_BASE_URL", "HTTPS://API.EXAMPLE.COM/") + first, err := auditLogsCredentialIdentity() + require.NoError(t, err) + assert.Contains(t, first, "https://api.example.com|key:") + + t.Setenv("KERNEL_BASE_URL", "https://api.other.example.com") + second, err := auditLogsCredentialIdentity() + require.NoError(t, err) + assert.NotEqual(t, first, second) +} + +func TestAuditLogsDownloadRejectsEmptyIdentity(t *testing.T) { c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} + err := c.Download(context.Background(), downloadInput(filepath.Join(t.TempDir(), "out.jsonl.gz"))) + require.ErrorContains(t, err, "without an API origin and credential identity") +} + +func TestAuditLogsDownloadRejectsModifiedCommittedOutput(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + chunk := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + }, + func() (*http.Response, error) { return nil, errors.New("boom") }, + }) + require.Error(t, auditLogsDownloadTestCmd(fake).Download(context.Background(), downloadInput(outPath))) + + require.NoError(t, os.WriteFile(outPath, bytes.Repeat([]byte{'x'}, len(chunk)), 0o600)) + called := false + resume := &FakeAuditLogsService{ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { + called = true + return nil, errors.New("should not fetch") + }} + err := auditLogsDownloadTestCmd(resume).Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "does not match the recorded checksum") + assert.False(t, called) +} + +func TestAuditLogsDownloadOutputLock(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + first, _, err := openAndLockAuditLogsDownloadOutput(outPath) + require.NoError(t, err) + + _, _, err = openAndLockAuditLogsDownloadOutput(outPath) + require.ErrorContains(t, err, "already using") + require.NoError(t, first.Close()) + + third, _, err := openAndLockAuditLogsDownloadOutput(outPath) + require.NoError(t, err) + require.NoError(t, third.Close()) +} + +func TestAuditLogsDownloadRejectsSymlinkOutput(t *testing.T) { + capturePtermOutput(t) + dir := t.TempDir() + target := filepath.Join(dir, "target") + outPath := filepath.Join(dir, "out.jsonl.gz") + require.NoError(t, os.WriteFile(target, []byte("keep"), 0o600)) + if err := os.Symlink(target, outPath); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + + in := downloadInput(outPath) + in.Force = true + err := auditLogsDownloadTestCmd(&FakeAuditLogsService{}).Download(context.Background(), in) + require.ErrorContains(t, err, "must be a regular file") + data, readErr := os.ReadFile(target) + require.NoError(t, readErr) + assert.Equal(t, "keep", string(data)) +} + +func TestAuditLogsDownloadStateReadIsBoundedAndForceBypassesIt(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "out.state.json") + require.NoError(t, os.WriteFile(statePath, make([]byte, auditLogsDownloadMaxStateBytes+1), 0o600)) + + _, _, err := loadAuditLogsDownloadState(statePath, "params", auditLogsDownloadTestIdentity, false) + require.ErrorContains(t, err, "is too large") + state, exists, err := loadAuditLogsDownloadState(statePath, "params", auditLogsDownloadTestIdentity, true) + require.NoError(t, err) + assert.False(t, exists) + assert.Equal(t, auditLogsDownloadEmptySHA256, state.CommittedSHA256) + _, err = os.Lstat(statePath) + assert.True(t, os.IsNotExist(err)) +} + +func TestAuditLogsDownloadRejectsInconsistentState(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "out.state.json") + require.NoError(t, saveAuditLogsDownloadState(statePath, auditLogsDownloadState{ + Version: auditLogsDownloadStateVersion, + Params: "params", + Identity: auditLogsDownloadTestIdentity, + BytesWritten: 1, + CommittedSHA256: auditLogsDownloadEmptySHA256, + })) + + _, _, err := loadAuditLogsDownloadState(statePath, "params", auditLogsDownloadTestIdentity, false) + require.ErrorContains(t, err, "zero-chunk state contains committed progress") +} + +func TestAuditLogsDownloadForceSecuresExistingOutput(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + require.NoError(t, os.WriteFile(outPath, []byte("old"), 0o644)) + require.NoError(t, os.Chmod(outPath, 0o644)) + chunk := gzipMember(t, "{\"n\":1}\n") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1}), nil + }, + }) + in := downloadInput(outPath) + in.Force = true + require.NoError(t, auditLogsDownloadTestCmd(fake).Download(context.Background(), in)) + + info, err := os.Stat(outPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) +} + +func TestAuditLogsDownloadRejectsInvalidRowCount(t *testing.T) { + capturePtermOutput(t) + chunk := gzipMember(t, "{\"n\":1}\n") + tests := []struct { + name string + chunk exportChunk + }{ + {name: "missing", chunk: exportChunk{body: chunk, omitRowCount: true}}, + {name: "malformed", chunk: exportChunk{body: chunk, rowCountRaw: "nope"}}, + {name: "negative", chunk: exportChunk{body: chunk, rowCountRaw: "-1"}}, + {name: "above server limit", chunk: exportChunk{body: chunk, rowCountRaw: "50001"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") + fake, _ := chunkServer(t, []func() (*http.Response, error){ + func() (*http.Response, error) { return exportChunkResponse(tc.chunk), nil }, + }) + err := auditLogsDownloadTestCmd(fake).Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "X-Row-Count") + data, readErr := os.ReadFile(outPath) + require.NoError(t, readErr) + assert.Empty(t, data) + }) + } +} + +func TestAuditLogsDownloadRejectsInvalidFormat(t *testing.T) { + c := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) in := downloadInput("out.csv") in.Format = "csv" err := c.Download(context.Background(), in) diff --git a/cmd/audit_logs_filelock_other.go b/cmd/audit_logs_filelock_other.go new file mode 100644 index 00000000..b4607e21 --- /dev/null +++ b/cmd/audit_logs_filelock_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux && !windows + +package cmd + +import ( + "fmt" + "os" +) + +func tryLockAuditLogsDownloadFile(file *os.File) (bool, error) { + return false, fmt.Errorf("audit-log download locking is unsupported on this platform") +} + +func unlockAuditLogsDownloadFile(file *os.File) error { + return nil +} diff --git a/cmd/audit_logs_filelock_unix.go b/cmd/audit_logs_filelock_unix.go new file mode 100644 index 00000000..e843c11c --- /dev/null +++ b/cmd/audit_logs_filelock_unix.go @@ -0,0 +1,25 @@ +//go:build darwin || linux + +package cmd + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func tryLockAuditLogsDownloadFile(file *os.File) (bool, error) { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func unlockAuditLogsDownloadFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_UN) +} diff --git a/cmd/audit_logs_filelock_windows.go b/cmd/audit_logs_filelock_windows.go new file mode 100644 index 00000000..401457fa --- /dev/null +++ b/cmd/audit_logs_filelock_windows.go @@ -0,0 +1,32 @@ +//go:build windows + +package cmd + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func tryLockAuditLogsDownloadFile(file *os.File) (bool, error) { + err := windows.LockFileEx( + windows.Handle(file.Fd()), + windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, + 0, + 1, + 0, + &windows.Overlapped{}, + ) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) || errors.Is(err, windows.ERROR_IO_PENDING) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func unlockAuditLogsDownloadFile(file *os.File) error { + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, &windows.Overlapped{}) +} diff --git a/go.mod b/go.mod index b3939d33..6d657039 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( golang.org/x/crypto v0.47.0 golang.org/x/oauth2 v0.30.0 golang.org/x/sync v0.19.0 + golang.org/x/sys v0.40.0 golang.org/x/term v0.39.0 ) @@ -56,7 +57,6 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) From 4add5a2cab88a799e732e1c8084e6ae396dc1129 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:24:05 +0000 Subject: [PATCH 05/17] Harden audit log download bounds and recovery --- cmd/audit_logs_download.go | 208 ++++++++++++++++++++--------- cmd/audit_logs_download_test.go | 58 +++++++- cmd/audit_logs_filelock_other.go | 6 +- cmd/audit_logs_filelock_unix.go | 16 ++- cmd/audit_logs_filelock_windows.go | 16 ++- 5 files changed, 237 insertions(+), 67 deletions(-) diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 0448bc3d..92571824 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "hash" "io" @@ -114,10 +115,7 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e // Keep a newly created output only after it has a valid matching state. removeCreatedOutput := created defer func() { - _ = out.Close() - if removeCreatedOutput { - _ = os.Remove(outPath) - } + _ = out.close(removeCreatedOutput) }() state, stateExists, err := loadAuditLogsDownloadState(statePath, fingerprint, c.identity, in.Force) @@ -146,6 +144,9 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e // A state file with chunks but no cursor means the download finished and // only the cleanup was interrupted. if completed { + if err := validateAuditLogsDownloadOutputPath(out.file, outPath); err != nil { + return err + } if err := removeAuditLogsDownloadState(statePath); err != nil { return err } @@ -204,12 +205,18 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e return fmt.Errorf("chunk count exceeds the supported range") } + if err := validateAuditLogsDownloadOutputPath(out.file, outPath); err != nil { + return err + } if _, err := out.file.Write(body); err != nil { return fmt.Errorf("write %s: %w", outPath, err) } if err := out.file.Sync(); err != nil { return fmt.Errorf("sync %s: %w", outPath, err) } + if err := validateAuditLogsDownloadOutputPath(out.file, outPath); err != nil { + return err + } _, _ = committedHash.Write(body) state.Chunks++ @@ -227,6 +234,9 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e } } + if err := validateAuditLogsDownloadOutputPath(out.file, outPath); err != nil { + return err + } if err := removeAuditLogsDownloadState(statePath); err != nil { return err } @@ -278,13 +288,10 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp if err != nil { return params, fmt.Errorf("--start: %w", err) } - end, dateOnly, err := parseAuditLogTime(in.End) + end, _, err := parseAuditLogTime(in.End) if err != nil { return params, fmt.Errorf("--end: %w", err) } - if dateOnly { - end = end.Add(24 * time.Hour) - } if !start.Before(end) { return params, fmt.Errorf("--start must be before --end") } @@ -502,7 +509,7 @@ func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) os.Remove(tmp.Name()) return fmt.Errorf("write state file: %w", err) } - if err := os.Rename(tmp.Name(), statePath); err != nil { + if err := commitAuditLogsDownloadStateFile(tmp.Name(), statePath); err != nil { os.Remove(tmp.Name()) return fmt.Errorf("commit state file: %w", err) } @@ -510,20 +517,30 @@ func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) } type lockedAuditLogsDownloadOutput struct { - file *os.File + file *os.File + pathLock *os.File + outPath string } func (o *lockedAuditLogsDownloadOutput) Close() error { - unlockErr := unlockAuditLogsDownloadFile(o.file) - closeErr := o.file.Close() - if unlockErr != nil { - return unlockErr + return o.close(false) +} + +func (o *lockedAuditLogsDownloadOutput) close(removeOutput bool) error { + outputErr := o.file.Close() + var removeErr error + if removeOutput { + removeErr = os.Remove(o.outPath) + if removeErr == nil { + removeErr = syncAuditLogsDownloadDir(filepath.Dir(o.outPath)) + } } - return closeErr + lockErr := o.pathLock.Close() + return errors.Join(outputErr, removeErr, lockErr) } // openAndLockAuditLogsDownloadOutput securely opens a regular output file and -// locks it before any state is read or output is modified. +// holds a stable path lock before any state is read or output is modified. func openAndLockAuditLogsDownloadOutput(outPath string) (*lockedAuditLogsDownloadOutput, bool, error) { if dir := filepath.Dir(outPath); dir != "." { if err := os.MkdirAll(dir, 0o700); err != nil { @@ -531,60 +548,131 @@ func openAndLockAuditLogsDownloadOutput(outPath string) (*lockedAuditLogsDownloa } } + pathLock, err := openAndLockAuditLogsDownloadPath(outPath + ".lock") + if err != nil { + return nil, false, err + } + closePathLock := true + defer func() { + if closePathLock { + _ = pathLock.Close() + } + }() + + out, created, err := openAuditLogsDownloadRegularFile(outPath) + if err != nil { + return nil, false, err + } + locked, err := tryLockAuditLogsDownloadFile(out) + if err != nil { + out.Close() + return nil, false, fmt.Errorf("lock %s: %w", outPath, err) + } + if !locked { + out.Close() + return nil, false, fmt.Errorf("another audit-log download is already using %s", outPath) + } + if created { + if err := out.Sync(); err != nil { + out.Close() + _ = os.Remove(outPath) + return nil, false, fmt.Errorf("sync %s: %w", outPath, err) + } + if err := syncAuditLogsDownloadDir(filepath.Dir(outPath)); err != nil { + out.Close() + _ = os.Remove(outPath) + return nil, false, fmt.Errorf("sync output directory: %w", err) + } + } + closePathLock = false + return &lockedAuditLogsDownloadOutput{file: out, pathLock: pathLock, outPath: outPath}, created, nil +} + +func openAndLockAuditLogsDownloadPath(lockPath string) (*os.File, error) { + lock, _, err := openAuditLogsDownloadRegularFile(lockPath) + if err != nil { + return nil, err + } + if err := lock.Chmod(0o600); err != nil { + lock.Close() + return nil, fmt.Errorf("secure lock file permissions: %w", err) + } + locked, err := tryLockAuditLogsDownloadFile(lock) + if err != nil { + lock.Close() + return nil, fmt.Errorf("lock %s: %w", lockPath, err) + } + if !locked { + lock.Close() + return nil, fmt.Errorf("another audit-log download is already using %s", strings.TrimSuffix(lockPath, ".lock")) + } + currentInfo, err := os.Lstat(lockPath) + if err != nil { + lock.Close() + return nil, fmt.Errorf("inspect lock file: %w", err) + } + openedInfo, err := lock.Stat() + if err != nil { + lock.Close() + return nil, fmt.Errorf("inspect lock file: %w", err) + } + if !currentInfo.Mode().IsRegular() || !os.SameFile(currentInfo, openedInfo) { + lock.Close() + return nil, fmt.Errorf("lock file %s changed while it was being locked; retry", lockPath) + } + return lock, nil +} + +func openAuditLogsDownloadRegularFile(path string) (*os.File, bool, error) { for attempt := 0; attempt < 2; attempt++ { - info, err := os.Lstat(outPath) - created := false - var out *os.File - switch { - case os.IsNotExist(err): - out, err = os.OpenFile(outPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) - if os.IsExist(err) { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + f, createErr := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if os.IsExist(createErr) { continue } - created = err == nil - case err != nil: - return nil, false, fmt.Errorf("inspect %s: %w", outPath, err) - case !info.Mode().IsRegular(): - return nil, false, fmt.Errorf("%s must be a regular file; refusing to follow a symlink or special file", outPath) - default: - out, err = os.OpenFile(outPath, os.O_RDWR, 0) - if err == nil { - openedInfo, statErr := out.Stat() - if statErr != nil { - out.Close() - return nil, false, fmt.Errorf("inspect %s: %w", outPath, statErr) - } - if !openedInfo.Mode().IsRegular() || !os.SameFile(info, openedInfo) { - out.Close() - return nil, false, fmt.Errorf("%s changed while it was being opened; retry", outPath) - } + if createErr != nil { + return nil, false, fmt.Errorf("open %s: %w", path, createErr) } + return f, true, nil } if err != nil { - return nil, false, fmt.Errorf("open %s: %w", outPath, err) + return nil, false, fmt.Errorf("inspect %s: %w", path, err) } - locked, err := tryLockAuditLogsDownloadFile(out) - // A second process can open and briefly lock a just-created file before - // its creator reaches the lock call. The creator retries long enough for - // that process to observe the missing state and release its lock. - for retries := 0; created && !locked && err == nil && retries < 25; retries++ { - time.Sleep(10 * time.Millisecond) - locked, err = tryLockAuditLogsDownloadFile(out) + if !info.Mode().IsRegular() { + return nil, false, fmt.Errorf("%s must be a regular file; refusing to follow a symlink or special file", path) } + f, err := os.OpenFile(path, os.O_RDWR, 0) if err != nil { - out.Close() - if created { - _ = os.Remove(outPath) - } - return nil, false, fmt.Errorf("lock %s: %w", outPath, err) + return nil, false, fmt.Errorf("open %s: %w", path, err) } - if !locked { - out.Close() - return nil, false, fmt.Errorf("another audit-log download is already using %s", outPath) + openedInfo, err := f.Stat() + if err != nil { + f.Close() + return nil, false, fmt.Errorf("inspect %s: %w", path, err) } - return &lockedAuditLogsDownloadOutput{file: out}, created, nil + if !openedInfo.Mode().IsRegular() || !os.SameFile(info, openedInfo) { + f.Close() + return nil, false, fmt.Errorf("%s changed while it was being opened; retry", path) + } + return f, false, nil } - return nil, false, fmt.Errorf("%s changed while it was being opened; retry", outPath) + return nil, false, fmt.Errorf("%s changed while it was being opened; retry", path) +} + +func validateAuditLogsDownloadOutputPath(out *os.File, outPath string) error { + pathInfo, err := os.Lstat(outPath) + if err != nil { + return fmt.Errorf("%s changed while the download was running: %w", outPath, err) + } + openedInfo, err := out.Stat() + if err != nil { + return fmt.Errorf("inspect %s: %w", outPath, err) + } + if !pathInfo.Mode().IsRegular() || !openedInfo.Mode().IsRegular() || !os.SameFile(pathInfo, openedInfo) { + return fmt.Errorf("%s changed while the download was running; restore it before resuming", outPath) + } + return nil } // prepareAuditLogsDownloadOutput verifies the committed prefix before @@ -707,7 +795,7 @@ var auditLogsDownloadCmd = &cobra.Command{ Use: "download", Short: "Download audit logs for a time range to a file", Long: "Download an organization's audit log records for a time range as a file, for archival, compliance, or offline analysis.\n\n" + - "Records are fetched in chunks and appended after each chunk is verified, newest first. If a download is interrupted, rerunning the same command resumes where it left off.\n\n" + + "Records in the half-open [start, end) window are fetched in chunks and appended after each chunk is verified, newest first. If a download is interrupted, rerunning the same command resumes where it left off.\n\n" + "GET requests are excluded by default; pass --include-get to include them, or --method GET to see only them.\n\n" + "The API allows at most 30 days per download; for longer ranges run one download per window.", Args: cobra.NoArgs, @@ -716,7 +804,7 @@ var auditLogsDownloadCmd = &cobra.Command{ func init() { auditLogsDownloadCmd.Flags().String("start", "", "Start of the export window, RFC3339 or YYYY-MM-DD (required)") - auditLogsDownloadCmd.Flags().String("end", "", "End of the export window, RFC3339 or YYYY-MM-DD inclusive (required)") + auditLogsDownloadCmd.Flags().String("end", "", "Exclusive end of the export window, RFC3339 or YYYY-MM-DD (required)") auditLogsDownloadCmd.Flags().String("search", "", "Free-text search") auditLogsDownloadCmd.Flags().String("method", "", "Filter by HTTP method (e.g. GET)") auditLogsDownloadCmd.Flags().String("exclude-method", "", "Exclude an HTTP method") diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index 14dd7625..64757ab9 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -11,8 +11,10 @@ import ( "net/http" "os" "path/filepath" + "runtime" "strconv" "testing" + "time" "github.com/kernel/kernel-go-sdk" "github.com/kernel/kernel-go-sdk/option" @@ -360,8 +362,16 @@ func TestAuditLogsDownloadRangeOver30DaysSuggestsWindows(t *testing.T) { c := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) err := c.Download(context.Background(), AuditLogsDownloadInput{Start: "2026-04-01", End: "2026-06-30"}) require.ErrorContains(t, err, "at most 30 days") - assert.Contains(t, buf.String(), "--start 2026-06-01T00:00:00Z --end 2026-07-01T00:00:00Z") - assert.Contains(t, buf.String(), "--start 2026-04-01T00:00:00Z --end 2026-04-02T00:00:00Z") + assert.Contains(t, buf.String(), "--start 2026-05-31T00:00:00Z --end 2026-06-30T00:00:00Z") + assert.Contains(t, buf.String(), "--start 2026-04-01T00:00:00Z --end 2026-05-01T00:00:00Z") +} + +func TestAuditLogsDownloadDateEndIsExclusive(t *testing.T) { + in := downloadInput("") + params, err := buildAuditLogsDownloadParams(in) + require.NoError(t, err) + + assert.Equal(t, time.Date(2026, 6, 28, 0, 0, 0, 0, time.UTC), params.End) } func TestAuditLogsDownloadServerCursorBugs(t *testing.T) { @@ -638,6 +648,9 @@ func TestAuditLogsDownloadFilePermissions(t *testing.T) { stateInfo, err := os.Stat(outPath + ".state.json") require.NoError(t, err) assert.Equal(t, os.FileMode(0o600), stateInfo.Mode().Perm()) + lockInfo, err := os.Stat(outPath + ".lock") + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), lockInfo.Mode().Perm()) } func TestAuditLogsDownloadRequiresStableTimeBounds(t *testing.T) { @@ -708,6 +721,47 @@ func TestAuditLogsDownloadOutputLock(t *testing.T) { require.NoError(t, third.Close()) } +func TestAuditLogsDownloadPathLockSurvivesOutputRename(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows prevents renaming an open output file") + } + dir := t.TempDir() + outPath := filepath.Join(dir, "out.jsonl.gz") + movedPath := filepath.Join(dir, "moved.jsonl.gz") + first, _, err := openAndLockAuditLogsDownloadOutput(outPath) + require.NoError(t, err) + require.NoError(t, os.Rename(outPath, movedPath)) + + _, _, err = openAndLockAuditLogsDownloadOutput(outPath) + require.ErrorContains(t, err, "already using") + require.NoError(t, first.Close()) + + third, created, err := openAndLockAuditLogsDownloadOutput(outPath) + require.NoError(t, err) + assert.True(t, created) + require.NoError(t, third.Close()) +} + +func TestAuditLogsDownloadDetectsOutputRename(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows prevents renaming an open output file") + } + capturePtermOutput(t) + dir := t.TempDir() + outPath := filepath.Join(dir, "out.jsonl.gz") + movedPath := filepath.Join(dir, "moved.jsonl.gz") + chunk := gzipMember(t, "{\"n\":1}\n") + fake := &FakeAuditLogsService{ + ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { + require.NoError(t, os.Rename(outPath, movedPath)) + return exportChunkResponse(exportChunk{body: chunk, rowCount: 1}), nil + }, + } + + err := auditLogsDownloadTestCmd(fake).Download(context.Background(), downloadInput(outPath)) + require.ErrorContains(t, err, "changed while the download was running") +} + func TestAuditLogsDownloadRejectsSymlinkOutput(t *testing.T) { capturePtermOutput(t) dir := t.TempDir() diff --git a/cmd/audit_logs_filelock_other.go b/cmd/audit_logs_filelock_other.go index b4607e21..84da28c5 100644 --- a/cmd/audit_logs_filelock_other.go +++ b/cmd/audit_logs_filelock_other.go @@ -11,6 +11,10 @@ func tryLockAuditLogsDownloadFile(file *os.File) (bool, error) { return false, fmt.Errorf("audit-log download locking is unsupported on this platform") } -func unlockAuditLogsDownloadFile(file *os.File) error { +func commitAuditLogsDownloadStateFile(oldPath, newPath string) error { + return os.Rename(oldPath, newPath) +} + +func syncAuditLogsDownloadDir(dir string) error { return nil } diff --git a/cmd/audit_logs_filelock_unix.go b/cmd/audit_logs_filelock_unix.go index e843c11c..0627210f 100644 --- a/cmd/audit_logs_filelock_unix.go +++ b/cmd/audit_logs_filelock_unix.go @@ -5,6 +5,7 @@ package cmd import ( "errors" "os" + "path/filepath" "golang.org/x/sys/unix" ) @@ -20,6 +21,17 @@ func tryLockAuditLogsDownloadFile(file *os.File) (bool, error) { return true, nil } -func unlockAuditLogsDownloadFile(file *os.File) error { - return unix.Flock(int(file.Fd()), unix.LOCK_UN) +func commitAuditLogsDownloadStateFile(oldPath, newPath string) error { + if err := os.Rename(oldPath, newPath); err != nil { + return err + } + return syncAuditLogsDownloadDir(filepath.Dir(newPath)) +} + +func syncAuditLogsDownloadDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + return errors.Join(f.Sync(), f.Close()) } diff --git a/cmd/audit_logs_filelock_windows.go b/cmd/audit_logs_filelock_windows.go index 401457fa..980ff771 100644 --- a/cmd/audit_logs_filelock_windows.go +++ b/cmd/audit_logs_filelock_windows.go @@ -27,6 +27,18 @@ func tryLockAuditLogsDownloadFile(file *os.File) (bool, error) { return true, nil } -func unlockAuditLogsDownloadFile(file *os.File) error { - return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, &windows.Overlapped{}) +func commitAuditLogsDownloadStateFile(oldPath, newPath string) error { + oldPathPtr, err := windows.UTF16PtrFromString(oldPath) + if err != nil { + return err + } + newPathPtr, err := windows.UTF16PtrFromString(newPath) + if err != nil { + return err + } + return windows.MoveFileEx(oldPathPtr, newPathPtr, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) +} + +func syncAuditLogsDownloadDir(dir string) error { + return nil } From 24d3a584957da04f8fc063aa1381aeb04b6b1bbe Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:42:42 +0000 Subject: [PATCH 06/17] Simplify audit log chunk downloads --- cmd/audit_logs.go | 2 - cmd/audit_logs_download.go | 659 ++++----------------- cmd/audit_logs_download_test.go | 911 +++++------------------------ cmd/audit_logs_filelock_other.go | 20 - cmd/audit_logs_filelock_unix.go | 37 -- cmd/audit_logs_filelock_windows.go | 44 -- go.mod | 2 +- 7 files changed, 270 insertions(+), 1405 deletions(-) delete mode 100644 cmd/audit_logs_filelock_other.go delete mode 100644 cmd/audit_logs_filelock_unix.go delete mode 100644 cmd/audit_logs_filelock_windows.go diff --git a/cmd/audit_logs.go b/cmd/audit_logs.go index b6ae9eef..da60f527 100644 --- a/cmd/audit_logs.go +++ b/cmd/audit_logs.go @@ -23,8 +23,6 @@ type AuditLogsService interface { type AuditLogsCmd struct { auditLogs AuditLogsService - // identity binds download resume state to an API origin and credential. - identity string } type AuditLogsSearchInput struct { diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 92571824..8e90fabc 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -5,20 +5,15 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "errors" "fmt" - "hash" "io" - "math" "net/http" - "net/url" "os" "path/filepath" "strconv" "strings" "time" - "github.com/kernel/cli/pkg/auth" "github.com/kernel/cli/pkg/util" "github.com/kernel/kernel-go-sdk" "github.com/pterm/pterm" @@ -41,47 +36,21 @@ type AuditLogsDownloadInput struct { } const ( - // auditLogsDownloadMaxRange mirrors the API's max export window. - auditLogsDownloadMaxRange = 30 * 24 * time.Hour - // auditLogsDownloadShaRetries is the number of extra attempts for a chunk - // whose body fails checksum verification. - auditLogsDownloadShaRetries = 2 - // auditLogsDownloadMaxRowsPerChunk mirrors the export endpoint's limit. - auditLogsDownloadMaxRowsPerChunk = 50_000 - // auditLogsDownloadMaxStateBytes bounds the local checkpoint before it is - // decoded. Valid state files are under a few kilobytes. - auditLogsDownloadMaxStateBytes = 64 << 10 + auditLogsDownloadMaxRange = 30 * 24 * time.Hour + auditLogsDownloadMaxChunkBytes = 256 << 20 + auditLogsDownloadStateVersion = 1 ) -// auditLogsDownloadMaxChunkBytes bounds how much of a chunk response is -// buffered. The server caps chunks at 50k rows, so a legitimate response is -// far smaller; this guards against a broken server streaming without end. -// Variable so tests can lower it. -var auditLogsDownloadMaxChunkBytes int64 = 256 << 20 - -// auditLogsDownloadState is the sidecar file that makes a download resumable. -// It is written atomically after every committed chunk and removed on -// completion. Version guards future format changes (e.g. multi-window splits). type auditLogsDownloadState struct { - Version int `json:"version"` - Params string `json:"params"` - Identity string `json:"identity"` - Cursor string `json:"cursor"` - BytesWritten int64 `json:"bytes_written"` - CommittedSHA256 string `json:"committed_sha256"` - Chunks int `json:"chunks"` - Rows int64 `json:"rows"` + Version int `json:"version"` + Params string `json:"params"` + Cursor string `json:"cursor"` + BytesWritten int64 `json:"bytes_written"` + Chunks int `json:"chunks"` + Rows int64 `json:"rows"` } -const auditLogsDownloadStateVersion = 2 - -const auditLogsDownloadEmptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) error { - if c.identity == "" { - return fmt.Errorf("cannot safely resume audit-log download without an API origin and credential identity") - } - params, err := buildAuditLogsDownloadParams(in) if err != nil { return err @@ -91,12 +60,10 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if format == "" { format = string(kernel.AuditLogExportChunkParamsFormatJSONLGz) } - switch format { - case string(kernel.AuditLogExportChunkParamsFormatJSONL), string(kernel.AuditLogExportChunkParamsFormatJSONLGz): - params.Format = kernel.AuditLogExportChunkParamsFormat(format) - default: + if format != string(kernel.AuditLogExportChunkParamsFormatJSONL) && format != string(kernel.AuditLogExportChunkParamsFormatJSONLGz) { return fmt.Errorf("--format must be jsonl or jsonl.gz") } + params.Format = kernel.AuditLogExportChunkParamsFormat(format) outPath := in.To if outPath == "" { @@ -107,53 +74,29 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if err != nil { return err } - - out, created, err := openAndLockAuditLogsDownloadOutput(outPath) + state, exists, err := loadAuditLogsDownloadState(statePath, outPath, fingerprint, in.Force) if err != nil { return err } - // Keep a newly created output only after it has a valid matching state. - removeCreatedOutput := created - defer func() { - _ = out.close(removeCreatedOutput) - }() - - state, stateExists, err := loadAuditLogsDownloadState(statePath, fingerprint, c.identity, in.Force) - if err != nil { - return err - } - if !stateExists && !in.Force && !created { - return fmt.Errorf("%s already exists; pass --force to overwrite or --to to pick another path", outPath) - } - if stateExists && state.Chunks > 0 && created { - return fmt.Errorf("state file records progress but %s is missing; pass --force to start over", outPath) - } - if !stateExists { + if !exists { if err := saveAuditLogsDownloadState(statePath, state); err != nil { return err } } - completed := state.Chunks > 0 && state.Cursor == "" - committedHash, err := prepareAuditLogsDownloadOutput(out.file, outPath, state, completed) + out, err := openAuditLogsDownloadOutput(outPath, state.BytesWritten) if err != nil { return err } - removeCreatedOutput = false + defer out.Close() - // A state file with chunks but no cursor means the download finished and - // only the cleanup was interrupted. - if completed { - if err := validateAuditLogsDownloadOutputPath(out.file, outPath); err != nil { - return err - } + if state.Chunks > 0 && state.Cursor == "" { if err := removeAuditLogsDownloadState(statePath); err != nil { return err } pterm.Success.Printf("Download already complete: %d rows (%s) in %s\n", state.Rows, util.FormatBytes(state.BytesWritten), outPath) return nil } - if state.Chunks > 0 { pterm.Info.Printf("Resuming download at chunk %d (%d rows so far)\n", state.Chunks+1, state.Rows) } @@ -164,79 +107,33 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e } body, header, err := c.fetchAuditLogsChunk(ctx, params) if err != nil { - if state.Chunks > 0 { - pterm.Info.Println("Progress saved; rerun the same command to resume") - } return err } - - // A missing pagination header must not end the download as a success: - // a stripped or renamed header would silently truncate the export. - // Validated before the write so a bad response never touches the file. - hasMore, err := strconv.ParseBool(header.Get("X-Has-More")) + rows, nextCursor, hasMore, err := parseAuditLogsChunkHeaders(header, state.Cursor) if err != nil { - return fmt.Errorf("response missing or invalid X-Has-More header %q", header.Get("X-Has-More")) - } - rowsHeader := header.Get("X-Row-Count") - rows, err := strconv.ParseInt(rowsHeader, 10, 64) - if err != nil || rows < 0 || rows > auditLogsDownloadMaxRowsPerChunk { - return fmt.Errorf("response missing or invalid X-Row-Count header %q", rowsHeader) - } - nextCursor := header.Get("X-Next-Cursor") - if hasMore { - // Guard against a server bug looping the download forever. - if nextCursor == "" { - return fmt.Errorf("server reported more records but returned no cursor; retry, and report this if it persists") - } - if nextCursor == state.Cursor { - return fmt.Errorf("server returned an unchanged cursor; retry, and report this if it persists") - } - } else if nextCursor != "" { - return fmt.Errorf("server returned a cursor after reporting no more records; retry, and report this if it persists") - } - if state.Rows > math.MaxInt64-rows { - return fmt.Errorf("row count exceeds the supported range") - } - bodyBytes := int64(len(body)) - if state.BytesWritten > math.MaxInt64-bodyBytes { - return fmt.Errorf("download size exceeds the supported range") - } - if state.Chunks == math.MaxInt { - return fmt.Errorf("chunk count exceeds the supported range") - } - - if err := validateAuditLogsDownloadOutputPath(out.file, outPath); err != nil { return err } - if _, err := out.file.Write(body); err != nil { + + if _, err := out.Write(body); err != nil { return fmt.Errorf("write %s: %w", outPath, err) } - if err := out.file.Sync(); err != nil { + if err := out.Sync(); err != nil { return fmt.Errorf("sync %s: %w", outPath, err) } - if err := validateAuditLogsDownloadOutputPath(out.file, outPath); err != nil { - return err - } - _, _ = committedHash.Write(body) + state.Cursor = nextCursor + state.BytesWritten += int64(len(body)) state.Chunks++ state.Rows += rows - state.BytesWritten += bodyBytes - state.CommittedSHA256 = hex.EncodeToString(committedHash.Sum(nil)) - state.Cursor = nextCursor if err := saveAuditLogsDownloadState(statePath, state); err != nil { return err } - pterm.Info.Printf("Chunk %d: %d rows (%d total, %s)\n", state.Chunks, rows, state.Rows, util.FormatBytes(state.BytesWritten)) if !hasMore { break } } - if err := validateAuditLogsDownloadOutputPath(out.file, outPath); err != nil { - return err - } if err := removeAuditLogsDownloadState(statePath); err != nil { return err } @@ -244,46 +141,55 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e return nil } -// fetchAuditLogsChunk downloads one chunk and verifies it against the -// X-Content-Sha256 header before returning it, retrying on mismatch. Nothing -// is written to disk until a chunk verifies; a response without a checksum is -// rejected rather than trusted. func (c AuditLogsCmd) fetchAuditLogsChunk(ctx context.Context, params kernel.AuditLogExportChunkParams) ([]byte, http.Header, error) { - var lastErr error - for attempt := 0; attempt <= auditLogsDownloadShaRetries; attempt++ { - res, err := c.auditLogs.ExportChunk(ctx, params) - if err != nil { - return nil, nil, util.CleanedUpSdkError{Err: err} - } - body, err := io.ReadAll(io.LimitReader(res.Body, auditLogsDownloadMaxChunkBytes+1)) - res.Body.Close() - if err != nil { - lastErr = fmt.Errorf("read chunk body: %w", err) - continue - } - if int64(len(body)) > auditLogsDownloadMaxChunkBytes { - return nil, nil, fmt.Errorf("chunk response exceeds %s; refusing to buffer it", util.FormatBytes(auditLogsDownloadMaxChunkBytes)) - } - want := res.Header.Get("X-Content-Sha256") - if want == "" { - return nil, nil, fmt.Errorf("response missing X-Content-Sha256 header; refusing to write unverified data") - } - sum := sha256.Sum256(body) - if got := hex.EncodeToString(sum[:]); got != want { - lastErr = fmt.Errorf("chunk checksum mismatch (got %s, want %s)", got, want) - continue - } - return body, res.Header, nil + res, err := c.auditLogs.ExportChunk(ctx, params) + if err != nil { + return nil, nil, util.CleanedUpSdkError{Err: err} + } + defer res.Body.Close() + + body, err := io.ReadAll(io.LimitReader(res.Body, auditLogsDownloadMaxChunkBytes+1)) + if err != nil { + return nil, nil, fmt.Errorf("read chunk body: %w", err) + } + if len(body) > auditLogsDownloadMaxChunkBytes { + return nil, nil, fmt.Errorf("chunk response exceeds %s", util.FormatBytes(auditLogsDownloadMaxChunkBytes)) + } + want := res.Header.Get("X-Content-Sha256") + if want == "" { + return nil, nil, fmt.Errorf("response missing X-Content-Sha256 header") + } + sum := sha256.Sum256(body) + if got := hex.EncodeToString(sum[:]); got != want { + return nil, nil, fmt.Errorf("chunk checksum mismatch (got %s, want %s)", got, want) + } + return body, res.Header, nil +} + +func parseAuditLogsChunkHeaders(header http.Header, currentCursor string) (int64, string, bool, error) { + hasMore, err := strconv.ParseBool(header.Get("X-Has-More")) + if err != nil { + return 0, "", false, fmt.Errorf("response missing or invalid X-Has-More header") + } + rows, err := strconv.ParseInt(header.Get("X-Row-Count"), 10, 64) + if err != nil || rows < 0 { + return 0, "", false, fmt.Errorf("response missing or invalid X-Row-Count header") + } + nextCursor := header.Get("X-Next-Cursor") + if hasMore && (nextCursor == "" || nextCursor == currentCursor) { + return 0, "", false, fmt.Errorf("response has invalid X-Next-Cursor header") } - return nil, nil, lastErr + if !hasMore && nextCursor != "" { + return 0, "", false, fmt.Errorf("response returned a cursor after the final chunk") + } + return rows, nextCursor, hasMore, nil } func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExportChunkParams, error) { var params kernel.AuditLogExportChunkParams if in.Start == "" || in.End == "" { - return params, fmt.Errorf("--start and --end are required so interrupted downloads can resume with the same time range") + return params, fmt.Errorf("--start and --end are required") } - start, _, err := parseAuditLogTime(in.Start) if err != nil { return params, fmt.Errorf("--start: %w", err) @@ -296,10 +202,7 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp return params, fmt.Errorf("--start must be before --end") } if end.Sub(start) > auditLogsDownloadMaxRange { - // The window list goes through pterm because the error renderer - // collapses newlines, which would mangle the copy-pasteable flags. - pterm.Info.Printf("Run one download per window:\n%s", suggestAuditLogsDownloadWindows(start, end)) - return params, fmt.Errorf("range is %d days; the API allows at most 30 days per download", int(end.Sub(start).Hours()/24)) + return params, fmt.Errorf("the API allows at most 30 days per download") } params.Start = start @@ -310,17 +213,12 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp if in.Method != "" { params.Method = kernel.String(in.Method) } - // GETs are excluded by default, like search. The API accepts a single - // exclude_method, and chunks are appended verbatim, so a second exclusion - // can't be filtered client-side the way search does; require --include-get - // so the default is never dropped silently. excludeMethod := in.ExcludeMethod if in.Method == "" && !in.IncludeGet { - if excludeMethod == "" { - excludeMethod = "GET" - } else if !strings.EqualFold(excludeMethod, "GET") { - return params, fmt.Errorf("--exclude-method %s would replace the default GET exclusion; add --include-get to confirm GET requests should be included", in.ExcludeMethod) + if excludeMethod != "" && !strings.EqualFold(excludeMethod, "GET") { + return params, fmt.Errorf("add --include-get when excluding a method other than GET") } + excludeMethod = "GET" } if excludeMethod != "" { params.ExcludeMethod = kernel.String(excludeMethod) @@ -337,30 +235,12 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp return params, nil } -// suggestAuditLogsDownloadWindows renders ready-to-run 30-day windows covering -// [start, end), newest first to match the export's record order. -func suggestAuditLogsDownloadWindows(start, end time.Time) string { - var out string - for windowEnd := end; windowEnd.After(start); { - windowStart := windowEnd.Add(-auditLogsDownloadMaxRange) - if windowStart.Before(start) { - windowStart = start - } - out += fmt.Sprintf(" --start %s --end %s\n", windowStart.Format(time.RFC3339), windowEnd.Format(time.RFC3339)) - windowEnd = windowStart - } - return out -} - -// auditLogsDownloadFingerprint identifies the query a state file belongs to, -// so a resume never mixes records from different queries in one output file. -// It must be computed before the cursor is set on params. func auditLogsDownloadFingerprint(params kernel.AuditLogExportChunkParams) (string, error) { - q, err := params.URLQuery() + query, err := params.URLQuery() if err != nil { return "", fmt.Errorf("fingerprint params: %w", err) } - return q.Encode(), nil + return query.Encode(), nil } func defaultAuditLogsDownloadPath(start, end time.Time, format string) string { @@ -368,400 +248,113 @@ func defaultAuditLogsDownloadPath(start, end time.Time, format string) string { return fmt.Sprintf("audit-logs-%s-%s.%s", start.UTC().Format(stamp), end.UTC().Format(stamp), format) } -// loadAuditLogsDownloadState decides whether a download starts fresh or resumes -// from a checkpoint bound to the same query, API origin, and credential. -func loadAuditLogsDownloadState(statePath, fingerprint, identity string, force bool) (auditLogsDownloadState, bool, error) { - fresh := auditLogsDownloadState{ - Version: auditLogsDownloadStateVersion, - Params: fingerprint, - Identity: identity, - CommittedSHA256: auditLogsDownloadEmptySHA256, - } +func loadAuditLogsDownloadState(statePath, outPath, fingerprint string, force bool) (auditLogsDownloadState, bool, error) { + fresh := auditLogsDownloadState{Version: auditLogsDownloadStateVersion, Params: fingerprint} if force { if err := removeAuditLogsDownloadState(statePath); err != nil { return fresh, false, err } return fresh, false, nil } - - raw, exists, err := readAuditLogsDownloadState(statePath) - if err != nil { - return fresh, false, err - } - if !exists { + raw, err := os.ReadFile(statePath) + if os.IsNotExist(err) { + if _, statErr := os.Stat(outPath); statErr == nil { + return fresh, false, fmt.Errorf("%s already exists; pass --force to overwrite", outPath) + } else if !os.IsNotExist(statErr) { + return fresh, false, fmt.Errorf("inspect %s: %w", outPath, statErr) + } return fresh, false, nil } - + if err != nil { + return fresh, false, fmt.Errorf("read state file: %w", err) + } var state auditLogsDownloadState if err := json.Unmarshal(raw, &state); err != nil { return fresh, false, fmt.Errorf("state file %s is corrupt; pass --force to start over", statePath) } - if state.Version != auditLogsDownloadStateVersion { - return fresh, false, fmt.Errorf("state file %s was written by an incompatible CLI version; pass --force to start over", statePath) - } - if state.Params != fingerprint { - return fresh, false, fmt.Errorf("state file %s belongs to a download with different parameters; pass --force to start over or --to to pick another path", statePath) - } - if state.Identity != identity { - return fresh, false, fmt.Errorf("state file %s was written for a different API origin or credential; pass --force to start over", statePath) - } - if err := validateAuditLogsDownloadState(state); err != nil { - return fresh, false, fmt.Errorf("state file %s is inconsistent (%v); pass --force to start over", statePath, err) + if state.Version != auditLogsDownloadStateVersion || state.Params != fingerprint || state.BytesWritten < 0 { + return fresh, false, fmt.Errorf("state file %s does not match this download; pass --force to start over", statePath) } return state, true, nil } -func readAuditLogsDownloadState(statePath string) ([]byte, bool, error) { - info, err := os.Lstat(statePath) - if os.IsNotExist(err) { - return nil, false, nil - } - if err != nil { - return nil, false, fmt.Errorf("read state file: %w", err) - } - if !info.Mode().IsRegular() { - return nil, false, fmt.Errorf("state file %s is not a regular file; pass --force to start over", statePath) - } - if info.Size() > auditLogsDownloadMaxStateBytes { - return nil, false, fmt.Errorf("state file %s is too large; pass --force to start over", statePath) - } - - f, err := os.Open(statePath) - if err != nil { - return nil, false, fmt.Errorf("read state file: %w", err) - } - defer f.Close() - openedInfo, err := f.Stat() - if err != nil { - return nil, false, fmt.Errorf("read state file: %w", err) - } - if !openedInfo.Mode().IsRegular() || !os.SameFile(info, openedInfo) { - return nil, false, fmt.Errorf("state file %s changed while it was being opened; retry", statePath) - } - raw, err := io.ReadAll(io.LimitReader(f, auditLogsDownloadMaxStateBytes+1)) - if err != nil { - return nil, false, fmt.Errorf("read state file: %w", err) - } - if len(raw) > auditLogsDownloadMaxStateBytes { - return nil, false, fmt.Errorf("state file %s is too large; pass --force to start over", statePath) - } - return raw, true, nil -} - -func removeAuditLogsDownloadState(statePath string) error { - info, err := os.Lstat(statePath) - if os.IsNotExist(err) { - return nil - } - if err != nil { - return fmt.Errorf("remove state file: %w", err) - } - if info.IsDir() { - return fmt.Errorf("state path %s is a directory; remove it before using --force", statePath) - } - if err := os.Remove(statePath); err != nil { - return fmt.Errorf("remove state file: %w", err) - } - return nil -} - -func validateAuditLogsDownloadState(state auditLogsDownloadState) error { - if state.BytesWritten < 0 || state.Chunks < 0 || state.Rows < 0 { - return fmt.Errorf("negative progress counters") - } - digest, err := hex.DecodeString(state.CommittedSHA256) - if err != nil || len(digest) != sha256.Size { - return fmt.Errorf("invalid committed checksum") - } - if state.Chunks == 0 { - if state.Cursor != "" || state.BytesWritten != 0 || state.Rows != 0 || state.CommittedSHA256 != auditLogsDownloadEmptySHA256 { - return fmt.Errorf("zero-chunk state contains committed progress") - } - } - return nil -} - func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) error { + if err := os.MkdirAll(filepath.Dir(statePath), 0o700); err != nil { + return fmt.Errorf("create output directory: %w", err) + } raw, err := json.Marshal(state) if err != nil { return fmt.Errorf("encode state: %w", err) } - if len(raw) > auditLogsDownloadMaxStateBytes { - return fmt.Errorf("state file would exceed %s; narrow the download filters", util.FormatBytes(auditLogsDownloadMaxStateBytes)) - } - // CreateTemp gives an unpredictable 0600 file, so a symlink planted at a - // guessable name in a shared directory can't redirect the write. tmp, err := os.CreateTemp(filepath.Dir(statePath), filepath.Base(statePath)+".*") if err != nil { return fmt.Errorf("write state file: %w", err) } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) if _, err := tmp.Write(raw); err != nil { tmp.Close() - os.Remove(tmp.Name()) return fmt.Errorf("write state file: %w", err) } if err := tmp.Sync(); err != nil { tmp.Close() - os.Remove(tmp.Name()) return fmt.Errorf("sync state file: %w", err) } if err := tmp.Close(); err != nil { - os.Remove(tmp.Name()) return fmt.Errorf("write state file: %w", err) } - if err := commitAuditLogsDownloadStateFile(tmp.Name(), statePath); err != nil { - os.Remove(tmp.Name()) + if err := os.Rename(tmpPath, statePath); err != nil { return fmt.Errorf("commit state file: %w", err) } return nil } -type lockedAuditLogsDownloadOutput struct { - file *os.File - pathLock *os.File - outPath string -} - -func (o *lockedAuditLogsDownloadOutput) Close() error { - return o.close(false) -} - -func (o *lockedAuditLogsDownloadOutput) close(removeOutput bool) error { - outputErr := o.file.Close() - var removeErr error - if removeOutput { - removeErr = os.Remove(o.outPath) - if removeErr == nil { - removeErr = syncAuditLogsDownloadDir(filepath.Dir(o.outPath)) - } +func removeAuditLogsDownloadState(statePath string) error { + if err := os.Remove(statePath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove state file: %w", err) } - lockErr := o.pathLock.Close() - return errors.Join(outputErr, removeErr, lockErr) + return nil } -// openAndLockAuditLogsDownloadOutput securely opens a regular output file and -// holds a stable path lock before any state is read or output is modified. -func openAndLockAuditLogsDownloadOutput(outPath string) (*lockedAuditLogsDownloadOutput, bool, error) { - if dir := filepath.Dir(outPath); dir != "." { - if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, false, fmt.Errorf("create output directory: %w", err) +func openAuditLogsDownloadOutput(outPath string, committed int64) (*os.File, error) { + if committed > 0 { + if _, err := os.Stat(outPath); err != nil { + return nil, fmt.Errorf("state records progress but %s is missing: %w", outPath, err) } } - - pathLock, err := openAndLockAuditLogsDownloadPath(outPath + ".lock") - if err != nil { - return nil, false, err + if err := os.MkdirAll(filepath.Dir(outPath), 0o700); err != nil { + return nil, fmt.Errorf("create output directory: %w", err) } - closePathLock := true - defer func() { - if closePathLock { - _ = pathLock.Close() - } - }() - - out, created, err := openAuditLogsDownloadRegularFile(outPath) + out, err := os.OpenFile(outPath, os.O_RDWR|os.O_CREATE, 0o600) if err != nil { - return nil, false, err + return nil, fmt.Errorf("open %s: %w", outPath, err) } - locked, err := tryLockAuditLogsDownloadFile(out) + info, err := out.Stat() if err != nil { out.Close() - return nil, false, fmt.Errorf("lock %s: %w", outPath, err) + return nil, fmt.Errorf("inspect %s: %w", outPath, err) } - if !locked { + if !info.Mode().IsRegular() || info.Size() < committed { out.Close() - return nil, false, fmt.Errorf("another audit-log download is already using %s", outPath) - } - if created { - if err := out.Sync(); err != nil { - out.Close() - _ = os.Remove(outPath) - return nil, false, fmt.Errorf("sync %s: %w", outPath, err) - } - if err := syncAuditLogsDownloadDir(filepath.Dir(outPath)); err != nil { - out.Close() - _ = os.Remove(outPath) - return nil, false, fmt.Errorf("sync output directory: %w", err) - } - } - closePathLock = false - return &lockedAuditLogsDownloadOutput{file: out, pathLock: pathLock, outPath: outPath}, created, nil -} - -func openAndLockAuditLogsDownloadPath(lockPath string) (*os.File, error) { - lock, _, err := openAuditLogsDownloadRegularFile(lockPath) - if err != nil { - return nil, err - } - if err := lock.Chmod(0o600); err != nil { - lock.Close() - return nil, fmt.Errorf("secure lock file permissions: %w", err) - } - locked, err := tryLockAuditLogsDownloadFile(lock) - if err != nil { - lock.Close() - return nil, fmt.Errorf("lock %s: %w", lockPath, err) - } - if !locked { - lock.Close() - return nil, fmt.Errorf("another audit-log download is already using %s", strings.TrimSuffix(lockPath, ".lock")) - } - currentInfo, err := os.Lstat(lockPath) - if err != nil { - lock.Close() - return nil, fmt.Errorf("inspect lock file: %w", err) - } - openedInfo, err := lock.Stat() - if err != nil { - lock.Close() - return nil, fmt.Errorf("inspect lock file: %w", err) - } - if !currentInfo.Mode().IsRegular() || !os.SameFile(currentInfo, openedInfo) { - lock.Close() - return nil, fmt.Errorf("lock file %s changed while it was being locked; retry", lockPath) - } - return lock, nil -} - -func openAuditLogsDownloadRegularFile(path string) (*os.File, bool, error) { - for attempt := 0; attempt < 2; attempt++ { - info, err := os.Lstat(path) - if os.IsNotExist(err) { - f, createErr := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) - if os.IsExist(createErr) { - continue - } - if createErr != nil { - return nil, false, fmt.Errorf("open %s: %w", path, createErr) - } - return f, true, nil - } - if err != nil { - return nil, false, fmt.Errorf("inspect %s: %w", path, err) - } - if !info.Mode().IsRegular() { - return nil, false, fmt.Errorf("%s must be a regular file; refusing to follow a symlink or special file", path) - } - f, err := os.OpenFile(path, os.O_RDWR, 0) - if err != nil { - return nil, false, fmt.Errorf("open %s: %w", path, err) - } - openedInfo, err := f.Stat() - if err != nil { - f.Close() - return nil, false, fmt.Errorf("inspect %s: %w", path, err) - } - if !openedInfo.Mode().IsRegular() || !os.SameFile(info, openedInfo) { - f.Close() - return nil, false, fmt.Errorf("%s changed while it was being opened; retry", path) - } - return f, false, nil - } - return nil, false, fmt.Errorf("%s changed while it was being opened; retry", path) -} - -func validateAuditLogsDownloadOutputPath(out *os.File, outPath string) error { - pathInfo, err := os.Lstat(outPath) - if err != nil { - return fmt.Errorf("%s changed while the download was running: %w", outPath, err) - } - openedInfo, err := out.Stat() - if err != nil { - return fmt.Errorf("inspect %s: %w", outPath, err) + return nil, fmt.Errorf("%s does not match saved progress", outPath) } - if !pathInfo.Mode().IsRegular() || !openedInfo.Mode().IsRegular() || !os.SameFile(pathInfo, openedInfo) { - return fmt.Errorf("%s changed while the download was running; restore it before resuming", outPath) - } - return nil -} - -// prepareAuditLogsDownloadOutput verifies the committed prefix before -// truncating an interrupted append and positioning the file for the next chunk. -func prepareAuditLogsDownloadOutput(out *os.File, outPath string, state auditLogsDownloadState, completed bool) (hash.Hash, error) { if err := out.Chmod(0o600); err != nil { - return nil, fmt.Errorf("secure permissions on %s: %w", outPath, err) - } - info, err := out.Stat() - if err != nil { - return nil, fmt.Errorf("inspect %s: %w", outPath, err) - } - if info.Size() < state.BytesWritten { - return nil, fmt.Errorf("%s is shorter than the recorded progress; it was modified or replaced — pass --force to start over", outPath) - } - if completed && info.Size() != state.BytesWritten { - return nil, fmt.Errorf("%s does not match the completed download size; pass --force to start over", outPath) - } - if _, err := out.Seek(0, io.SeekStart); err != nil { - return nil, fmt.Errorf("seek %s: %w", outPath, err) - } - committedHash := sha256.New() - if _, err := io.CopyN(committedHash, out, state.BytesWritten); err != nil { - return nil, fmt.Errorf("verify %s: %w", outPath, err) - } - got := hex.EncodeToString(committedHash.Sum(nil)) - if got != state.CommittedSHA256 { - return nil, fmt.Errorf("%s does not match the recorded checksum; it was modified or replaced — pass --force to start over", outPath) + out.Close() + return nil, fmt.Errorf("secure %s: %w", outPath, err) } - if !completed { - if err := out.Truncate(state.BytesWritten); err != nil { - return nil, fmt.Errorf("truncate %s to last committed chunk: %w", outPath, err) - } + if err := out.Truncate(committed); err != nil { + out.Close() + return nil, fmt.Errorf("truncate %s: %w", outPath, err) } - if _, err := out.Seek(state.BytesWritten, io.SeekStart); err != nil { + if _, err := out.Seek(committed, io.SeekStart); err != nil { + out.Close() return nil, fmt.Errorf("seek %s: %w", outPath, err) } - return committedHash, nil -} - -// auditLogsCredentialIdentity binds resume state to both the effective API -// origin and the authenticated organization or API key. -func auditLogsCredentialIdentity() (string, error) { - baseURL, err := normalizeAuditLogsAPIBaseURL(util.GetBaseURL()) - if err != nil { - return "", err - } - if key := os.Getenv("KERNEL_API_KEY"); key != "" { - sum := sha256.Sum256([]byte("kernel-cli-audit-logs-download:" + key)) - return baseURL + "|key:" + hex.EncodeToString(sum[:]), nil - } - tokens, err := auth.LoadTokens() - if err != nil { - return "", fmt.Errorf("determine OAuth organization for resumable download: %w", err) - } - if tokens.OrgID == "" { - return "", fmt.Errorf("cannot determine OAuth organization for resumable download; run 'kernel login --force' and retry") - } - return baseURL + "|org:" + tokens.OrgID, nil -} - -func normalizeAuditLogsAPIBaseURL(raw string) (string, error) { - u, err := url.Parse(strings.TrimSpace(raw)) - if err != nil || u.Scheme == "" || u.Host == "" { - return "", fmt.Errorf("invalid KERNEL_BASE_URL %q", raw) - } - if u.Scheme != "http" && u.Scheme != "https" { - return "", fmt.Errorf("invalid KERNEL_BASE_URL scheme %q", u.Scheme) - } - if u.User != nil { - return "", fmt.Errorf("KERNEL_BASE_URL must not contain user information") - } - if u.RawQuery != "" || u.Fragment != "" { - return "", fmt.Errorf("KERNEL_BASE_URL must not contain a query or fragment") - } - u.Scheme = strings.ToLower(u.Scheme) - u.Host = strings.ToLower(u.Host) - u.Path = strings.TrimRight(u.Path, "/") - u.RawPath = strings.TrimRight(u.RawPath, "/") - return u.String(), nil + return out, nil } func runAuditLogsDownload(cmd *cobra.Command, args []string) error { c := getAuditLogsHandler(cmd) - identity, err := auditLogsCredentialIdentity() - if err != nil { - return err - } - c.identity = identity start, _ := cmd.Flags().GetString("start") end, _ := cmd.Flags().GetString("end") search, _ := cmd.Flags().GetString("search") @@ -776,28 +369,17 @@ func runAuditLogsDownload(cmd *cobra.Command, args []string) error { force, _ := cmd.Flags().GetBool("force") return c.Download(cmd.Context(), AuditLogsDownloadInput{ - Start: start, - End: end, - Search: search, - Method: method, - ExcludeMethod: excludeMethod, - IncludeGet: includeGet, - Service: service, - AuthStrategy: authStrategy, - UserIDs: userIDs, - To: to, - Format: format, - Force: force, + Start: start, End: end, Search: search, Method: method, + ExcludeMethod: excludeMethod, IncludeGet: includeGet, Service: service, + AuthStrategy: authStrategy, UserIDs: userIDs, To: to, Format: format, Force: force, }) } var auditLogsDownloadCmd = &cobra.Command{ Use: "download", Short: "Download audit logs for a time range to a file", - Long: "Download an organization's audit log records for a time range as a file, for archival, compliance, or offline analysis.\n\n" + - "Records in the half-open [start, end) window are fetched in chunks and appended after each chunk is verified, newest first. If a download is interrupted, rerunning the same command resumes where it left off.\n\n" + - "GET requests are excluded by default; pass --include-get to include them, or --method GET to see only them.\n\n" + - "The API allows at most 30 days per download; for longer ranges run one download per window.", + Long: "Download audit logs in verified, resumable chunks. The time range is [start, end).\n\n" + + "GET requests are excluded by default; pass --include-get to include them.", Args: cobra.NoArgs, RunE: runAuditLogsDownload, } @@ -806,17 +388,16 @@ func init() { auditLogsDownloadCmd.Flags().String("start", "", "Start of the export window, RFC3339 or YYYY-MM-DD (required)") auditLogsDownloadCmd.Flags().String("end", "", "Exclusive end of the export window, RFC3339 or YYYY-MM-DD (required)") auditLogsDownloadCmd.Flags().String("search", "", "Free-text search") - auditLogsDownloadCmd.Flags().String("method", "", "Filter by HTTP method (e.g. GET)") + auditLogsDownloadCmd.Flags().String("method", "", "Filter by HTTP method") auditLogsDownloadCmd.Flags().String("exclude-method", "", "Exclude an HTTP method") auditLogsDownloadCmd.Flags().Bool("include-get", false, "Include GET requests, which are excluded by default") auditLogsDownloadCmd.Flags().String("service", "", "Filter by service") auditLogsDownloadCmd.Flags().String("auth-strategy", "", "Filter by authentication strategy") auditLogsDownloadCmd.Flags().StringArray("user-id", nil, "Filter by user ID (repeatable)") - auditLogsDownloadCmd.Flags().String("to", "", "Output file path (default: audit-logs--.)") + auditLogsDownloadCmd.Flags().String("to", "", "Output file path") auditLogsDownloadCmd.Flags().String("format", "jsonl.gz", "Output format: jsonl or jsonl.gz") auditLogsDownloadCmd.Flags().Bool("force", false, "Overwrite the output file and ignore saved progress") _ = auditLogsDownloadCmd.MarkFlagRequired("start") _ = auditLogsDownloadCmd.MarkFlagRequired("end") - auditLogsCmd.AddCommand(auditLogsDownloadCmd) } diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index 64757ab9..82c3c473 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -11,7 +11,6 @@ import ( "net/http" "os" "path/filepath" - "runtime" "strconv" "testing" "time" @@ -22,844 +21,232 @@ import ( "github.com/stretchr/testify/require" ) -type exportChunk struct { - body []byte - rowCount int - rowCountRaw string - hasMore bool - nextCursor string - sha string // overrides the computed body sha when set - omitSha bool - omitHasMore bool - omitRowCount bool +type auditLogTestChunk struct { + body []byte + rows int + hasMore bool + nextCursor string + checksum string } -func exportChunkResponse(c exportChunk) *http.Response { - sha := c.sha - if sha == "" { - sum := sha256.Sum256(c.body) - sha = hex.EncodeToString(sum[:]) +func auditLogChunkResponse(chunk auditLogTestChunk) *http.Response { + checksum := chunk.checksum + if checksum == "" { + sum := sha256.Sum256(chunk.body) + checksum = hex.EncodeToString(sum[:]) } header := http.Header{} - if !c.omitHasMore { - header.Set("X-Has-More", strconv.FormatBool(c.hasMore)) + header.Set("X-Content-Sha256", checksum) + header.Set("X-Row-Count", strconv.Itoa(chunk.rows)) + header.Set("X-Has-More", strconv.FormatBool(chunk.hasMore)) + if chunk.nextCursor != "" { + header.Set("X-Next-Cursor", chunk.nextCursor) } - if !c.omitRowCount { - rowCount := c.rowCountRaw - if rowCount == "" { - rowCount = strconv.Itoa(c.rowCount) - } - header.Set("X-Row-Count", rowCount) - } - if !c.omitSha { - header.Set("X-Content-Sha256", sha) - } - if c.nextCursor != "" { - header.Set("X-Next-Cursor", c.nextCursor) - } - return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(bytes.NewReader(c.body))} -} - -func gzipMember(t *testing.T, lines string) []byte { - t.Helper() - var buf bytes.Buffer - gz := gzip.NewWriter(&buf) - _, err := gz.Write([]byte(lines)) - require.NoError(t, err) - require.NoError(t, gz.Close()) - return buf.Bytes() -} - -func gunzipAll(t *testing.T, data []byte) string { - t.Helper() - r, err := gzip.NewReader(bytes.NewReader(data)) - require.NoError(t, err) - out, err := io.ReadAll(r) - require.NoError(t, err) - return string(out) + return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(bytes.NewReader(chunk.body))} } -// chunkServer serves a scripted sequence of responses keyed by call order and -// records the cursor each call carried. -func chunkServer(t *testing.T, responses []func() (*http.Response, error)) (*FakeAuditLogsService, *[]string) { +func auditLogChunkService(t *testing.T, responses ...func() (*http.Response, error)) (*FakeAuditLogsService, *[]string) { t.Helper() - var cursors []string - calls := 0 - fake := &FakeAuditLogsService{ + call := 0 + cursors := make([]string, 0, len(responses)) + service := &FakeAuditLogsService{ ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { cursors = append(cursors, query.Cursor.Value) - require.Less(t, calls, len(responses), "more ExportChunk calls than scripted responses") - res, err := responses[calls]() - calls++ - return res, err + require.Less(t, call, len(responses)) + response, err := responses[call]() + call++ + return response, err }, } - return fake, &cursors + return service, &cursors } -func downloadInput(outPath string) AuditLogsDownloadInput { - return AuditLogsDownloadInput{ - Start: "2026-06-01", - End: "2026-06-28", - To: outPath, - Format: "jsonl.gz", - } -} - -const auditLogsDownloadTestIdentity = "https://api.test|org:org-test" - -func auditLogsDownloadTestCmd(service AuditLogsService) AuditLogsCmd { - return AuditLogsCmd{auditLogs: service, identity: auditLogsDownloadTestIdentity} -} - -func TestAuditLogsDownloadMultiChunk(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk1 := gzipMember(t, "{\"n\":1}\n{\"n\":2}\n") - chunk2 := gzipMember(t, "{\"n\":3}\n") - chunk3 := gzipMember(t, "{\"n\":4}\n{\"n\":5}\n") - fake, cursors := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk1, rowCount: 2, hasMore: true, nextCursor: "c1"}), nil - }, - // A short chunk with hasMore=true must not end the download. - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk2, rowCount: 1, hasMore: true, nextCursor: "c2"}), nil - }, - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk3, rowCount: 2, hasMore: false}), nil - }, - }) - - c := auditLogsDownloadTestCmd(fake) - require.NoError(t, c.Download(context.Background(), downloadInput(outPath))) - - assert.Equal(t, []string{"", "c1", "c2"}, *cursors) - data, err := os.ReadFile(outPath) +func auditLogGzip(t *testing.T, body string) []byte { + t.Helper() + var out bytes.Buffer + writer := gzip.NewWriter(&out) + _, err := writer.Write([]byte(body)) require.NoError(t, err) - assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n{\"n\":4}\n{\"n\":5}\n", gunzipAll(t, data)) - _, err = os.Stat(outPath + ".state.json") - assert.True(t, os.IsNotExist(err), "state file should be removed on completion") + require.NoError(t, writer.Close()) + return out.Bytes() } -func TestAuditLogsDownloadChecksumMismatchRetriesThenSucceeds(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, sha: "deadbeef"}), nil - }, - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1}), nil - }, - }) - - c := auditLogsDownloadTestCmd(fake) - require.NoError(t, c.Download(context.Background(), downloadInput(outPath))) - - data, err := os.ReadFile(outPath) +func readAuditLogGzip(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) require.NoError(t, err) - assert.Equal(t, "{\"n\":1}\n", gunzipAll(t, data), "corrupt chunk must not be written") + reader, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + body, err := io.ReadAll(reader) + require.NoError(t, err) + return string(body) } -func TestAuditLogsDownloadChecksumMismatchExhaustsRetries(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk := gzipMember(t, "{\"n\":1}\n") - bad := func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, sha: "deadbeef"}), nil - } - fake, _ := chunkServer(t, []func() (*http.Response, error){bad, bad, bad}) - - c := auditLogsDownloadTestCmd(fake) - err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "checksum mismatch") - - data, err := os.ReadFile(outPath) - require.NoError(t, err) - assert.Empty(t, data, "no bytes may be written for a chunk that never verified") +func auditLogsDownloadInput(path string) AuditLogsDownloadInput { + return AuditLogsDownloadInput{Start: "2026-06-01", End: "2026-06-28", To: path, Format: "jsonl.gz"} } -func TestAuditLogsDownloadResumesAfterFailure(t *testing.T) { +func TestAuditLogsDownloadWritesAllChunks(t *testing.T) { capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk1 := gzipMember(t, "{\"n\":1}\n") - chunk2 := gzipMember(t, "{\"n\":2}\n") - - fake, _ := chunkServer(t, []func() (*http.Response, error){ + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + first := auditLogGzip(t, "{\"n\":1}\n") + second := auditLogGzip(t, "{\"n\":2}\n") + service, cursors := auditLogChunkService(t, func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + return auditLogChunkResponse(auditLogTestChunk{body: first, rows: 1, hasMore: true, nextCursor: "next"}), nil }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - c := auditLogsDownloadTestCmd(fake) - require.Error(t, c.Download(context.Background(), downloadInput(outPath))) - - fake2, cursors := chunkServer(t, []func() (*http.Response, error){ func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk2, rowCount: 1, hasMore: false}), nil + return auditLogChunkResponse(auditLogTestChunk{body: second, rows: 1}), nil }, - }) - c2 := auditLogsDownloadTestCmd(fake2) - require.NoError(t, c2.Download(context.Background(), downloadInput(outPath))) + ) - assert.Equal(t, []string{"c1"}, *cursors, "resume must continue from the saved cursor") - data, err := os.ReadFile(outPath) + err := (AuditLogsCmd{auditLogs: service}).Download(context.Background(), auditLogsDownloadInput(outPath)) require.NoError(t, err) - assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", gunzipAll(t, data), "resumed file must have no duplicate or missing rows") + assert.Equal(t, []string{"", "next"}, *cursors) + assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", readAuditLogGzip(t, outPath)) _, err = os.Stat(outPath + ".state.json") assert.True(t, os.IsNotExist(err)) } -func TestAuditLogsDownloadResumeTruncatesTornWrite(t *testing.T) { +func TestAuditLogsDownloadResumesFromCommittedChunk(t *testing.T) { capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk1 := gzipMember(t, "{\"n\":1}\n") - chunk2 := gzipMember(t, "{\"n\":2}\n") - - fake, _ := chunkServer(t, []func() (*http.Response, error){ + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + first := auditLogGzip(t, "{\"n\":1}\n") + service, _ := auditLogChunkService(t, func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + return auditLogChunkResponse(auditLogTestChunk{body: first, rows: 1, hasMore: true, nextCursor: "next"}), nil }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - c := auditLogsDownloadTestCmd(fake) - require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + func() (*http.Response, error) { return nil, errors.New("network error") }, + ) + require.Error(t, (AuditLogsCmd{auditLogs: service}).Download(context.Background(), auditLogsDownloadInput(outPath))) - // Simulate a crash mid-append: garbage past the committed offset. - f, err := os.OpenFile(outPath, os.O_WRONLY|os.O_APPEND, 0o644) + file, err := os.OpenFile(outPath, os.O_APPEND|os.O_WRONLY, 0) require.NoError(t, err) - _, err = f.Write([]byte("torn partial chunk")) + _, err = file.WriteString("partial") require.NoError(t, err) - require.NoError(t, f.Close()) + require.NoError(t, file.Close()) - fake2, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk2, rowCount: 1, hasMore: false}), nil - }, + second := auditLogGzip(t, "{\"n\":2}\n") + resumed, cursors := auditLogChunkService(t, func() (*http.Response, error) { + return auditLogChunkResponse(auditLogTestChunk{body: second, rows: 1}), nil }) - c2 := auditLogsDownloadTestCmd(fake2) - require.NoError(t, c2.Download(context.Background(), downloadInput(outPath))) - - data, err := os.ReadFile(outPath) - require.NoError(t, err) - assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", gunzipAll(t, data)) + require.NoError(t, (AuditLogsCmd{auditLogs: resumed}).Download(context.Background(), auditLogsDownloadInput(outPath))) + assert.Equal(t, []string{"next"}, *cursors) + assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", readAuditLogGzip(t, outPath)) } -func TestAuditLogsDownloadRejectsMismatchedState(t *testing.T) { +func TestAuditLogsDownloadRejectsChangedParamsOnResume(t *testing.T) { capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk1 := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + chunk := auditLogGzip(t, "{\"n\":1}\n") + service, _ := auditLogChunkService(t, func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil + return auditLogChunkResponse(auditLogTestChunk{body: chunk, rows: 1, hasMore: true, nextCursor: "next"}), nil }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - c := auditLogsDownloadTestCmd(fake) - require.Error(t, c.Download(context.Background(), downloadInput(outPath))) + func() (*http.Response, error) { return nil, errors.New("network error") }, + ) + in := auditLogsDownloadInput(outPath) + require.Error(t, (AuditLogsCmd{auditLogs: service}).Download(context.Background(), in)) - in := downloadInput(outPath) in.Service = "api" - err := c.Download(context.Background(), in) - require.ErrorContains(t, err, "different parameters") + err := (AuditLogsCmd{auditLogs: service}).Download(context.Background(), in) + require.ErrorContains(t, err, "does not match this download") } -func TestAuditLogsDownloadForceRestarts(t *testing.T) { +func TestAuditLogsDownloadRejectsBadChunkBeforeWriting(t *testing.T) { capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk1 := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil - }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + service, _ := auditLogChunkService(t, func() (*http.Response, error) { + return auditLogChunkResponse(auditLogTestChunk{body: []byte("bad"), rows: 1, checksum: "wrong"}), nil }) - c := auditLogsDownloadTestCmd(fake) - require.Error(t, c.Download(context.Background(), downloadInput(outPath))) - chunk := gzipMember(t, "{\"n\":9}\n") - fake2, cursors := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: false}), nil - }, - }) - in := downloadInput(outPath) - in.Force = true - c2 := auditLogsDownloadTestCmd(fake2) - require.NoError(t, c2.Download(context.Background(), in)) - - assert.Equal(t, []string{""}, *cursors, "--force must restart from the beginning") - data, err := os.ReadFile(outPath) - require.NoError(t, err) - assert.Equal(t, "{\"n\":9}\n", gunzipAll(t, data)) + err := (AuditLogsCmd{auditLogs: service}).Download(context.Background(), auditLogsDownloadInput(outPath)) + require.ErrorContains(t, err, "checksum mismatch") + data, readErr := os.ReadFile(outPath) + require.NoError(t, readErr) + assert.Empty(t, data) } -func TestAuditLogsDownloadRefusesExistingOutputWithoutState(t *testing.T) { +func TestAuditLogsDownloadForceRestarts(t *testing.T) { capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - require.NoError(t, os.WriteFile(outPath, []byte("existing"), 0o644)) - - c := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) - err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "already exists") -} - -func TestAuditLogsDownloadFinishesCompletedLeftoverState(t *testing.T) { - buf := capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk1 := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: false}), nil - }, + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + require.NoError(t, os.WriteFile(outPath, []byte("old"), 0o600)) + chunk := auditLogGzip(t, "{\"n\":1}\n") + service, cursors := auditLogChunkService(t, func() (*http.Response, error) { + return auditLogChunkResponse(auditLogTestChunk{body: chunk, rows: 1}), nil }) - c := auditLogsDownloadTestCmd(fake) - require.NoError(t, c.Download(context.Background(), downloadInput(outPath))) - - // Simulate a crash after the final state save but before cleanup by - // re-creating the completed state file. - statePath := outPath + ".state.json" - sum := sha256.Sum256(chunk1) - require.NoError(t, saveAuditLogsDownloadState(statePath, auditLogsDownloadState{ - Version: auditLogsDownloadStateVersion, - Params: mustFingerprint(t), - Identity: auditLogsDownloadTestIdentity, - BytesWritten: int64(len(chunk1)), - CommittedSHA256: hex.EncodeToString(sum[:]), - Chunks: 1, - Rows: 1, - })) - - c2 := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) - require.NoError(t, c2.Download(context.Background(), downloadInput(outPath))) - assert.Contains(t, buf.String(), "already complete") - _, err := os.Stat(statePath) - assert.True(t, os.IsNotExist(err)) -} - -func mustFingerprint(t *testing.T) string { - t.Helper() - params, err := buildAuditLogsDownloadParams(downloadInput("")) - require.NoError(t, err) - params.Format = kernel.AuditLogExportChunkParamsFormatJSONLGz - fp, err := auditLogsDownloadFingerprint(params) - require.NoError(t, err) - return fp -} + in := auditLogsDownloadInput(outPath) + in.Force = true -func TestAuditLogsDownloadRangeOver30DaysSuggestsWindows(t *testing.T) { - buf := capturePtermOutput(t) - c := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) - err := c.Download(context.Background(), AuditLogsDownloadInput{Start: "2026-04-01", End: "2026-06-30"}) - require.ErrorContains(t, err, "at most 30 days") - assert.Contains(t, buf.String(), "--start 2026-05-31T00:00:00Z --end 2026-06-30T00:00:00Z") - assert.Contains(t, buf.String(), "--start 2026-04-01T00:00:00Z --end 2026-05-01T00:00:00Z") + require.NoError(t, (AuditLogsCmd{auditLogs: service}).Download(context.Background(), in)) + assert.Equal(t, []string{""}, *cursors) + assert.Equal(t, "{\"n\":1}\n", readAuditLogGzip(t, outPath)) } -func TestAuditLogsDownloadDateEndIsExclusive(t *testing.T) { - in := downloadInput("") +func TestBuildAuditLogsDownloadParams(t *testing.T) { + in := auditLogsDownloadInput("") + in.Search = "browser" + in.Service = "api" + in.UserIDs = []string{"user_1"} params, err := buildAuditLogsDownloadParams(in) require.NoError(t, err) assert.Equal(t, time.Date(2026, 6, 28, 0, 0, 0, 0, time.UTC), params.End) + assert.Equal(t, "browser", params.Search.Value) + assert.Equal(t, "GET", params.ExcludeMethod.Value) + assert.Equal(t, "api", params.Service.Value) + assert.Equal(t, []string{"user_1"}, params.SearchUserID) } -func TestAuditLogsDownloadServerCursorBugs(t *testing.T) { - capturePtermOutput(t) - chunk := gzipMember(t, "{\"n\":1}\n") - - t.Run("has more without cursor", func(t *testing.T) { - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true}), nil - }, - }) - c := auditLogsDownloadTestCmd(fake) - err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "no cursor") - }) - - t.Run("unchanged cursor", func(t *testing.T) { - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - repeat := func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil - } - fake, _ := chunkServer(t, []func() (*http.Response, error){repeat, repeat}) - c := auditLogsDownloadTestCmd(fake) - err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "unchanged cursor") - }) -} - -func TestAuditLogsDownloadJSONLFormat(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl") - var gotFormat kernel.AuditLogExportChunkParamsFormat - fake := &FakeAuditLogsService{ - ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { - gotFormat = query.Format - return exportChunkResponse(exportChunk{body: []byte("{\"n\":1}\n"), rowCount: 1}), nil - }, - } - in := downloadInput(outPath) - in.Format = "jsonl" - c := auditLogsDownloadTestCmd(fake) - require.NoError(t, c.Download(context.Background(), in)) - - assert.Equal(t, kernel.AuditLogExportChunkParamsFormatJSONL, gotFormat) - data, err := os.ReadFile(outPath) - require.NoError(t, err) - assert.Equal(t, "{\"n\":1}\n", string(data)) -} - -func TestAuditLogsDownloadExcludesGetByDefault(t *testing.T) { - cases := []struct { - name string - mutate func(*AuditLogsDownloadInput) - wantExclude string - wantErr string +func TestBuildAuditLogsDownloadParamsRejectsInvalidInput(t *testing.T) { + tests := []struct { + name string + in AuditLogsDownloadInput + want string }{ - {name: "default excludes GET", mutate: func(in *AuditLogsDownloadInput) {}, wantExclude: "GET"}, - {name: "include-get lifts the default", mutate: func(in *AuditLogsDownloadInput) { in.IncludeGet = true }, wantExclude: ""}, - {name: "method filter lifts the default", mutate: func(in *AuditLogsDownloadInput) { in.Method = "POST" }, wantExclude: ""}, - {name: "explicit GET exclusion", mutate: func(in *AuditLogsDownloadInput) { in.ExcludeMethod = "get" }, wantExclude: "get"}, - {name: "other exclusion with include-get", mutate: func(in *AuditLogsDownloadInput) { - in.ExcludeMethod = "OPTIONS" - in.IncludeGet = true - }, wantExclude: "OPTIONS"}, - {name: "other exclusion without include-get is rejected", mutate: func(in *AuditLogsDownloadInput) { in.ExcludeMethod = "OPTIONS" }, - wantErr: "add --include-get"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - in := downloadInput("") - tc.mutate(&in) - params, err := buildAuditLogsDownloadParams(in) - if tc.wantErr != "" { - require.ErrorContains(t, err, tc.wantErr) - return - } - require.NoError(t, err) - assert.Equal(t, tc.wantExclude, params.ExcludeMethod.Value) - }) - } -} - -func TestAuditLogsDownloadRejectsDifferentCredentials(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk1 := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil - }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - c := AuditLogsCmd{auditLogs: fake, identity: "org:org-a"} - require.Error(t, c.Download(context.Background(), downloadInput(outPath))) - - c2 := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}, identity: "org:org-b"} - err := c2.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "different API origin or credential") -} - -func TestAuditLogsDownloadRejectsMissingOrShortenedOutput(t *testing.T) { - capturePtermOutput(t) - interrupted := func(t *testing.T) (string, AuditLogsCmd) { - t.Helper() - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk1 := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil - }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - c := auditLogsDownloadTestCmd(fake) - require.Error(t, c.Download(context.Background(), downloadInput(outPath))) - return outPath, auditLogsDownloadTestCmd(&FakeAuditLogsService{}) - } - - t.Run("missing output", func(t *testing.T) { - outPath, c := interrupted(t) - require.NoError(t, os.Remove(outPath)) - err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "is missing") - }) - - t.Run("shortened output", func(t *testing.T) { - outPath, c := interrupted(t) - require.NoError(t, os.Truncate(outPath, 1)) - err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "shorter than the recorded progress") - }) -} - -func TestAuditLogsDownloadFirstChunkFailureRetriesWithoutForce(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - failing, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - c := auditLogsDownloadTestCmd(failing) - require.Error(t, c.Download(context.Background(), downloadInput(outPath))) - - chunk := gzipMember(t, "{\"n\":1}\n") - fake, cursors := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: false}), nil - }, - }) - c2 := auditLogsDownloadTestCmd(fake) - require.NoError(t, c2.Download(context.Background(), downloadInput(outPath)), "a failed first chunk must not require --force to retry") - - assert.Equal(t, []string{""}, *cursors) - data, err := os.ReadFile(outPath) - require.NoError(t, err) - assert.Equal(t, "{\"n\":1}\n", gunzipAll(t, data)) -} - -func TestAuditLogsDownloadFailedForceLeavesNoStaleProgress(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk1 := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk1, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil - }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - c := auditLogsDownloadTestCmd(fake) - require.Error(t, c.Download(context.Background(), downloadInput(outPath))) - - // --force run that dies before its first chunk. - failing, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - in := downloadInput(outPath) - in.Force = true - c2 := auditLogsDownloadTestCmd(failing) - require.Error(t, c2.Download(context.Background(), in)) - - // The plain rerun must start from scratch, not resume chunk 1's stale cursor. - chunk := gzipMember(t, "{\"n\":9}\n") - fake3, cursors := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: false}), nil - }, - }) - c3 := auditLogsDownloadTestCmd(fake3) - require.NoError(t, c3.Download(context.Background(), downloadInput(outPath))) - - assert.Equal(t, []string{""}, *cursors) - data, err := os.ReadFile(outPath) - require.NoError(t, err) - assert.Equal(t, "{\"n\":9}\n", gunzipAll(t, data)) -} - -func TestAuditLogsDownloadFailsClosedOnMissingHeaders(t *testing.T) { - capturePtermOutput(t) - chunk := gzipMember(t, "{\"n\":1}\n") - - t.Run("missing X-Has-More", func(t *testing.T) { - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, omitHasMore: true}), nil - }, - }) - c := auditLogsDownloadTestCmd(fake) - err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "X-Has-More") - data, readErr := os.ReadFile(outPath) - require.NoError(t, readErr) - assert.Empty(t, data, "an unvalidated response must not be written") - }) - - t.Run("missing X-Content-Sha256", func(t *testing.T) { - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, omitSha: true}), nil - }, + {name: "missing bounds", in: AuditLogsDownloadInput{}, want: "--start and --end are required"}, + {name: "reversed bounds", in: AuditLogsDownloadInput{Start: "2026-06-02", End: "2026-06-01"}, want: "--start must be before --end"}, + {name: "range too large", in: AuditLogsDownloadInput{Start: "2026-05-01", End: "2026-06-01"}, want: "at most 30 days"}, + {name: "conflicting exclusion", in: AuditLogsDownloadInput{Start: "2026-06-01", End: "2026-06-02", ExcludeMethod: "POST"}, want: "--include-get"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := buildAuditLogsDownloadParams(test.in) + require.ErrorContains(t, err, test.want) }) - c := auditLogsDownloadTestCmd(fake) - err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "X-Content-Sha256") - }) -} - -func TestAuditLogsDownloadRejectsOversizedChunk(t *testing.T) { - capturePtermOutput(t) - prev := auditLogsDownloadMaxChunkBytes - auditLogsDownloadMaxChunkBytes = 8 - t.Cleanup(func() { auditLogsDownloadMaxChunkBytes = prev }) - - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: []byte("well over eight bytes"), rowCount: 1}), nil - }, - }) - c := auditLogsDownloadTestCmd(fake) - err := c.Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "refusing to buffer") -} - -func TestAuditLogsDownloadFilePermissions(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil - }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - c := auditLogsDownloadTestCmd(fake) - require.Error(t, c.Download(context.Background(), downloadInput(outPath))) - - outInfo, err := os.Stat(outPath) - require.NoError(t, err) - assert.Equal(t, os.FileMode(0o600), outInfo.Mode().Perm()) - stateInfo, err := os.Stat(outPath + ".state.json") - require.NoError(t, err) - assert.Equal(t, os.FileMode(0o600), stateInfo.Mode().Perm()) - lockInfo, err := os.Stat(outPath + ".lock") - require.NoError(t, err) - assert.Equal(t, os.FileMode(0o600), lockInfo.Mode().Perm()) -} - -func TestAuditLogsDownloadRequiresStableTimeBounds(t *testing.T) { - c := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) - for _, in := range []AuditLogsDownloadInput{ - {End: "2026-06-28"}, - {Start: "2026-06-01"}, - {}, - } { - err := c.Download(context.Background(), in) - require.ErrorContains(t, err, "--start and --end are required") } } -func TestAuditLogsCredentialIdentityIncludesAPIOrigin(t *testing.T) { - t.Setenv("KERNEL_API_KEY", "test-key") - t.Setenv("KERNEL_BASE_URL", "HTTPS://API.EXAMPLE.COM/") - first, err := auditLogsCredentialIdentity() - require.NoError(t, err) - assert.Contains(t, first, "https://api.example.com|key:") - - t.Setenv("KERNEL_BASE_URL", "https://api.other.example.com") - second, err := auditLogsCredentialIdentity() - require.NoError(t, err) - assert.NotEqual(t, first, second) -} - -func TestAuditLogsDownloadRejectsEmptyIdentity(t *testing.T) { - c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} - err := c.Download(context.Background(), downloadInput(filepath.Join(t.TempDir(), "out.jsonl.gz"))) - require.ErrorContains(t, err, "without an API origin and credential identity") -} - -func TestAuditLogsDownloadRejectsModifiedCommittedOutput(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - chunk := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1, hasMore: true, nextCursor: "c1"}), nil - }, - func() (*http.Response, error) { return nil, errors.New("boom") }, - }) - require.Error(t, auditLogsDownloadTestCmd(fake).Download(context.Background(), downloadInput(outPath))) - - require.NoError(t, os.WriteFile(outPath, bytes.Repeat([]byte{'x'}, len(chunk)), 0o600)) - called := false - resume := &FakeAuditLogsService{ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { - called = true - return nil, errors.New("should not fetch") - }} - err := auditLogsDownloadTestCmd(resume).Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "does not match the recorded checksum") - assert.False(t, called) -} - -func TestAuditLogsDownloadOutputLock(t *testing.T) { - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - first, _, err := openAndLockAuditLogsDownloadOutput(outPath) - require.NoError(t, err) - - _, _, err = openAndLockAuditLogsDownloadOutput(outPath) - require.ErrorContains(t, err, "already using") - require.NoError(t, first.Close()) - - third, _, err := openAndLockAuditLogsDownloadOutput(outPath) - require.NoError(t, err) - require.NoError(t, third.Close()) -} - -func TestAuditLogsDownloadPathLockSurvivesOutputRename(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows prevents renaming an open output file") - } - dir := t.TempDir() - outPath := filepath.Join(dir, "out.jsonl.gz") - movedPath := filepath.Join(dir, "moved.jsonl.gz") - first, _, err := openAndLockAuditLogsDownloadOutput(outPath) - require.NoError(t, err) - require.NoError(t, os.Rename(outPath, movedPath)) - - _, _, err = openAndLockAuditLogsDownloadOutput(outPath) - require.ErrorContains(t, err, "already using") - require.NoError(t, first.Close()) - - third, created, err := openAndLockAuditLogsDownloadOutput(outPath) - require.NoError(t, err) - assert.True(t, created) - require.NoError(t, third.Close()) -} - -func TestAuditLogsDownloadDetectsOutputRename(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows prevents renaming an open output file") - } - capturePtermOutput(t) - dir := t.TempDir() - outPath := filepath.Join(dir, "out.jsonl.gz") - movedPath := filepath.Join(dir, "moved.jsonl.gz") - chunk := gzipMember(t, "{\"n\":1}\n") - fake := &FakeAuditLogsService{ - ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { - require.NoError(t, os.Rename(outPath, movedPath)) - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1}), nil - }, - } - - err := auditLogsDownloadTestCmd(fake).Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "changed while the download was running") -} - -func TestAuditLogsDownloadRejectsSymlinkOutput(t *testing.T) { - capturePtermOutput(t) - dir := t.TempDir() - target := filepath.Join(dir, "target") - outPath := filepath.Join(dir, "out.jsonl.gz") - require.NoError(t, os.WriteFile(target, []byte("keep"), 0o600)) - if err := os.Symlink(target, outPath); err != nil { - t.Skipf("symlinks unavailable: %v", err) - } - - in := downloadInput(outPath) - in.Force = true - err := auditLogsDownloadTestCmd(&FakeAuditLogsService{}).Download(context.Background(), in) - require.ErrorContains(t, err, "must be a regular file") - data, readErr := os.ReadFile(target) - require.NoError(t, readErr) - assert.Equal(t, "keep", string(data)) -} - -func TestAuditLogsDownloadStateReadIsBoundedAndForceBypassesIt(t *testing.T) { - statePath := filepath.Join(t.TempDir(), "out.state.json") - require.NoError(t, os.WriteFile(statePath, make([]byte, auditLogsDownloadMaxStateBytes+1), 0o600)) - - _, _, err := loadAuditLogsDownloadState(statePath, "params", auditLogsDownloadTestIdentity, false) - require.ErrorContains(t, err, "is too large") - state, exists, err := loadAuditLogsDownloadState(statePath, "params", auditLogsDownloadTestIdentity, true) - require.NoError(t, err) - assert.False(t, exists) - assert.Equal(t, auditLogsDownloadEmptySHA256, state.CommittedSHA256) - _, err = os.Lstat(statePath) - assert.True(t, os.IsNotExist(err)) -} - -func TestAuditLogsDownloadRejectsInconsistentState(t *testing.T) { - statePath := filepath.Join(t.TempDir(), "out.state.json") - require.NoError(t, saveAuditLogsDownloadState(statePath, auditLogsDownloadState{ - Version: auditLogsDownloadStateVersion, - Params: "params", - Identity: auditLogsDownloadTestIdentity, - BytesWritten: 1, - CommittedSHA256: auditLogsDownloadEmptySHA256, - })) - - _, _, err := loadAuditLogsDownloadState(statePath, "params", auditLogsDownloadTestIdentity, false) - require.ErrorContains(t, err, "zero-chunk state contains committed progress") -} - -func TestAuditLogsDownloadForceSecuresExistingOutput(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - require.NoError(t, os.WriteFile(outPath, []byte("old"), 0o644)) - require.NoError(t, os.Chmod(outPath, 0o644)) - chunk := gzipMember(t, "{\"n\":1}\n") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { - return exportChunkResponse(exportChunk{body: chunk, rowCount: 1}), nil - }, - }) - in := downloadInput(outPath) - in.Force = true - require.NoError(t, auditLogsDownloadTestCmd(fake).Download(context.Background(), in)) - - info, err := os.Stat(outPath) - require.NoError(t, err) - assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) -} - -func TestAuditLogsDownloadRejectsInvalidRowCount(t *testing.T) { - capturePtermOutput(t) - chunk := gzipMember(t, "{\"n\":1}\n") +func TestParseAuditLogsChunkHeaders(t *testing.T) { tests := []struct { - name string - chunk exportChunk + name string + header http.Header + current string + want string }{ - {name: "missing", chunk: exportChunk{body: chunk, omitRowCount: true}}, - {name: "malformed", chunk: exportChunk{body: chunk, rowCountRaw: "nope"}}, - {name: "negative", chunk: exportChunk{body: chunk, rowCountRaw: "-1"}}, - {name: "above server limit", chunk: exportChunk{body: chunk, rowCountRaw: "50001"}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - outPath := filepath.Join(t.TempDir(), "out.jsonl.gz") - fake, _ := chunkServer(t, []func() (*http.Response, error){ - func() (*http.Response, error) { return exportChunkResponse(tc.chunk), nil }, - }) - err := auditLogsDownloadTestCmd(fake).Download(context.Background(), downloadInput(outPath)) - require.ErrorContains(t, err, "X-Row-Count") - data, readErr := os.ReadFile(outPath) - require.NoError(t, readErr) - assert.Empty(t, data) + {name: "missing has more", header: http.Header{"X-Row-Count": []string{"1"}}, want: "X-Has-More"}, + {name: "missing row count", header: http.Header{"X-Has-More": []string{"false"}}, want: "X-Row-Count"}, + {name: "missing cursor", header: http.Header{"X-Has-More": []string{"true"}, "X-Row-Count": []string{"1"}}, want: "X-Next-Cursor"}, + {name: "unchanged cursor", current: "next", header: http.Header{"X-Has-More": []string{"true"}, "X-Row-Count": []string{"1"}, "X-Next-Cursor": []string{"next"}}, want: "X-Next-Cursor"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, _, err := parseAuditLogsChunkHeaders(test.header, test.current) + require.ErrorContains(t, err, test.want) }) } } +func TestAuditLogsDownloadRejectsExistingOutput(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + require.NoError(t, os.WriteFile(outPath, []byte("keep"), 0o600)) + err := (AuditLogsCmd{auditLogs: &FakeAuditLogsService{}}).Download(context.Background(), auditLogsDownloadInput(outPath)) + require.ErrorContains(t, err, "already exists") +} + func TestAuditLogsDownloadRejectsInvalidFormat(t *testing.T) { - c := auditLogsDownloadTestCmd(&FakeAuditLogsService{}) - in := downloadInput("out.csv") + in := auditLogsDownloadInput(filepath.Join(t.TempDir(), "audit")) in.Format = "csv" - err := c.Download(context.Background(), in) - require.ErrorContains(t, err, "--format must be jsonl or jsonl.gz") + err := (AuditLogsCmd{auditLogs: &FakeAuditLogsService{}}).Download(context.Background(), in) + require.ErrorContains(t, err, "--format") } diff --git a/cmd/audit_logs_filelock_other.go b/cmd/audit_logs_filelock_other.go deleted file mode 100644 index 84da28c5..00000000 --- a/cmd/audit_logs_filelock_other.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build !darwin && !linux && !windows - -package cmd - -import ( - "fmt" - "os" -) - -func tryLockAuditLogsDownloadFile(file *os.File) (bool, error) { - return false, fmt.Errorf("audit-log download locking is unsupported on this platform") -} - -func commitAuditLogsDownloadStateFile(oldPath, newPath string) error { - return os.Rename(oldPath, newPath) -} - -func syncAuditLogsDownloadDir(dir string) error { - return nil -} diff --git a/cmd/audit_logs_filelock_unix.go b/cmd/audit_logs_filelock_unix.go deleted file mode 100644 index 0627210f..00000000 --- a/cmd/audit_logs_filelock_unix.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build darwin || linux - -package cmd - -import ( - "errors" - "os" - "path/filepath" - - "golang.org/x/sys/unix" -) - -func tryLockAuditLogsDownloadFile(file *os.File) (bool, error) { - err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) - if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { - return false, nil - } - if err != nil { - return false, err - } - return true, nil -} - -func commitAuditLogsDownloadStateFile(oldPath, newPath string) error { - if err := os.Rename(oldPath, newPath); err != nil { - return err - } - return syncAuditLogsDownloadDir(filepath.Dir(newPath)) -} - -func syncAuditLogsDownloadDir(dir string) error { - f, err := os.Open(dir) - if err != nil { - return err - } - return errors.Join(f.Sync(), f.Close()) -} diff --git a/cmd/audit_logs_filelock_windows.go b/cmd/audit_logs_filelock_windows.go deleted file mode 100644 index 980ff771..00000000 --- a/cmd/audit_logs_filelock_windows.go +++ /dev/null @@ -1,44 +0,0 @@ -//go:build windows - -package cmd - -import ( - "errors" - "os" - - "golang.org/x/sys/windows" -) - -func tryLockAuditLogsDownloadFile(file *os.File) (bool, error) { - err := windows.LockFileEx( - windows.Handle(file.Fd()), - windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, - 0, - 1, - 0, - &windows.Overlapped{}, - ) - if errors.Is(err, windows.ERROR_LOCK_VIOLATION) || errors.Is(err, windows.ERROR_IO_PENDING) { - return false, nil - } - if err != nil { - return false, err - } - return true, nil -} - -func commitAuditLogsDownloadStateFile(oldPath, newPath string) error { - oldPathPtr, err := windows.UTF16PtrFromString(oldPath) - if err != nil { - return err - } - newPathPtr, err := windows.UTF16PtrFromString(newPath) - if err != nil { - return err - } - return windows.MoveFileEx(oldPathPtr, newPathPtr, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) -} - -func syncAuditLogsDownloadDir(dir string) error { - return nil -} diff --git a/go.mod b/go.mod index 6d657039..b3939d33 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( golang.org/x/crypto v0.47.0 golang.org/x/oauth2 v0.30.0 golang.org/x/sync v0.19.0 - golang.org/x/sys v0.40.0 golang.org/x/term v0.39.0 ) @@ -57,6 +56,7 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) From f93b06f58f39b630c2f362a795515259015c41a6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:56:52 +0000 Subject: [PATCH 07/17] Disambiguate audit log download files --- cmd/audit_logs.go | 3 ++- cmd/audit_logs_download.go | 32 +++++++++++++++++++++++--------- cmd/audit_logs_download_test.go | 22 ++++++++++++++++++++++ 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/cmd/audit_logs.go b/cmd/audit_logs.go index da60f527..9c6a6e7d 100644 --- a/cmd/audit_logs.go +++ b/cmd/audit_logs.go @@ -22,7 +22,8 @@ type AuditLogsService interface { } type AuditLogsCmd struct { - auditLogs AuditLogsService + auditLogs AuditLogsService + downloadIdentity string } type AuditLogsSearchInput struct { diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 8e90fabc..f8cb11c1 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/kernel/cli/pkg/auth" "github.com/kernel/cli/pkg/util" "github.com/kernel/kernel-go-sdk" "github.com/pterm/pterm" @@ -65,15 +66,15 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e } params.Format = kernel.AuditLogExportChunkParamsFormat(format) + fingerprint, err := auditLogsDownloadFingerprint(params, c.downloadIdentity) + if err != nil { + return err + } outPath := in.To if outPath == "" { - outPath = defaultAuditLogsDownloadPath(params.Start, params.End, format) + outPath = defaultAuditLogsDownloadPath(params.Start, params.End, format, fingerprint) } statePath := outPath + ".state.json" - fingerprint, err := auditLogsDownloadFingerprint(params) - if err != nil { - return err - } state, exists, err := loadAuditLogsDownloadState(statePath, outPath, fingerprint, in.Force) if err != nil { return err @@ -235,17 +236,29 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp return params, nil } -func auditLogsDownloadFingerprint(params kernel.AuditLogExportChunkParams) (string, error) { +func auditLogsDownloadFingerprint(params kernel.AuditLogExportChunkParams, identity string) (string, error) { query, err := params.URLQuery() if err != nil { return "", fmt.Errorf("fingerprint params: %w", err) } - return query.Encode(), nil + sum := sha256.Sum256([]byte(query.Encode() + "\n" + identity)) + return hex.EncodeToString(sum[:]), nil } -func defaultAuditLogsDownloadPath(start, end time.Time, format string) string { +func defaultAuditLogsDownloadPath(start, end time.Time, format, fingerprint string) string { const stamp = "20060102T150405Z" - return fmt.Sprintf("audit-logs-%s-%s.%s", start.UTC().Format(stamp), end.UTC().Format(stamp), format) + return fmt.Sprintf("audit-logs-%s-%s-%s.%s", start.UTC().Format(stamp), end.UTC().Format(stamp), fingerprint[:8], format) +} + +func currentAuditLogsDownloadIdentity() string { + identity := strings.TrimRight(strings.TrimSpace(util.GetBaseURL()), "/") + if apiKey := os.Getenv("KERNEL_API_KEY"); apiKey != "" { + return identity + "\napi-key:" + apiKey + } + if tokens, err := auth.LoadTokens(); err == nil { + return identity + "\norg:" + tokens.OrgID + } + return identity } func loadAuditLogsDownloadState(statePath, outPath, fingerprint string, force bool) (auditLogsDownloadState, bool, error) { @@ -355,6 +368,7 @@ func openAuditLogsDownloadOutput(outPath string, committed int64) (*os.File, err func runAuditLogsDownload(cmd *cobra.Command, args []string) error { c := getAuditLogsHandler(cmd) + c.downloadIdentity = currentAuditLogsDownloadIdentity() start, _ := cmd.Flags().GetString("start") end, _ := cmd.Flags().GetString("end") search, _ := cmd.Flags().GetString("search") diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index 82c3c473..1f1707b7 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -153,6 +153,28 @@ func TestAuditLogsDownloadRejectsChangedParamsOnResume(t *testing.T) { require.ErrorContains(t, err, "does not match this download") } +func TestAuditLogsDownloadFingerprintIncludesIdentity(t *testing.T) { + params, err := buildAuditLogsDownloadParams(auditLogsDownloadInput("")) + require.NoError(t, err) + params.Format = kernel.AuditLogExportChunkParamsFormatJSONLGz + + first, err := auditLogsDownloadFingerprint(params, "https://api.onkernel.com\norg:first") + require.NoError(t, err) + second, err := auditLogsDownloadFingerprint(params, "https://api.onkernel.com\norg:second") + require.NoError(t, err) + + assert.NotEqual(t, first, second) + assert.Len(t, first, 64) +} + +func TestDefaultAuditLogsDownloadPathIncludesFingerprint(t *testing.T) { + start := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2026, 6, 28, 0, 0, 0, 0, time.UTC) + path := defaultAuditLogsDownloadPath(start, end, "jsonl.gz", "1234567890abcdef") + + assert.Equal(t, "audit-logs-20260601T000000Z-20260628T000000Z-12345678.jsonl.gz", path) +} + func TestAuditLogsDownloadRejectsBadChunkBeforeWriting(t *testing.T) { capturePtermOutput(t) outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") From 678009bcfbfb95868e8127268196185a6251e7da Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:46:08 +0000 Subject: [PATCH 08/17] Require compressed audit log downloads --- cmd/audit_logs_download.go | 27 ++++++++------------------- cmd/audit_logs_download_test.go | 12 ++---------- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index f8cb11c1..8e67e10e 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -32,7 +32,6 @@ type AuditLogsDownloadInput struct { AuthStrategy string UserIDs []string To string - Format string Force bool } @@ -57,22 +56,13 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e return err } - format := in.Format - if format == "" { - format = string(kernel.AuditLogExportChunkParamsFormatJSONLGz) - } - if format != string(kernel.AuditLogExportChunkParamsFormatJSONL) && format != string(kernel.AuditLogExportChunkParamsFormatJSONLGz) { - return fmt.Errorf("--format must be jsonl or jsonl.gz") - } - params.Format = kernel.AuditLogExportChunkParamsFormat(format) - fingerprint, err := auditLogsDownloadFingerprint(params, c.downloadIdentity) if err != nil { return err } outPath := in.To if outPath == "" { - outPath = defaultAuditLogsDownloadPath(params.Start, params.End, format, fingerprint) + outPath = defaultAuditLogsDownloadPath(params.Start, params.End, fingerprint) } statePath := outPath + ".state.json" state, exists, err := loadAuditLogsDownloadState(statePath, outPath, fingerprint, in.Force) @@ -208,6 +198,7 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp params.Start = start params.End = end + params.Format = kernel.AuditLogExportChunkParamsFormatJSONLGz if in.Search != "" { params.Search = kernel.String(in.Search) } @@ -245,9 +236,9 @@ func auditLogsDownloadFingerprint(params kernel.AuditLogExportChunkParams, ident return hex.EncodeToString(sum[:]), nil } -func defaultAuditLogsDownloadPath(start, end time.Time, format, fingerprint string) string { +func defaultAuditLogsDownloadPath(start, end time.Time, fingerprint string) string { const stamp = "20060102T150405Z" - return fmt.Sprintf("audit-logs-%s-%s-%s.%s", start.UTC().Format(stamp), end.UTC().Format(stamp), fingerprint[:8], format) + return fmt.Sprintf("audit-logs-%s-%s-%s.jsonl.gz", start.UTC().Format(stamp), end.UTC().Format(stamp), fingerprint[:8]) } func currentAuditLogsDownloadIdentity() string { @@ -379,20 +370,19 @@ func runAuditLogsDownload(cmd *cobra.Command, args []string) error { authStrategy, _ := cmd.Flags().GetString("auth-strategy") userIDs, _ := cmd.Flags().GetStringArray("user-id") to, _ := cmd.Flags().GetString("to") - format, _ := cmd.Flags().GetString("format") force, _ := cmd.Flags().GetBool("force") return c.Download(cmd.Context(), AuditLogsDownloadInput{ Start: start, End: end, Search: search, Method: method, ExcludeMethod: excludeMethod, IncludeGet: includeGet, Service: service, - AuthStrategy: authStrategy, UserIDs: userIDs, To: to, Format: format, Force: force, + AuthStrategy: authStrategy, UserIDs: userIDs, To: to, Force: force, }) } var auditLogsDownloadCmd = &cobra.Command{ Use: "download", - Short: "Download audit logs for a time range to a file", - Long: "Download audit logs in verified, resumable chunks. The time range is [start, end).\n\n" + + Short: "Download audit logs as gzip-compressed JSONL", + Long: "Download audit logs as gzip-compressed JSONL in verified, resumable chunks. The time range is [start, end).\n\n" + "GET requests are excluded by default; pass --include-get to include them.", Args: cobra.NoArgs, RunE: runAuditLogsDownload, @@ -408,8 +398,7 @@ func init() { auditLogsDownloadCmd.Flags().String("service", "", "Filter by service") auditLogsDownloadCmd.Flags().String("auth-strategy", "", "Filter by authentication strategy") auditLogsDownloadCmd.Flags().StringArray("user-id", nil, "Filter by user ID (repeatable)") - auditLogsDownloadCmd.Flags().String("to", "", "Output file path") - auditLogsDownloadCmd.Flags().String("format", "jsonl.gz", "Output format: jsonl or jsonl.gz") + auditLogsDownloadCmd.Flags().String("to", "", "Output .jsonl.gz file path") auditLogsDownloadCmd.Flags().Bool("force", false, "Overwrite the output file and ignore saved progress") _ = auditLogsDownloadCmd.MarkFlagRequired("start") _ = auditLogsDownloadCmd.MarkFlagRequired("end") diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index 1f1707b7..da6f47f8 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -83,7 +83,7 @@ func readAuditLogGzip(t *testing.T, path string) string { } func auditLogsDownloadInput(path string) AuditLogsDownloadInput { - return AuditLogsDownloadInput{Start: "2026-06-01", End: "2026-06-28", To: path, Format: "jsonl.gz"} + return AuditLogsDownloadInput{Start: "2026-06-01", End: "2026-06-28", To: path} } func TestAuditLogsDownloadWritesAllChunks(t *testing.T) { @@ -156,7 +156,6 @@ func TestAuditLogsDownloadRejectsChangedParamsOnResume(t *testing.T) { func TestAuditLogsDownloadFingerprintIncludesIdentity(t *testing.T) { params, err := buildAuditLogsDownloadParams(auditLogsDownloadInput("")) require.NoError(t, err) - params.Format = kernel.AuditLogExportChunkParamsFormatJSONLGz first, err := auditLogsDownloadFingerprint(params, "https://api.onkernel.com\norg:first") require.NoError(t, err) @@ -170,7 +169,7 @@ func TestAuditLogsDownloadFingerprintIncludesIdentity(t *testing.T) { func TestDefaultAuditLogsDownloadPathIncludesFingerprint(t *testing.T) { start := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) end := time.Date(2026, 6, 28, 0, 0, 0, 0, time.UTC) - path := defaultAuditLogsDownloadPath(start, end, "jsonl.gz", "1234567890abcdef") + path := defaultAuditLogsDownloadPath(start, end, "1234567890abcdef") assert.Equal(t, "audit-logs-20260601T000000Z-20260628T000000Z-12345678.jsonl.gz", path) } @@ -265,10 +264,3 @@ func TestAuditLogsDownloadRejectsExistingOutput(t *testing.T) { err := (AuditLogsCmd{auditLogs: &FakeAuditLogsService{}}).Download(context.Background(), auditLogsDownloadInput(outPath)) require.ErrorContains(t, err, "already exists") } - -func TestAuditLogsDownloadRejectsInvalidFormat(t *testing.T) { - in := auditLogsDownloadInput(filepath.Join(t.TempDir(), "audit")) - in.Format = "csv" - err := (AuditLogsCmd{auditLogs: &FakeAuditLogsService{}}).Download(context.Background(), in) - require.ErrorContains(t, err, "--format") -} From 61b3b95901973f9453a267f08f71ec3ab953b58b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:50:56 +0000 Subject: [PATCH 09/17] Shorten audit log download filenames --- cmd/audit_logs_download.go | 2 +- cmd/audit_logs_download_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 8e67e10e..5db49250 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -237,7 +237,7 @@ func auditLogsDownloadFingerprint(params kernel.AuditLogExportChunkParams, ident } func defaultAuditLogsDownloadPath(start, end time.Time, fingerprint string) string { - const stamp = "20060102T150405Z" + const stamp = "20060102" return fmt.Sprintf("audit-logs-%s-%s-%s.jsonl.gz", start.UTC().Format(stamp), end.UTC().Format(stamp), fingerprint[:8]) } diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index da6f47f8..4e4ccf7e 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -171,7 +171,7 @@ func TestDefaultAuditLogsDownloadPathIncludesFingerprint(t *testing.T) { end := time.Date(2026, 6, 28, 0, 0, 0, 0, time.UTC) path := defaultAuditLogsDownloadPath(start, end, "1234567890abcdef") - assert.Equal(t, "audit-logs-20260601T000000Z-20260628T000000Z-12345678.jsonl.gz", path) + assert.Equal(t, "audit-logs-20260601-20260628-12345678.jsonl.gz", path) } func TestAuditLogsDownloadRejectsBadChunkBeforeWriting(t *testing.T) { From 5a843cea03b5bf6babf54feb3d2e1c14614b5a4f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:29:44 +0000 Subject: [PATCH 10/17] Publish completed audit log downloads --- cmd/audit_logs_download.go | 116 +++++++++++++++++++++++++++----- cmd/audit_logs_download_test.go | 58 +++++++++++++++- 2 files changed, 156 insertions(+), 18 deletions(-) diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 5db49250..d5677b53 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -64,8 +64,9 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if outPath == "" { outPath = defaultAuditLogsDownloadPath(params.Start, params.End, fingerprint) } + partialPath := outPath + ".partial" statePath := outPath + ".state.json" - state, exists, err := loadAuditLogsDownloadState(statePath, outPath, fingerprint, in.Force) + state, exists, err := loadAuditLogsDownloadState(statePath, partialPath, outPath, fingerprint, in.Force) if err != nil { return err } @@ -75,19 +76,27 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e } } - out, err := openAuditLogsDownloadOutput(outPath, state.BytesWritten) - if err != nil { - return err - } - defer out.Close() - if state.Chunks > 0 && state.Cursor == "" { + if err := commitAuditLogsDownloadOutput(partialPath, outPath, state.BytesWritten); err != nil { + return err + } if err := removeAuditLogsDownloadState(statePath); err != nil { return err } pterm.Success.Printf("Download already complete: %d rows (%s) in %s\n", state.Rows, util.FormatBytes(state.BytesWritten), outPath) return nil } + + out, err := openAuditLogsDownloadOutput(partialPath, state.BytesWritten) + if err != nil { + return err + } + defer func() { + if out != nil { + out.Close() + } + }() + if state.Chunks > 0 { pterm.Info.Printf("Resuming download at chunk %d (%d rows so far)\n", state.Chunks+1, state.Rows) } @@ -106,10 +115,10 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e } if _, err := out.Write(body); err != nil { - return fmt.Errorf("write %s: %w", outPath, err) + return fmt.Errorf("write %s: %w", partialPath, err) } if err := out.Sync(); err != nil { - return fmt.Errorf("sync %s: %w", outPath, err) + return fmt.Errorf("sync %s: %w", partialPath, err) } state.Cursor = nextCursor @@ -125,6 +134,14 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e } } + closeErr := out.Close() + out = nil + if closeErr != nil { + return fmt.Errorf("close %s: %w", partialPath, closeErr) + } + if err := commitAuditLogsDownloadOutput(partialPath, outPath, state.BytesWritten); err != nil { + return err + } if err := removeAuditLogsDownloadState(statePath); err != nil { return err } @@ -252,20 +269,36 @@ func currentAuditLogsDownloadIdentity() string { return identity } -func loadAuditLogsDownloadState(statePath, outPath, fingerprint string, force bool) (auditLogsDownloadState, bool, error) { +func loadAuditLogsDownloadState(statePath, partialPath, outPath, fingerprint string, force bool) (auditLogsDownloadState, bool, error) { fresh := auditLogsDownloadState{Version: auditLogsDownloadStateVersion, Params: fingerprint} if force { - if err := removeAuditLogsDownloadState(statePath); err != nil { + info, err := os.Lstat(outPath) + outExists := err == nil + if outExists { + if !info.Mode().IsRegular() { + return fresh, false, fmt.Errorf("%s is not a regular file", outPath) + } + } else if !os.IsNotExist(err) { + return fresh, false, fmt.Errorf("inspect %s: %w", outPath, err) + } + if err := resetAuditLogsDownload(partialPath, statePath); err != nil { return fresh, false, err } + if outExists { + if err := os.Rename(outPath, partialPath); err != nil { + return fresh, false, fmt.Errorf("prepare %s for overwrite: %w", outPath, err) + } + } return fresh, false, nil } raw, err := os.ReadFile(statePath) if os.IsNotExist(err) { - if _, statErr := os.Stat(outPath); statErr == nil { - return fresh, false, fmt.Errorf("%s already exists; pass --force to overwrite", outPath) - } else if !os.IsNotExist(statErr) { - return fresh, false, fmt.Errorf("inspect %s: %w", outPath, statErr) + for _, path := range []string{outPath, partialPath} { + if _, statErr := os.Stat(path); statErr == nil { + return fresh, false, fmt.Errorf("%s already exists; pass --force to overwrite", path) + } else if !os.IsNotExist(statErr) { + return fresh, false, fmt.Errorf("inspect %s: %w", path, statErr) + } } return fresh, false, nil } @@ -320,6 +353,27 @@ func removeAuditLogsDownloadState(statePath string) error { return nil } +func resetAuditLogsDownload(paths ...string) error { + for _, path := range paths { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + continue + } + if err != nil { + return fmt.Errorf("inspect %s: %w", path, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%s is not a regular file", path) + } + } + for _, path := range paths { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove %s: %w", path, err) + } + } + return nil +} + func openAuditLogsDownloadOutput(outPath string, committed int64) (*os.File, error) { if committed > 0 { if _, err := os.Stat(outPath); err != nil { @@ -357,6 +411,35 @@ func openAuditLogsDownloadOutput(outPath string, committed int64) (*os.File, err return out, nil } +func commitAuditLogsDownloadOutput(partialPath, outPath string, expected int64) error { + info, err := os.Stat(partialPath) + if os.IsNotExist(err) { + info, err = os.Stat(outPath) + if err != nil { + return fmt.Errorf("completed download is missing: %w", err) + } + if !info.Mode().IsRegular() || info.Size() != expected { + return fmt.Errorf("%s does not match completed download", outPath) + } + return nil + } + if err != nil { + return fmt.Errorf("inspect %s: %w", partialPath, err) + } + if !info.Mode().IsRegular() || info.Size() != expected { + return fmt.Errorf("%s does not match completed download", partialPath) + } + if _, err := os.Stat(outPath); err == nil { + return fmt.Errorf("%s already exists; move it or pass --force to start over", outPath) + } else if !os.IsNotExist(err) { + return fmt.Errorf("inspect %s: %w", outPath, err) + } + if err := os.Rename(partialPath, outPath); err != nil { + return fmt.Errorf("finalize %s: %w", outPath, err) + } + return nil +} + func runAuditLogsDownload(cmd *cobra.Command, args []string) error { c := getAuditLogsHandler(cmd) c.downloadIdentity = currentAuditLogsDownloadIdentity() @@ -383,7 +466,8 @@ var auditLogsDownloadCmd = &cobra.Command{ Use: "download", Short: "Download audit logs as gzip-compressed JSONL", Long: "Download audit logs as gzip-compressed JSONL in verified, resumable chunks. The time range is [start, end).\n\n" + - "GET requests are excluded by default; pass --include-get to include them.", + "GET requests are excluded by default; pass --include-get to include them.\n\n" + + "Incomplete downloads use an adjacent .partial file until finalization.", Args: cobra.NoArgs, RunE: runAuditLogsDownload, } diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index 4e4ccf7e..c0c2595e 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -106,6 +106,8 @@ func TestAuditLogsDownloadWritesAllChunks(t *testing.T) { assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", readAuditLogGzip(t, outPath)) _, err = os.Stat(outPath + ".state.json") assert.True(t, os.IsNotExist(err)) + _, err = os.Stat(outPath + ".partial") + assert.True(t, os.IsNotExist(err)) } func TestAuditLogsDownloadResumesFromCommittedChunk(t *testing.T) { @@ -119,8 +121,11 @@ func TestAuditLogsDownloadResumesFromCommittedChunk(t *testing.T) { func() (*http.Response, error) { return nil, errors.New("network error") }, ) require.Error(t, (AuditLogsCmd{auditLogs: service}).Download(context.Background(), auditLogsDownloadInput(outPath))) + _, err := os.Stat(outPath) + assert.True(t, os.IsNotExist(err)) - file, err := os.OpenFile(outPath, os.O_APPEND|os.O_WRONLY, 0) + partialPath := outPath + ".partial" + file, err := os.OpenFile(partialPath, os.O_APPEND|os.O_WRONLY, 0) require.NoError(t, err) _, err = file.WriteString("partial") require.NoError(t, err) @@ -133,6 +138,53 @@ func TestAuditLogsDownloadResumesFromCommittedChunk(t *testing.T) { require.NoError(t, (AuditLogsCmd{auditLogs: resumed}).Download(context.Background(), auditLogsDownloadInput(outPath))) assert.Equal(t, []string{"next"}, *cursors) assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", readAuditLogGzip(t, outPath)) + _, err = os.Stat(partialPath) + assert.True(t, os.IsNotExist(err)) +} + +func TestAuditLogsDownloadRecoversCompletedState(t *testing.T) { + for _, test := range []struct { + name string + renamed bool + }{ + {name: "before final rename"}, + {name: "after final rename", renamed: true}, + } { + t.Run(test.name, func(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + partialPath := outPath + ".partial" + statePath := outPath + ".state.json" + body := auditLogGzip(t, "{\"n\":1}\n") + require.NoError(t, os.WriteFile(partialPath, body, 0o600)) + if test.renamed { + require.NoError(t, os.Rename(partialPath, outPath)) + } + params, err := buildAuditLogsDownloadParams(auditLogsDownloadInput(outPath)) + require.NoError(t, err) + fingerprint, err := auditLogsDownloadFingerprint(params, "") + require.NoError(t, err) + require.NoError(t, saveAuditLogsDownloadState(statePath, auditLogsDownloadState{ + Version: auditLogsDownloadStateVersion, Params: fingerprint, + BytesWritten: int64(len(body)), Chunks: 1, Rows: 1, + })) + + require.NoError(t, (AuditLogsCmd{auditLogs: &FakeAuditLogsService{}}).Download(context.Background(), auditLogsDownloadInput(outPath))) + assert.Equal(t, "{\"n\":1}\n", readAuditLogGzip(t, outPath)) + _, err = os.Stat(partialPath) + assert.True(t, os.IsNotExist(err)) + _, err = os.Stat(statePath) + assert.True(t, os.IsNotExist(err)) + }) + } +} + +func TestAuditLogsDownloadRejectsCorruptState(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + require.NoError(t, os.WriteFile(outPath+".state.json", []byte("{"), 0o600)) + + err := (AuditLogsCmd{auditLogs: &FakeAuditLogsService{}}).Download(context.Background(), auditLogsDownloadInput(outPath)) + require.ErrorContains(t, err, "is corrupt") } func TestAuditLogsDownloadRejectsChangedParamsOnResume(t *testing.T) { @@ -183,7 +235,9 @@ func TestAuditLogsDownloadRejectsBadChunkBeforeWriting(t *testing.T) { err := (AuditLogsCmd{auditLogs: service}).Download(context.Background(), auditLogsDownloadInput(outPath)) require.ErrorContains(t, err, "checksum mismatch") - data, readErr := os.ReadFile(outPath) + _, statErr := os.Stat(outPath) + assert.True(t, os.IsNotExist(statErr)) + data, readErr := os.ReadFile(outPath + ".partial") require.NoError(t, readErr) assert.Empty(t, data) } From 1255a4eef22fb6592a69b9f570f571d0a7c901bd Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:14:23 +0000 Subject: [PATCH 11/17] Drop the audit log download chunk size cap The server controls chunk sizing and every chunk is checksum-verified, so the client-side byte ceiling only added a failure mode. Co-Authored-By: Claude Fable 5 --- cmd/audit_logs_download.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index d5677b53..9ff1c02a 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -36,9 +36,8 @@ type AuditLogsDownloadInput struct { } const ( - auditLogsDownloadMaxRange = 30 * 24 * time.Hour - auditLogsDownloadMaxChunkBytes = 256 << 20 - auditLogsDownloadStateVersion = 1 + auditLogsDownloadMaxRange = 30 * 24 * time.Hour + auditLogsDownloadStateVersion = 1 ) type auditLogsDownloadState struct { @@ -156,13 +155,10 @@ func (c AuditLogsCmd) fetchAuditLogsChunk(ctx context.Context, params kernel.Aud } defer res.Body.Close() - body, err := io.ReadAll(io.LimitReader(res.Body, auditLogsDownloadMaxChunkBytes+1)) + body, err := io.ReadAll(res.Body) if err != nil { return nil, nil, fmt.Errorf("read chunk body: %w", err) } - if len(body) > auditLogsDownloadMaxChunkBytes { - return nil, nil, fmt.Errorf("chunk response exceeds %s", util.FormatBytes(auditLogsDownloadMaxChunkBytes)) - } want := res.Header.Get("X-Content-Sha256") if want == "" { return nil, nil, fmt.Errorf("response missing X-Content-Sha256 header") From 5974b7c30d2b731180b4ff38de6633d6b8eede45 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:16:34 +0000 Subject: [PATCH 12/17] Drop the audit log download state version field The params fingerprint already invalidates stale state files, and the state is too short-lived to need format versioning. Co-Authored-By: Claude Fable 5 --- cmd/audit_logs_download.go | 10 +++------- cmd/audit_logs_download_test.go | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 9ff1c02a..0dfb6747 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -35,13 +35,9 @@ type AuditLogsDownloadInput struct { Force bool } -const ( - auditLogsDownloadMaxRange = 30 * 24 * time.Hour - auditLogsDownloadStateVersion = 1 -) +const auditLogsDownloadMaxRange = 30 * 24 * time.Hour type auditLogsDownloadState struct { - Version int `json:"version"` Params string `json:"params"` Cursor string `json:"cursor"` BytesWritten int64 `json:"bytes_written"` @@ -266,7 +262,7 @@ func currentAuditLogsDownloadIdentity() string { } func loadAuditLogsDownloadState(statePath, partialPath, outPath, fingerprint string, force bool) (auditLogsDownloadState, bool, error) { - fresh := auditLogsDownloadState{Version: auditLogsDownloadStateVersion, Params: fingerprint} + fresh := auditLogsDownloadState{Params: fingerprint} if force { info, err := os.Lstat(outPath) outExists := err == nil @@ -305,7 +301,7 @@ func loadAuditLogsDownloadState(statePath, partialPath, outPath, fingerprint str if err := json.Unmarshal(raw, &state); err != nil { return fresh, false, fmt.Errorf("state file %s is corrupt; pass --force to start over", statePath) } - if state.Version != auditLogsDownloadStateVersion || state.Params != fingerprint || state.BytesWritten < 0 { + if state.Params != fingerprint || state.BytesWritten < 0 { return fresh, false, fmt.Errorf("state file %s does not match this download; pass --force to start over", statePath) } return state, true, nil diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index c0c2595e..8dbb30b7 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -165,7 +165,7 @@ func TestAuditLogsDownloadRecoversCompletedState(t *testing.T) { fingerprint, err := auditLogsDownloadFingerprint(params, "") require.NoError(t, err) require.NoError(t, saveAuditLogsDownloadState(statePath, auditLogsDownloadState{ - Version: auditLogsDownloadStateVersion, Params: fingerprint, + Params: fingerprint, BytesWritten: int64(len(body)), Chunks: 1, Rows: 1, })) From bee9b4958516275c9d30776e778f7cb1c6ca6852 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:30:08 +0000 Subject: [PATCH 13/17] Remove durable audit log download state --- cmd/audit_logs.go | 3 +- cmd/audit_logs_download.go | 285 ++++++-------------------------- cmd/audit_logs_download_test.go | 115 +++---------- 3 files changed, 72 insertions(+), 331 deletions(-) diff --git a/cmd/audit_logs.go b/cmd/audit_logs.go index 9c6a6e7d..da60f527 100644 --- a/cmd/audit_logs.go +++ b/cmd/audit_logs.go @@ -22,8 +22,7 @@ type AuditLogsService interface { } type AuditLogsCmd struct { - auditLogs AuditLogsService - downloadIdentity string + auditLogs AuditLogsService } type AuditLogsSearchInput struct { diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 0dfb6747..d847e273 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" "net/http" @@ -14,7 +13,6 @@ import ( "strings" "time" - "github.com/kernel/cli/pkg/auth" "github.com/kernel/cli/pkg/util" "github.com/kernel/kernel-go-sdk" "github.com/pterm/pterm" @@ -37,74 +35,43 @@ type AuditLogsDownloadInput struct { const auditLogsDownloadMaxRange = 30 * 24 * time.Hour -type auditLogsDownloadState struct { - Params string `json:"params"` - Cursor string `json:"cursor"` - BytesWritten int64 `json:"bytes_written"` - Chunks int `json:"chunks"` - Rows int64 `json:"rows"` -} - func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) error { params, err := buildAuditLogsDownloadParams(in) if err != nil { return err } - fingerprint, err := auditLogsDownloadFingerprint(params, c.downloadIdentity) - if err != nil { - return err - } outPath := in.To if outPath == "" { - outPath = defaultAuditLogsDownloadPath(params.Start, params.End, fingerprint) + outPath = defaultAuditLogsDownloadPath(params.Start, params.End) } partialPath := outPath + ".partial" - statePath := outPath + ".state.json" - state, exists, err := loadAuditLogsDownloadState(statePath, partialPath, outPath, fingerprint, in.Force) - if err != nil { - return err - } - if !exists { - if err := saveAuditLogsDownloadState(statePath, state); err != nil { - return err - } - } - - if state.Chunks > 0 && state.Cursor == "" { - if err := commitAuditLogsDownloadOutput(partialPath, outPath, state.BytesWritten); err != nil { - return err - } - if err := removeAuditLogsDownloadState(statePath); err != nil { - return err - } - pterm.Success.Printf("Download already complete: %d rows (%s) in %s\n", state.Rows, util.FormatBytes(state.BytesWritten), outPath) - return nil - } - - out, err := openAuditLogsDownloadOutput(partialPath, state.BytesWritten) + out, err := openAuditLogsDownloadOutput(partialPath, outPath, in.Force) if err != nil { return err } + complete := false defer func() { if out != nil { out.Close() } + if !complete { + os.Remove(partialPath) + } }() - if state.Chunks > 0 { - pterm.Info.Printf("Resuming download at chunk %d (%d rows so far)\n", state.Chunks+1, state.Rows) - } - + var cursor string + var bytesWritten, rows int64 + chunks := 0 for { - if state.Cursor != "" { - params.Cursor = kernel.String(state.Cursor) + if cursor != "" { + params.Cursor = kernel.String(cursor) } body, header, err := c.fetchAuditLogsChunk(ctx, params) if err != nil { return err } - rows, nextCursor, hasMore, err := parseAuditLogsChunkHeaders(header, state.Cursor) + chunkRows, nextCursor, hasMore, err := parseAuditLogsChunkHeaders(header, cursor) if err != nil { return err } @@ -112,35 +79,29 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if _, err := out.Write(body); err != nil { return fmt.Errorf("write %s: %w", partialPath, err) } - if err := out.Sync(); err != nil { - return fmt.Errorf("sync %s: %w", partialPath, err) - } - - state.Cursor = nextCursor - state.BytesWritten += int64(len(body)) - state.Chunks++ - state.Rows += rows - if err := saveAuditLogsDownloadState(statePath, state); err != nil { - return err - } - pterm.Info.Printf("Chunk %d: %d rows (%d total, %s)\n", state.Chunks, rows, state.Rows, util.FormatBytes(state.BytesWritten)) + cursor = nextCursor + bytesWritten += int64(len(body)) + chunks++ + rows += chunkRows + pterm.Info.Printf("Chunk %d: %d rows (%d total, %s)\n", chunks, chunkRows, rows, util.FormatBytes(bytesWritten)) if !hasMore { break } } + if err := out.Sync(); err != nil { + return fmt.Errorf("sync %s: %w", partialPath, err) + } closeErr := out.Close() out = nil if closeErr != nil { return fmt.Errorf("close %s: %w", partialPath, closeErr) } - if err := commitAuditLogsDownloadOutput(partialPath, outPath, state.BytesWritten); err != nil { + if err := commitAuditLogsDownloadOutput(partialPath, outPath, in.Force); err != nil { return err } - if err := removeAuditLogsDownloadState(statePath); err != nil { - return err - } - pterm.Success.Printf("Downloaded %d rows (%s) to %s\n", state.Rows, util.FormatBytes(state.BytesWritten), outPath) + complete = true + pterm.Success.Printf("Downloaded %d rows (%s) to %s\n", rows, util.FormatBytes(bytesWritten), outPath) return nil } @@ -236,193 +197,52 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp return params, nil } -func auditLogsDownloadFingerprint(params kernel.AuditLogExportChunkParams, identity string) (string, error) { - query, err := params.URLQuery() - if err != nil { - return "", fmt.Errorf("fingerprint params: %w", err) - } - sum := sha256.Sum256([]byte(query.Encode() + "\n" + identity)) - return hex.EncodeToString(sum[:]), nil -} - -func defaultAuditLogsDownloadPath(start, end time.Time, fingerprint string) string { +func defaultAuditLogsDownloadPath(start, end time.Time) string { const stamp = "20060102" - return fmt.Sprintf("audit-logs-%s-%s-%s.jsonl.gz", start.UTC().Format(stamp), end.UTC().Format(stamp), fingerprint[:8]) -} - -func currentAuditLogsDownloadIdentity() string { - identity := strings.TrimRight(strings.TrimSpace(util.GetBaseURL()), "/") - if apiKey := os.Getenv("KERNEL_API_KEY"); apiKey != "" { - return identity + "\napi-key:" + apiKey - } - if tokens, err := auth.LoadTokens(); err == nil { - return identity + "\norg:" + tokens.OrgID - } - return identity -} - -func loadAuditLogsDownloadState(statePath, partialPath, outPath, fingerprint string, force bool) (auditLogsDownloadState, bool, error) { - fresh := auditLogsDownloadState{Params: fingerprint} - if force { - info, err := os.Lstat(outPath) - outExists := err == nil - if outExists { - if !info.Mode().IsRegular() { - return fresh, false, fmt.Errorf("%s is not a regular file", outPath) - } - } else if !os.IsNotExist(err) { - return fresh, false, fmt.Errorf("inspect %s: %w", outPath, err) - } - if err := resetAuditLogsDownload(partialPath, statePath); err != nil { - return fresh, false, err - } - if outExists { - if err := os.Rename(outPath, partialPath); err != nil { - return fresh, false, fmt.Errorf("prepare %s for overwrite: %w", outPath, err) - } - } - return fresh, false, nil - } - raw, err := os.ReadFile(statePath) - if os.IsNotExist(err) { - for _, path := range []string{outPath, partialPath} { - if _, statErr := os.Stat(path); statErr == nil { - return fresh, false, fmt.Errorf("%s already exists; pass --force to overwrite", path) - } else if !os.IsNotExist(statErr) { - return fresh, false, fmt.Errorf("inspect %s: %w", path, statErr) - } - } - return fresh, false, nil - } - if err != nil { - return fresh, false, fmt.Errorf("read state file: %w", err) - } - var state auditLogsDownloadState - if err := json.Unmarshal(raw, &state); err != nil { - return fresh, false, fmt.Errorf("state file %s is corrupt; pass --force to start over", statePath) - } - if state.Params != fingerprint || state.BytesWritten < 0 { - return fresh, false, fmt.Errorf("state file %s does not match this download; pass --force to start over", statePath) - } - return state, true, nil + return fmt.Sprintf("audit-logs-%s-%s.jsonl.gz", start.UTC().Format(stamp), end.UTC().Format(stamp)) } -func saveAuditLogsDownloadState(statePath string, state auditLogsDownloadState) error { - if err := os.MkdirAll(filepath.Dir(statePath), 0o700); err != nil { - return fmt.Errorf("create output directory: %w", err) - } - raw, err := json.Marshal(state) - if err != nil { - return fmt.Errorf("encode state: %w", err) - } - tmp, err := os.CreateTemp(filepath.Dir(statePath), filepath.Base(statePath)+".*") - if err != nil { - return fmt.Errorf("write state file: %w", err) - } - tmpPath := tmp.Name() - defer os.Remove(tmpPath) - if _, err := tmp.Write(raw); err != nil { - tmp.Close() - return fmt.Errorf("write state file: %w", err) - } - if err := tmp.Sync(); err != nil { - tmp.Close() - return fmt.Errorf("sync state file: %w", err) - } - if err := tmp.Close(); err != nil { - return fmt.Errorf("write state file: %w", err) - } - if err := os.Rename(tmpPath, statePath); err != nil { - return fmt.Errorf("commit state file: %w", err) - } - return nil -} - -func removeAuditLogsDownloadState(statePath string) error { - if err := os.Remove(statePath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("remove state file: %w", err) - } - return nil -} - -func resetAuditLogsDownload(paths ...string) error { - for _, path := range paths { - info, err := os.Lstat(path) - if os.IsNotExist(err) { - continue - } - if err != nil { - return fmt.Errorf("inspect %s: %w", path, err) - } +func openAuditLogsDownloadOutput(partialPath, outPath string, force bool) (*os.File, error) { + if info, err := os.Lstat(outPath); err == nil { if !info.Mode().IsRegular() { - return fmt.Errorf("%s is not a regular file", path) + return nil, fmt.Errorf("%s is not a regular file", outPath) } - } - for _, path := range paths { - if err := os.Remove(path); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("remove %s: %w", path, err) + if !force { + return nil, fmt.Errorf("%s already exists; pass --force to overwrite", outPath) } + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("inspect %s: %w", outPath, err) } - return nil -} - -func openAuditLogsDownloadOutput(outPath string, committed int64) (*os.File, error) { - if committed > 0 { - if _, err := os.Stat(outPath); err != nil { - return nil, fmt.Errorf("state records progress but %s is missing: %w", outPath, err) - } + if info, err := os.Lstat(partialPath); err == nil && !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s is not a regular file", partialPath) + } else if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("inspect %s: %w", partialPath, err) } - if err := os.MkdirAll(filepath.Dir(outPath), 0o700); err != nil { + if err := os.MkdirAll(filepath.Dir(partialPath), 0o700); err != nil { return nil, fmt.Errorf("create output directory: %w", err) } - out, err := os.OpenFile(outPath, os.O_RDWR|os.O_CREATE, 0o600) + out, err := os.OpenFile(partialPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { - return nil, fmt.Errorf("open %s: %w", outPath, err) - } - info, err := out.Stat() - if err != nil { - out.Close() - return nil, fmt.Errorf("inspect %s: %w", outPath, err) - } - if !info.Mode().IsRegular() || info.Size() < committed { - out.Close() - return nil, fmt.Errorf("%s does not match saved progress", outPath) + return nil, fmt.Errorf("open %s: %w", partialPath, err) } if err := out.Chmod(0o600); err != nil { out.Close() - return nil, fmt.Errorf("secure %s: %w", outPath, err) - } - if err := out.Truncate(committed); err != nil { - out.Close() - return nil, fmt.Errorf("truncate %s: %w", outPath, err) - } - if _, err := out.Seek(committed, io.SeekStart); err != nil { - out.Close() - return nil, fmt.Errorf("seek %s: %w", outPath, err) + return nil, fmt.Errorf("secure %s: %w", partialPath, err) } return out, nil } -func commitAuditLogsDownloadOutput(partialPath, outPath string, expected int64) error { - info, err := os.Stat(partialPath) - if os.IsNotExist(err) { - info, err = os.Stat(outPath) - if err != nil { - return fmt.Errorf("completed download is missing: %w", err) +func commitAuditLogsDownloadOutput(partialPath, outPath string, force bool) error { + if info, err := os.Lstat(outPath); err == nil { + if !info.Mode().IsRegular() { + return fmt.Errorf("%s is not a regular file", outPath) } - if !info.Mode().IsRegular() || info.Size() != expected { - return fmt.Errorf("%s does not match completed download", outPath) + if !force { + return fmt.Errorf("%s already exists; pass --force to overwrite", outPath) + } + if err := os.Remove(outPath); err != nil { + return fmt.Errorf("remove %s: %w", outPath, err) } - return nil - } - if err != nil { - return fmt.Errorf("inspect %s: %w", partialPath, err) - } - if !info.Mode().IsRegular() || info.Size() != expected { - return fmt.Errorf("%s does not match completed download", partialPath) - } - if _, err := os.Stat(outPath); err == nil { - return fmt.Errorf("%s already exists; move it or pass --force to start over", outPath) } else if !os.IsNotExist(err) { return fmt.Errorf("inspect %s: %w", outPath, err) } @@ -434,7 +254,6 @@ func commitAuditLogsDownloadOutput(partialPath, outPath string, expected int64) func runAuditLogsDownload(cmd *cobra.Command, args []string) error { c := getAuditLogsHandler(cmd) - c.downloadIdentity = currentAuditLogsDownloadIdentity() start, _ := cmd.Flags().GetString("start") end, _ := cmd.Flags().GetString("end") search, _ := cmd.Flags().GetString("search") @@ -457,9 +276,9 @@ func runAuditLogsDownload(cmd *cobra.Command, args []string) error { var auditLogsDownloadCmd = &cobra.Command{ Use: "download", Short: "Download audit logs as gzip-compressed JSONL", - Long: "Download audit logs as gzip-compressed JSONL in verified, resumable chunks. The time range is [start, end).\n\n" + + Long: "Download audit logs as gzip-compressed JSONL in verified chunks. The time range is [start, end).\n\n" + "GET requests are excluded by default; pass --include-get to include them.\n\n" + - "Incomplete downloads use an adjacent .partial file until finalization.", + "The output file is published only after every chunk is downloaded.", Args: cobra.NoArgs, RunE: runAuditLogsDownload, } @@ -475,7 +294,7 @@ func init() { auditLogsDownloadCmd.Flags().String("auth-strategy", "", "Filter by authentication strategy") auditLogsDownloadCmd.Flags().StringArray("user-id", nil, "Filter by user ID (repeatable)") auditLogsDownloadCmd.Flags().String("to", "", "Output .jsonl.gz file path") - auditLogsDownloadCmd.Flags().Bool("force", false, "Overwrite the output file and ignore saved progress") + auditLogsDownloadCmd.Flags().Bool("force", false, "Overwrite the output file") _ = auditLogsDownloadCmd.MarkFlagRequired("start") _ = auditLogsDownloadCmd.MarkFlagRequired("end") auditLogsCmd.AddCommand(auditLogsDownloadCmd) diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index 8dbb30b7..ad3eaed8 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -104,13 +104,11 @@ func TestAuditLogsDownloadWritesAllChunks(t *testing.T) { require.NoError(t, err) assert.Equal(t, []string{"", "next"}, *cursors) assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", readAuditLogGzip(t, outPath)) - _, err = os.Stat(outPath + ".state.json") - assert.True(t, os.IsNotExist(err)) _, err = os.Stat(outPath + ".partial") assert.True(t, os.IsNotExist(err)) } -func TestAuditLogsDownloadResumesFromCommittedChunk(t *testing.T) { +func TestAuditLogsDownloadRestartsAfterFailure(t *testing.T) { capturePtermOutput(t) outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") first := auditLogGzip(t, "{\"n\":1}\n") @@ -123,107 +121,33 @@ func TestAuditLogsDownloadResumesFromCommittedChunk(t *testing.T) { require.Error(t, (AuditLogsCmd{auditLogs: service}).Download(context.Background(), auditLogsDownloadInput(outPath))) _, err := os.Stat(outPath) assert.True(t, os.IsNotExist(err)) - partialPath := outPath + ".partial" - file, err := os.OpenFile(partialPath, os.O_APPEND|os.O_WRONLY, 0) - require.NoError(t, err) - _, err = file.WriteString("partial") - require.NoError(t, err) - require.NoError(t, file.Close()) - - second := auditLogGzip(t, "{\"n\":2}\n") - resumed, cursors := auditLogChunkService(t, func() (*http.Response, error) { - return auditLogChunkResponse(auditLogTestChunk{body: second, rows: 1}), nil - }) - require.NoError(t, (AuditLogsCmd{auditLogs: resumed}).Download(context.Background(), auditLogsDownloadInput(outPath))) - assert.Equal(t, []string{"next"}, *cursors) - assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", readAuditLogGzip(t, outPath)) _, err = os.Stat(partialPath) assert.True(t, os.IsNotExist(err)) -} - -func TestAuditLogsDownloadRecoversCompletedState(t *testing.T) { - for _, test := range []struct { - name string - renamed bool - }{ - {name: "before final rename"}, - {name: "after final rename", renamed: true}, - } { - t.Run(test.name, func(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") - partialPath := outPath + ".partial" - statePath := outPath + ".state.json" - body := auditLogGzip(t, "{\"n\":1}\n") - require.NoError(t, os.WriteFile(partialPath, body, 0o600)) - if test.renamed { - require.NoError(t, os.Rename(partialPath, outPath)) - } - params, err := buildAuditLogsDownloadParams(auditLogsDownloadInput(outPath)) - require.NoError(t, err) - fingerprint, err := auditLogsDownloadFingerprint(params, "") - require.NoError(t, err) - require.NoError(t, saveAuditLogsDownloadState(statePath, auditLogsDownloadState{ - Params: fingerprint, - BytesWritten: int64(len(body)), Chunks: 1, Rows: 1, - })) - - require.NoError(t, (AuditLogsCmd{auditLogs: &FakeAuditLogsService{}}).Download(context.Background(), auditLogsDownloadInput(outPath))) - assert.Equal(t, "{\"n\":1}\n", readAuditLogGzip(t, outPath)) - _, err = os.Stat(partialPath) - assert.True(t, os.IsNotExist(err)) - _, err = os.Stat(statePath) - assert.True(t, os.IsNotExist(err)) - }) - } -} - -func TestAuditLogsDownloadRejectsCorruptState(t *testing.T) { - outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") - require.NoError(t, os.WriteFile(outPath+".state.json", []byte("{"), 0o600)) - - err := (AuditLogsCmd{auditLogs: &FakeAuditLogsService{}}).Download(context.Background(), auditLogsDownloadInput(outPath)) - require.ErrorContains(t, err, "is corrupt") -} + require.NoError(t, os.WriteFile(partialPath, []byte("stale"), 0o600)) -func TestAuditLogsDownloadRejectsChangedParamsOnResume(t *testing.T) { - capturePtermOutput(t) - outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") - chunk := auditLogGzip(t, "{\"n\":1}\n") - service, _ := auditLogChunkService(t, + second := auditLogGzip(t, "{\"n\":2}\n") + restarted, cursors := auditLogChunkService(t, func() (*http.Response, error) { - return auditLogChunkResponse(auditLogTestChunk{body: chunk, rows: 1, hasMore: true, nextCursor: "next"}), nil + return auditLogChunkResponse(auditLogTestChunk{body: first, rows: 1, hasMore: true, nextCursor: "next"}), nil + }, + func() (*http.Response, error) { + return auditLogChunkResponse(auditLogTestChunk{body: second, rows: 1}), nil }, - func() (*http.Response, error) { return nil, errors.New("network error") }, ) - in := auditLogsDownloadInput(outPath) - require.Error(t, (AuditLogsCmd{auditLogs: service}).Download(context.Background(), in)) - - in.Service = "api" - err := (AuditLogsCmd{auditLogs: service}).Download(context.Background(), in) - require.ErrorContains(t, err, "does not match this download") -} - -func TestAuditLogsDownloadFingerprintIncludesIdentity(t *testing.T) { - params, err := buildAuditLogsDownloadParams(auditLogsDownloadInput("")) - require.NoError(t, err) - - first, err := auditLogsDownloadFingerprint(params, "https://api.onkernel.com\norg:first") - require.NoError(t, err) - second, err := auditLogsDownloadFingerprint(params, "https://api.onkernel.com\norg:second") - require.NoError(t, err) - - assert.NotEqual(t, first, second) - assert.Len(t, first, 64) + require.NoError(t, (AuditLogsCmd{auditLogs: restarted}).Download(context.Background(), auditLogsDownloadInput(outPath))) + assert.Equal(t, []string{"", "next"}, *cursors) + assert.Equal(t, "{\"n\":1}\n{\"n\":2}\n", readAuditLogGzip(t, outPath)) + _, err = os.Stat(partialPath) + assert.True(t, os.IsNotExist(err)) } -func TestDefaultAuditLogsDownloadPathIncludesFingerprint(t *testing.T) { +func TestDefaultAuditLogsDownloadPath(t *testing.T) { start := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) end := time.Date(2026, 6, 28, 0, 0, 0, 0, time.UTC) - path := defaultAuditLogsDownloadPath(start, end, "1234567890abcdef") + path := defaultAuditLogsDownloadPath(start, end) - assert.Equal(t, "audit-logs-20260601-20260628-12345678.jsonl.gz", path) + assert.Equal(t, "audit-logs-20260601-20260628.jsonl.gz", path) } func TestAuditLogsDownloadRejectsBadChunkBeforeWriting(t *testing.T) { @@ -237,12 +161,11 @@ func TestAuditLogsDownloadRejectsBadChunkBeforeWriting(t *testing.T) { require.ErrorContains(t, err, "checksum mismatch") _, statErr := os.Stat(outPath) assert.True(t, os.IsNotExist(statErr)) - data, readErr := os.ReadFile(outPath + ".partial") - require.NoError(t, readErr) - assert.Empty(t, data) + _, statErr = os.Stat(outPath + ".partial") + assert.True(t, os.IsNotExist(statErr)) } -func TestAuditLogsDownloadForceRestarts(t *testing.T) { +func TestAuditLogsDownloadForceOverwrites(t *testing.T) { capturePtermOutput(t) outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") require.NoError(t, os.WriteFile(outPath, []byte("old"), 0o600)) From 801a40a7199aea7fc369572146c5a1b43c47bdba Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:31:20 +0000 Subject: [PATCH 14/17] Adapt download to audit log time parsing --- cmd/audit_logs_download.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index d847e273..5b81f1c0 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -151,11 +151,11 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp if in.Start == "" || in.End == "" { return params, fmt.Errorf("--start and --end are required") } - start, _, err := parseAuditLogTime(in.Start) + start, err := parseAuditLogTime(in.Start) if err != nil { return params, fmt.Errorf("--start: %w", err) } - end, _, err := parseAuditLogTime(in.End) + end, err := parseAuditLogTime(in.End) if err != nil { return params, fmt.Errorf("--end: %w", err) } From ee8ccd4717b4b64e8f61b20883fe55160e90c7be Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:39:59 +0000 Subject: [PATCH 15/17] Preserve completed audit log downloads --- cmd/audit_logs_download.go | 11 ++++------- cmd/audit_logs_download_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 5b81f1c0..6926d663 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -50,12 +50,12 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if err != nil { return err } - complete := false + cleanupPartial := true defer func() { if out != nil { out.Close() } - if !complete { + if cleanupPartial { os.Remove(partialPath) } }() @@ -97,10 +97,10 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if closeErr != nil { return fmt.Errorf("close %s: %w", partialPath, closeErr) } + cleanupPartial = false if err := commitAuditLogsDownloadOutput(partialPath, outPath, in.Force); err != nil { - return err + return fmt.Errorf("%w; completed download remains at %s", err, partialPath) } - complete = true pterm.Success.Printf("Downloaded %d rows (%s) to %s\n", rows, util.FormatBytes(bytesWritten), outPath) return nil } @@ -240,9 +240,6 @@ func commitAuditLogsDownloadOutput(partialPath, outPath string, force bool) erro if !force { return fmt.Errorf("%s already exists; pass --force to overwrite", outPath) } - if err := os.Remove(outPath); err != nil { - return fmt.Errorf("remove %s: %w", outPath, err) - } } else if !os.IsNotExist(err) { return fmt.Errorf("inspect %s: %w", outPath, err) } diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index ad3eaed8..a02c111e 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -181,6 +181,32 @@ func TestAuditLogsDownloadForceOverwrites(t *testing.T) { assert.Equal(t, "{\"n\":1}\n", readAuditLogGzip(t, outPath)) } +func TestAuditLogsDownloadPreservesCompletedPartialOnFinalizeFailure(t *testing.T) { + capturePtermOutput(t) + dir := t.TempDir() + outPath := filepath.Join(dir, "audit.jsonl.gz") + chunk := auditLogGzip(t, "{\"n\":1}\n") + service, _ := auditLogChunkService(t, func() (*http.Response, error) { + require.NoError(t, os.Mkdir(outPath, 0o700)) + return auditLogChunkResponse(auditLogTestChunk{body: chunk, rows: 1}), nil + }) + + err := (AuditLogsCmd{auditLogs: service}).Download(context.Background(), auditLogsDownloadInput(outPath)) + require.ErrorContains(t, err, "completed download remains") + assert.Equal(t, "{\"n\":1}\n", readAuditLogGzip(t, outPath+".partial")) +} + +func TestCommitAuditLogsDownloadOutputPreservesExistingFileOnFailure(t *testing.T) { + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + require.NoError(t, os.WriteFile(outPath, []byte("existing"), 0o600)) + + err := commitAuditLogsDownloadOutput(outPath+".partial", outPath, true) + require.Error(t, err) + data, readErr := os.ReadFile(outPath) + require.NoError(t, readErr) + assert.Equal(t, "existing", string(data)) +} + func TestBuildAuditLogsDownloadParams(t *testing.T) { in := auditLogsDownloadInput("") in.Search = "browser" From 52aa065348f59e2a591a1485a3e4b1560d3996ea Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:01:33 +0000 Subject: [PATCH 16/17] Match audit log download exclusions to search --- cmd/audit_logs.go | 22 ++++++----- cmd/audit_logs_download.go | 67 ++++++++++++++++++++++++++++----- cmd/audit_logs_download_test.go | 18 ++++++++- 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/cmd/audit_logs.go b/cmd/audit_logs.go index 22149263..37a177dd 100644 --- a/cmd/audit_logs.go +++ b/cmd/audit_logs.go @@ -74,16 +74,7 @@ func (c AuditLogsCmd) Search(ctx context.Context, in AuditLogsSearchInput) error if in.Method != "" { params.Method = kernel.String(in.Method) } - // The API accepts a single exclude_method. When the default GET exclusion - // stacks with a user-provided one, GET goes server-side (it drops the most - // rows) and the user's method is filtered client-side. - excludeGetByDefault := in.Method == "" && !in.IncludeGet - var clientExclude string - serverExclude := in.ExcludeMethod - if excludeGetByDefault && !strings.EqualFold(in.ExcludeMethod, "GET") { - clientExclude = in.ExcludeMethod - serverExclude = "GET" - } + serverExclude, clientExclude := auditLogExcludeMethods(in.Method, in.ExcludeMethod, in.IncludeGet) if serverExclude != "" { params.ExcludeMethod = kernel.String(serverExclude) } @@ -145,6 +136,17 @@ func (c AuditLogsCmd) Search(ctx context.Context, in AuditLogsSearchInput) error return nil } +// The API accepts one exclude_method. When the default GET exclusion stacks +// with another method, GET is filtered server-side and the other method is +// filtered by the caller. +func auditLogExcludeMethods(method, excludeMethod string, includeGet bool) (server, client string) { + server = excludeMethod + if method == "" && !includeGet && !strings.EqualFold(excludeMethod, "GET") { + return "GET", excludeMethod + } + return server, "" +} + // peekForMore reports whether entries matching the client-side exclusion // remain past the limit. The scan is capped at one page so a long run of // excluded entries can't trigger unbounded fetching; hitting the cap assumes diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 6926d663..5a84079e 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -1,9 +1,13 @@ package cmd import ( + "bufio" + "bytes" + "compress/gzip" "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "net/http" @@ -46,6 +50,7 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e outPath = defaultAuditLogsDownloadPath(params.Start, params.End) } partialPath := outPath + ".partial" + _, clientExclude := auditLogExcludeMethods(in.Method, in.ExcludeMethod, in.IncludeGet) out, err := openAuditLogsDownloadOutput(partialPath, outPath, in.Force) if err != nil { return err @@ -75,6 +80,12 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if err != nil { return err } + if clientExclude != "" { + body, chunkRows, err = filterAuditLogsChunk(body, clientExclude) + if err != nil { + return err + } + } if _, err := out.Write(body); err != nil { return fmt.Errorf("write %s: %w", partialPath, err) @@ -105,6 +116,50 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e return nil } +func filterAuditLogsChunk(body []byte, excludeMethod string) ([]byte, int64, error) { + compressed, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, 0, fmt.Errorf("decompress chunk: %w", err) + } + defer compressed.Close() + + var filtered bytes.Buffer + writer, err := gzip.NewWriterLevel(&filtered, gzip.BestSpeed) + if err != nil { + return nil, 0, fmt.Errorf("compress filtered chunk: %w", err) + } + + var rows int64 + reader := bufio.NewReader(compressed) + for { + line, readErr := reader.ReadBytes('\n') + if len(line) > 0 { + var entry struct { + Method string `json:"method"` + } + if err := json.Unmarshal(bytes.TrimSpace(line), &entry); err != nil { + return nil, 0, fmt.Errorf("decode audit log entry: %w", err) + } + if !strings.EqualFold(entry.Method, excludeMethod) { + if _, err := writer.Write(line); err != nil { + return nil, 0, fmt.Errorf("compress filtered chunk: %w", err) + } + rows++ + } + } + if readErr == io.EOF { + break + } + if readErr != nil { + return nil, 0, fmt.Errorf("decompress chunk: %w", readErr) + } + } + if err := writer.Close(); err != nil { + return nil, 0, fmt.Errorf("compress filtered chunk: %w", err) + } + return filtered.Bytes(), rows, nil +} + func (c AuditLogsCmd) fetchAuditLogsChunk(ctx context.Context, params kernel.AuditLogExportChunkParams) ([]byte, http.Header, error) { res, err := c.auditLogs.ExportChunk(ctx, params) if err != nil { @@ -175,15 +230,9 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp if in.Method != "" { params.Method = kernel.String(in.Method) } - excludeMethod := in.ExcludeMethod - if in.Method == "" && !in.IncludeGet { - if excludeMethod != "" && !strings.EqualFold(excludeMethod, "GET") { - return params, fmt.Errorf("add --include-get when excluding a method other than GET") - } - excludeMethod = "GET" - } - if excludeMethod != "" { - params.ExcludeMethod = kernel.String(excludeMethod) + serverExclude, _ := auditLogExcludeMethods(in.Method, in.ExcludeMethod, in.IncludeGet) + if serverExclude != "" { + params.ExcludeMethod = kernel.String(serverExclude) } if in.Service != "" { params.Service = kernel.String(in.Service) diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index a02c111e..2ac21ed8 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -108,6 +108,23 @@ func TestAuditLogsDownloadWritesAllChunks(t *testing.T) { assert.True(t, os.IsNotExist(err)) } +func TestAuditLogsDownloadExcludeMethodStacksWithDefaultGetExclusion(t *testing.T) { + capturePtermOutput(t) + outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") + body := auditLogGzip(t, "{\"method\":\"POST\",\"n\":1}\n{\"method\":\"DELETE\",\"n\":2}\n") + service := &FakeAuditLogsService{ + ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { + assert.Equal(t, "GET", query.ExcludeMethod.Value) + return auditLogChunkResponse(auditLogTestChunk{body: body, rows: 2}), nil + }, + } + in := auditLogsDownloadInput(outPath) + in.ExcludeMethod = "post" + + require.NoError(t, (AuditLogsCmd{auditLogs: service}).Download(context.Background(), in)) + assert.Equal(t, "{\"method\":\"DELETE\",\"n\":2}\n", readAuditLogGzip(t, outPath)) +} + func TestAuditLogsDownloadRestartsAfterFailure(t *testing.T) { capturePtermOutput(t) outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") @@ -231,7 +248,6 @@ func TestBuildAuditLogsDownloadParamsRejectsInvalidInput(t *testing.T) { {name: "missing bounds", in: AuditLogsDownloadInput{}, want: "--start and --end are required"}, {name: "reversed bounds", in: AuditLogsDownloadInput{Start: "2026-06-02", End: "2026-06-01"}, want: "--start must be before --end"}, {name: "range too large", in: AuditLogsDownloadInput{Start: "2026-05-01", End: "2026-06-01"}, want: "at most 30 days"}, - {name: "conflicting exclusion", in: AuditLogsDownloadInput{Start: "2026-06-01", End: "2026-06-02", ExcludeMethod: "POST"}, want: "--include-get"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { From d3b22e65d13849f7d8215bb448f62a7b3bd8fc81 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:05:27 +0000 Subject: [PATCH 17/17] Use SDK audit log method exclusions --- cmd/audit_logs.go | 43 ++++++----------------- cmd/audit_logs_download.go | 62 +-------------------------------- cmd/audit_logs_download_test.go | 8 ++--- cmd/audit_logs_test.go | 50 +++++--------------------- go.mod | 2 +- go.sum | 4 +-- 6 files changed, 26 insertions(+), 143 deletions(-) diff --git a/cmd/audit_logs.go b/cmd/audit_logs.go index 37a177dd..eb38674e 100644 --- a/cmd/audit_logs.go +++ b/cmd/audit_logs.go @@ -74,10 +74,7 @@ func (c AuditLogsCmd) Search(ctx context.Context, in AuditLogsSearchInput) error if in.Method != "" { params.Method = kernel.String(in.Method) } - serverExclude, clientExclude := auditLogExcludeMethods(in.Method, in.ExcludeMethod, in.IncludeGet) - if serverExclude != "" { - params.ExcludeMethod = kernel.String(serverExclude) - } + params.ExcludeMethod = auditLogExcludeMethods(in.Method, in.ExcludeMethod, in.IncludeGet) if in.Service != "" { params.Service = kernel.String(in.Service) } @@ -93,13 +90,9 @@ func (c AuditLogsCmd) Search(ctx context.Context, in AuditLogsSearchInput) error hasMore := false pager := c.auditLogs.ListAutoPaging(ctx, params) for pager.Next() { - entry := pager.Current() - if clientExclude != "" && strings.EqualFold(entry.Method, clientExclude) { - continue - } - entries = append(entries, entry) + entries = append(entries, pager.Current()) if len(entries) >= in.Limit { - hasMore = peekForMore(pager, clientExclude) + hasMore = pager.Next() break } } @@ -136,31 +129,15 @@ func (c AuditLogsCmd) Search(ctx context.Context, in AuditLogsSearchInput) error return nil } -// The API accepts one exclude_method. When the default GET exclusion stacks -// with another method, GET is filtered server-side and the other method is -// filtered by the caller. -func auditLogExcludeMethods(method, excludeMethod string, includeGet bool) (server, client string) { - server = excludeMethod - if method == "" && !includeGet && !strings.EqualFold(excludeMethod, "GET") { - return "GET", excludeMethod +func auditLogExcludeMethods(method, excludeMethod string, includeGet bool) []string { + var methods []string + if method == "" && !includeGet { + methods = append(methods, "GET") } - return server, "" -} - -// peekForMore reports whether entries matching the client-side exclusion -// remain past the limit. The scan is capped at one page so a long run of -// excluded entries can't trigger unbounded fetching; hitting the cap assumes -// more may match. -func peekForMore(pager *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry], clientExclude string) bool { - for peeked := 0; peeked < auditLogsMaxPageSize; peeked++ { - if !pager.Next() { - return false - } - if clientExclude == "" || !strings.EqualFold(pager.Current().Method, clientExclude) { - return true - } + if excludeMethod != "" && (len(methods) == 0 || !strings.EqualFold(excludeMethod, "GET")) { + methods = append(methods, excludeMethod) } - return true + return methods } func parseAuditLogTime(value string) (time.Time, error) { diff --git a/cmd/audit_logs_download.go b/cmd/audit_logs_download.go index 5a84079e..e02a2bd9 100644 --- a/cmd/audit_logs_download.go +++ b/cmd/audit_logs_download.go @@ -1,20 +1,15 @@ package cmd import ( - "bufio" - "bytes" - "compress/gzip" "context" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" "net/http" "os" "path/filepath" "strconv" - "strings" "time" "github.com/kernel/cli/pkg/util" @@ -50,7 +45,6 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e outPath = defaultAuditLogsDownloadPath(params.Start, params.End) } partialPath := outPath + ".partial" - _, clientExclude := auditLogExcludeMethods(in.Method, in.ExcludeMethod, in.IncludeGet) out, err := openAuditLogsDownloadOutput(partialPath, outPath, in.Force) if err != nil { return err @@ -80,13 +74,6 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e if err != nil { return err } - if clientExclude != "" { - body, chunkRows, err = filterAuditLogsChunk(body, clientExclude) - if err != nil { - return err - } - } - if _, err := out.Write(body); err != nil { return fmt.Errorf("write %s: %w", partialPath, err) } @@ -116,50 +103,6 @@ func (c AuditLogsCmd) Download(ctx context.Context, in AuditLogsDownloadInput) e return nil } -func filterAuditLogsChunk(body []byte, excludeMethod string) ([]byte, int64, error) { - compressed, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - return nil, 0, fmt.Errorf("decompress chunk: %w", err) - } - defer compressed.Close() - - var filtered bytes.Buffer - writer, err := gzip.NewWriterLevel(&filtered, gzip.BestSpeed) - if err != nil { - return nil, 0, fmt.Errorf("compress filtered chunk: %w", err) - } - - var rows int64 - reader := bufio.NewReader(compressed) - for { - line, readErr := reader.ReadBytes('\n') - if len(line) > 0 { - var entry struct { - Method string `json:"method"` - } - if err := json.Unmarshal(bytes.TrimSpace(line), &entry); err != nil { - return nil, 0, fmt.Errorf("decode audit log entry: %w", err) - } - if !strings.EqualFold(entry.Method, excludeMethod) { - if _, err := writer.Write(line); err != nil { - return nil, 0, fmt.Errorf("compress filtered chunk: %w", err) - } - rows++ - } - } - if readErr == io.EOF { - break - } - if readErr != nil { - return nil, 0, fmt.Errorf("decompress chunk: %w", readErr) - } - } - if err := writer.Close(); err != nil { - return nil, 0, fmt.Errorf("compress filtered chunk: %w", err) - } - return filtered.Bytes(), rows, nil -} - func (c AuditLogsCmd) fetchAuditLogsChunk(ctx context.Context, params kernel.AuditLogExportChunkParams) ([]byte, http.Header, error) { res, err := c.auditLogs.ExportChunk(ctx, params) if err != nil { @@ -230,10 +173,7 @@ func buildAuditLogsDownloadParams(in AuditLogsDownloadInput) (kernel.AuditLogExp if in.Method != "" { params.Method = kernel.String(in.Method) } - serverExclude, _ := auditLogExcludeMethods(in.Method, in.ExcludeMethod, in.IncludeGet) - if serverExclude != "" { - params.ExcludeMethod = kernel.String(serverExclude) - } + params.ExcludeMethod = auditLogExcludeMethods(in.Method, in.ExcludeMethod, in.IncludeGet) if in.Service != "" { params.Service = kernel.String(in.Service) } diff --git a/cmd/audit_logs_download_test.go b/cmd/audit_logs_download_test.go index 2ac21ed8..0f4c72db 100644 --- a/cmd/audit_logs_download_test.go +++ b/cmd/audit_logs_download_test.go @@ -111,11 +111,11 @@ func TestAuditLogsDownloadWritesAllChunks(t *testing.T) { func TestAuditLogsDownloadExcludeMethodStacksWithDefaultGetExclusion(t *testing.T) { capturePtermOutput(t) outPath := filepath.Join(t.TempDir(), "audit.jsonl.gz") - body := auditLogGzip(t, "{\"method\":\"POST\",\"n\":1}\n{\"method\":\"DELETE\",\"n\":2}\n") + body := auditLogGzip(t, "{\"method\":\"DELETE\",\"n\":2}\n") service := &FakeAuditLogsService{ ExportChunkFunc: func(ctx context.Context, query kernel.AuditLogExportChunkParams, opts ...option.RequestOption) (*http.Response, error) { - assert.Equal(t, "GET", query.ExcludeMethod.Value) - return auditLogChunkResponse(auditLogTestChunk{body: body, rows: 2}), nil + assert.Equal(t, []string{"GET", "post"}, query.ExcludeMethod) + return auditLogChunkResponse(auditLogTestChunk{body: body, rows: 1}), nil }, } in := auditLogsDownloadInput(outPath) @@ -234,7 +234,7 @@ func TestBuildAuditLogsDownloadParams(t *testing.T) { assert.Equal(t, time.Date(2026, 6, 28, 0, 0, 0, 0, time.UTC), params.End) assert.Equal(t, "browser", params.Search.Value) - assert.Equal(t, "GET", params.ExcludeMethod.Value) + assert.Equal(t, []string{"GET"}, params.ExcludeMethod) assert.Equal(t, "api", params.Service.Value) assert.Equal(t, []string{"user_1"}, params.SearchUserID) } diff --git a/cmd/audit_logs_test.go b/cmd/audit_logs_test.go index 76ff6335..efc33f2d 100644 --- a/cmd/audit_logs_test.go +++ b/cmd/audit_logs_test.go @@ -60,7 +60,7 @@ func TestAuditLogsSearchBuildsParamsAndPrintsTable(t *testing.T) { assert.Equal(t, time.Date(2026, 7, 2, 15, 0, 0, 0, time.UTC), query.End) assert.Equal(t, "browsers", query.Search.Value) assert.Equal(t, "POST", query.Method.Value) - assert.Equal(t, "OPTIONS", query.ExcludeMethod.Value) + assert.Equal(t, []string{"OPTIONS"}, query.ExcludeMethod) assert.Equal(t, "api", query.Service.Value) assert.Equal(t, "api_key", query.AuthStrategy.Value) assert.Equal(t, []string{"user_123", "user_456"}, query.SearchUserID) @@ -168,7 +168,7 @@ func TestAuditLogsSearchExcludesGetByDefault(t *testing.T) { capturePtermOutput(t) fake := &FakeAuditLogsService{ ListAutoPagingFunc: func(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] { - assert.Equal(t, "GET", query.ExcludeMethod.Value) + assert.Equal(t, []string{"GET"}, query.ExcludeMethod) return auditLogPager() }, } @@ -182,7 +182,7 @@ func TestAuditLogsSearchIncludeGetDisablesDefaultExclusion(t *testing.T) { capturePtermOutput(t) fake := &FakeAuditLogsService{ ListAutoPagingFunc: func(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] { - assert.False(t, query.ExcludeMethod.Valid()) + assert.Empty(t, query.ExcludeMethod) return auditLogPager() }, } @@ -197,7 +197,7 @@ func TestAuditLogsSearchMethodDisablesDefaultExclusion(t *testing.T) { fake := &FakeAuditLogsService{ ListAutoPagingFunc: func(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] { assert.Equal(t, "GET", query.Method.Value) - assert.False(t, query.ExcludeMethod.Valid()) + assert.Empty(t, query.ExcludeMethod) return auditLogPager() }, } @@ -211,25 +211,22 @@ func TestAuditLogsSearchExcludeMethodStacksWithDefaultGetExclusion(t *testing.T) buf := capturePtermOutput(t) fake := &FakeAuditLogsService{ ListAutoPagingFunc: func(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] { - assert.Equal(t, "GET", query.ExcludeMethod.Value) - return auditLogPager(sampleAuditLogEntry("POST", "/posted"), sampleAuditLogEntry("DELETE", "/deleted")) + assert.Equal(t, []string{"GET", "post"}, query.ExcludeMethod) + return auditLogPager(sampleAuditLogEntry("DELETE", "/deleted")) }, } c := AuditLogsCmd{auditLogs: fake} err := c.Search(context.Background(), AuditLogsSearchInput{ExcludeMethod: "post", Limit: 100}) require.NoError(t, err) - - out := buf.String() - assert.Contains(t, out, "/deleted") - assert.NotContains(t, out, "/posted") + assert.Contains(t, buf.String(), "/deleted") } func TestAuditLogsSearchIncludeGetWithExcludeMethodSendsUserExclusion(t *testing.T) { capturePtermOutput(t) fake := &FakeAuditLogsService{ ListAutoPagingFunc: func(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] { - assert.Equal(t, "POST", query.ExcludeMethod.Value) + assert.Equal(t, []string{"POST"}, query.ExcludeMethod) return auditLogPager() }, } @@ -263,37 +260,6 @@ func TestAuditLogsSearchLimitTruncatesResults(t *testing.T) { assert.Contains(t, out, "Showing first 2 results") } -func TestAuditLogsSearchNoTruncationNoticeWhenOnlyExcludedEntriesRemain(t *testing.T) { - buf := capturePtermOutput(t) - fake := &FakeAuditLogsService{ - ListAutoPagingFunc: func(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] { - return auditLogPager(sampleAuditLogEntry("DELETE", "/deleted"), sampleAuditLogEntry("POST", "/posted"), sampleAuditLogEntry("POST", "/posted-too")) - }, - } - c := AuditLogsCmd{auditLogs: fake} - - err := c.Search(context.Background(), AuditLogsSearchInput{ExcludeMethod: "POST", Limit: 1}) - require.NoError(t, err) - - out := buf.String() - assert.Contains(t, out, "/deleted") - assert.NotContains(t, out, "Showing first") -} - -func TestAuditLogsSearchTruncationNoticeWhenMatchRemainsPastExcludedEntries(t *testing.T) { - buf := capturePtermOutput(t) - fake := &FakeAuditLogsService{ - ListAutoPagingFunc: func(ctx context.Context, query kernel.AuditLogListParams, opts ...option.RequestOption) *pagination.PageTokenPaginationAutoPager[kernel.AuditLogEntry] { - return auditLogPager(sampleAuditLogEntry("DELETE", "/first"), sampleAuditLogEntry("POST", "/posted"), sampleAuditLogEntry("DELETE", "/second")) - }, - } - c := AuditLogsCmd{auditLogs: fake} - - err := c.Search(context.Background(), AuditLogsSearchInput{ExcludeMethod: "POST", Limit: 1}) - require.NoError(t, err) - assert.Contains(t, buf.String(), "Showing first 1 results") -} - func TestAuditLogsSearchPrintsEmptyMessage(t *testing.T) { buf := capturePtermOutput(t) c := AuditLogsCmd{auditLogs: &FakeAuditLogsService{}} diff --git a/go.mod b/go.mod index b3939d33..96399c8d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss/v2 v2.0.0-beta.1 github.com/golang-jwt/jwt/v5 v5.2.2 github.com/joho/godotenv v1.5.1 - github.com/kernel/kernel-go-sdk v0.75.0 + github.com/kernel/kernel-go-sdk v0.77.0 github.com/klauspost/compress v1.18.5 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/pterm/pterm v0.12.80 diff --git a/go.sum b/go.sum index a182f786..d6b9e6a8 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/kernel/kernel-go-sdk v0.75.0 h1:UTlBLOVGQIFa0Vcn9DrTbhR44IQRrcZuhaKcMTwHvms= -github.com/kernel/kernel-go-sdk v0.75.0/go.mod h1:EeZzSuHZVeHKxKCPUzxou2bovNGhXaz0RXrSqKNf1AQ= +github.com/kernel/kernel-go-sdk v0.77.0 h1:c7MGZ1Uv7TQ4LvZ2E7jH8/IoHkoEhuwRPMLy9mKzIQI= +github.com/kernel/kernel-go-sdk v0.77.0/go.mod h1:EeZzSuHZVeHKxKCPUzxou2bovNGhXaz0RXrSqKNf1AQ= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=