Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 57 additions & 12 deletions packages/cli/src/lib/init/tools/list-dir.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
import fs from "node:fs";
import path from "node:path";
import { DEFAULT_SKIP_DIRS, normalizePath } from "../../scan/index.js";
import type { DirEntry, ListDirPayload, ToolResult } from "../types.js";
import type {
DirEntry,
ListDirPayload,
OmittedDirectoryReason,
ToolResult,
} from "../types.js";
import { safePath } from "./shared.js";
import type { InitToolDefinition } from "./types.js";

Expand Down Expand Up @@ -35,11 +40,22 @@ export async function listDir(payload: ListDirPayload): Promise<ToolResult> {
entries: [],
maxDepth,
maxEntries,
omittedDirectories: [],
recursive,
truncated: false,
};

await walkDirectory(targetPath, 0, state);
return { ok: true, data: { entries: state.entries } };
return {
ok: true,
data: {
entries: state.entries,
metadata: {
truncated: state.truncated,
omittedDirectories: state.omittedDirectories,
},
},
};
}

type WalkState = {
Expand All @@ -48,7 +64,12 @@ type WalkState = {
entries: DirEntry[];
maxDepth: number;
maxEntries: number;
omittedDirectories: Array<{
path: string;
reason: OmittedDirectoryReason;
}>;
recursive: boolean;
truncated: boolean;
};

async function readDirEntries(dir: string): Promise<fs.Dirent[]> {
Expand All @@ -59,14 +80,27 @@ async function readDirEntries(dir: string): Promise<fs.Dirent[]> {
}
}

function shouldRecurseInto(entry: fs.Dirent, state: WalkState): boolean {
return (
state.recursive &&
entry.isDirectory() &&
!entry.isSymbolicLink() &&
!entry.name.startsWith(".") &&
!INIT_SKIP_DIRS.has(entry.name)
);
function omissionReason(
entry: fs.Dirent,
state: WalkState,
depth: number
): OmittedDirectoryReason | undefined {
if (!entry.isDirectory() || entry.isSymbolicLink()) {
return;
}
if (!state.recursive) {
return "non-recursive";
}
if (entry.name.startsWith(".")) {
return "hidden";
}
if (INIT_SKIP_DIRS.has(entry.name)) {
return "excluded";
}
if (depth >= state.maxDepth) {
return "max-depth";
}
return;
}

/**
Expand Down Expand Up @@ -105,20 +139,31 @@ async function walkDirectory(
depth: number,
state: WalkState
): Promise<void> {
if (depth > state.maxDepth || state.entries.length >= state.maxEntries) {
if (depth > state.maxDepth) {
return;
}
if (state.entries.length >= state.maxEntries) {
state.truncated = true;
return;
}

for (const entry of await readDirEntries(dir)) {
if (state.entries.length >= state.maxEntries) {
state.truncated = true;
return;
}
const nextEntry = toDirEntry(state, dir, entry);
if (!nextEntry) {
continue;
}
state.entries.push(nextEntry);
if (shouldRecurseInto(entry, state)) {
const omittedBecause = omissionReason(entry, state, depth);
if (omittedBecause) {
state.omittedDirectories.push({
path: nextEntry.path,
reason: omittedBecause,
});
} else if (entry.isDirectory() && !entry.isSymbolicLink()) {
await walkDirectory(dir + NATIVE_SEP + entry.name, depth + 1, state);
}
}
Expand Down
87 changes: 66 additions & 21 deletions packages/cli/src/lib/init/tools/read-files.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import fs from "node:fs";
import { MAX_FILE_BYTES } from "../constants.js";
import type { ReadFilesPayload, ToolResult } from "../types.js";
import type { FileReadResult, ReadFilesPayload, ToolResult } from "../types.js";
import { safePath } from "./shared.js";
import type { InitToolDefinition } from "./types.js";

Expand All @@ -15,47 +15,92 @@ export async function readFiles(
const maxBytes = payload.params.maxBytes ?? MAX_FILE_BYTES;
const results = await Promise.all(
payload.params.paths.map(async (filePath) => {
const content = await readSingleFile(payload.cwd, filePath, maxBytes);
return [filePath, content] as const;
const result = await readSingleFile(payload.cwd, filePath, maxBytes);
return [filePath, result] as const;
})
);

const files: Record<string, string | null> = {};
for (const [filePath, content] of results) {
files[filePath] = content;
const readResults: Record<string, FileReadResult> = {};
for (const [filePath, result] of results) {
files[filePath] = result.content;
readResults[filePath] = result.metadata;
}

return { ok: true, data: { files } };
return { ok: true, data: { files, readResults } };
}

type SingleFileRead = {
content: string | null;
metadata: FileReadResult;
};

async function readSingleFile(
cwd: string,
filePath: string,
maxBytes: number
): Promise<string | null> {
): Promise<SingleFileRead> {
try {
const absPath = safePath(cwd, filePath);
const stat = await fs.promises.stat(absPath);
// Guard against FIFOs / sockets / devices — both `readFile` and
// `open("r")` block indefinitely on a FIFO waiting for a writer.
// `stat` follows symlinks, so symlink → FIFO is caught too.
if (!stat.isFile()) {
return null;
}
if (stat.size <= maxBytes) {
return await fs.promises.readFile(absPath, "utf-8");
return {
content: null,
metadata: { status: "skipped", reason: "not-regular-file" },
};
}
const buffer =
stat.size <= maxBytes
? await fs.promises.readFile(absPath)
: await readFilePrefix(absPath, maxBytes);

const handle = await fs.promises.open(absPath, "r");
try {
const buffer = Buffer.alloc(maxBytes);
await handle.read(buffer, 0, maxBytes, 0);
return buffer.toString("utf-8");
} finally {
await handle.close();
}
} catch {
return null;
return {
content: buffer.toString("utf-8"),
metadata: {
status: stat.size > maxBytes ? "truncated" : "read",
bytesRead: buffer.byteLength,
totalBytes: stat.size,
},
};
} catch (error) {
return {
content: null,
metadata: { status: "error", reason: readFailureReason(error) },
};
}
}

function readFailureReason(error: unknown): FileReadResult["reason"] {
const code =
typeof error === "object" && error !== null && "code" in error
? String(error.code)
: undefined;
if (code === "ENOENT") {
return "not-found";
}
if (code === "EACCES" || code === "EPERM") {
return "permission-denied";
}
if (error instanceof Error && error.message.includes("outside project")) {
return "outside-project";
}
return "read-failed";
}

async function readFilePrefix(
absPath: string,
maxBytes: number
): Promise<Buffer> {
const handle = await fs.promises.open(absPath, "r");
try {
const buffer = Buffer.alloc(maxBytes);
const { bytesRead } = await handle.read(buffer, 0, maxBytes, 0);
return buffer.subarray(0, bytesRead);
} finally {
await handle.close();
}
}

Expand Down
35 changes: 35 additions & 0 deletions packages/cli/src/lib/init/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,41 @@ export type DirEntry = {
type: "file" | "directory";
};

export type OmittedDirectoryReason =
| "excluded"
| "hidden"
| "max-depth"
| "non-recursive";

export type OmittedDirectory = {
path: string;
reason: OmittedDirectoryReason;
};

export type DirectoryScanMetadata = {
/** True when maxEntries prevented the scanner from inspecting more entries. */
truncated: boolean;
/** Directories present in the listing whose contents were not inspected. */
omittedDirectories: OmittedDirectory[];
};

export type DirectoryScanResult = {
entries: DirEntry[];
metadata: DirectoryScanMetadata;
};

export type FileReadResult = {
status: "read" | "truncated" | "skipped" | "error";
bytesRead?: number;
totalBytes?: number;
reason?:
| "not-found"
| "not-regular-file"
| "outside-project"
| "permission-denied"
| "read-failed";
};

export type ExistingProjectData = {
orgSlug: string;
projectSlug: string;
Expand Down
9 changes: 8 additions & 1 deletion packages/cli/src/lib/init/wizard-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -950,10 +950,16 @@ export async function runWizard(initialOptions: WizardOptions): Promise<void> {
let run: Awaited<ReturnType<typeof workflow.createRun>>;
let result: WorkflowRunResult;
try {
const [dirListing, existingSentry] = await Promise.all([
const [directoryScan, existingSentry] = await Promise.all([
precomputeDirListing(directory),
precomputeSentryDetection(directory).catch(() => null),
]);
const { entries: dirListing, metadata: dirListingMetadata } = directoryScan;
setTag("wizard.scan.truncated", dirListingMetadata.truncated);
setTag(
"wizard.scan.omitted_directory_count",
dirListingMetadata.omittedDirectories.length
);
const fileCache = await preReadCommonFiles(directory, dirListing);
ui.setIntroMode?.(false);
spin.message("Connecting to wizard...");
Expand Down Expand Up @@ -981,6 +987,7 @@ export async function runWizard(initialOptions: WizardOptions): Promise<void> {
},
initialState: {
dirListing,
dirListingMetadata,
fileCache,
existingSentry: existingSentry?.data,
knownPlatform: context.existingProject?.platform,
Expand Down
13 changes: 10 additions & 3 deletions packages/cli/src/lib/init/workflow-inputs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { isLikelyBinary } from "../scan/index.js";
import { MAX_FILE_BYTES } from "./constants.js";
import { detectSentry } from "./tools/detect-sentry.js";
import { listDir } from "./tools/list-dir.js";
import type { DirEntry } from "./types.js";
import type { DirEntry, DirectoryScanResult } from "./types.js";

/**
* Common config files that multiple init steps frequently inspect.
Expand Down Expand Up @@ -87,14 +87,21 @@ const MAX_PREREAD_TOTAL_BYTES = 512 * 1024;
*/
export async function precomputeDirListing(
directory: string
): Promise<DirEntry[]> {
): Promise<DirectoryScanResult> {
const result = await listDir({
type: "tool",
operation: "list-dir",
cwd: directory,
params: { path: ".", recursive: true, maxDepth: 3, maxEntries: 500 },
});
return (result.data as { entries?: DirEntry[] } | undefined)?.entries ?? [];
const data = result.data as DirectoryScanResult | undefined;
return {
entries: data?.entries ?? [],
metadata: data?.metadata ?? {
truncated: false,
omittedDirectories: [],
},
};
}

/**
Expand Down
29 changes: 28 additions & 1 deletion packages/cli/test/lib/init/tools/filesystem-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ describe("filesystem tools", () => {

expect(result.ok).toBe(true);
expect(entries.map((entry) => entry.path)).toContain("src/app.ts");
expect(precomputed.map((entry) => entry.path)).toContain("src/app.ts");
expect(precomputed.entries.map((entry) => entry.path)).toContain(
"src/app.ts"
);
});

test("reads files and checks existence in batches", async () => {
Expand All @@ -99,6 +101,10 @@ describe("filesystem tools", () => {

expect((readResult.data as any).files["exists.txt"]).toBe("hello");
expect((readResult.data as any).files["missing.txt"]).toBeNull();
expect((readResult.data as any).readResults).toEqual({
"exists.txt": { status: "read", bytesRead: 5, totalBytes: 5 },
"missing.txt": { status: "error", reason: "not-found" },
});
expect((existsResult.data as any).exists["exists.txt"]).toBe(true);
expect((existsResult.data as any).exists["missing.txt"]).toBe(false);
});
Expand All @@ -123,6 +129,27 @@ describe("filesystem tools", () => {
);
});

test("reports when a file read is truncated", async () => {
fs.writeFileSync(path.join(testDir, "large.txt"), "1234567890");

const result = await executeTool(
{
type: "tool",
operation: "read-files",
cwd: testDir,
params: { paths: ["large.txt"], maxBytes: 4 },
},
makeContext(testDir)
);

expect((result.data as any).files["large.txt"]).toBe("1234");
expect((result.data as any).readResults["large.txt"]).toEqual({
status: "truncated",
bytesRead: 4,
totalBytes: 10,
});
});

test("applies patchsets and injects auth tokens into env files", async () => {
const result = await executeTool(
{
Expand Down
Loading
Loading