diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e15cd32..a10a52e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,21 +26,16 @@ jobs: run: | set -e BIN=./target/release/bitvanes - # --no-tui with no input prints help (and exits successfully). - $BIN --no-tui | head -1 - # Markdown -> JSON / CSV / Arrow IPC - printf '# Hello\n\nThis is a test document.\n\nContact alice@example.com.\n' > /tmp/t.md - $BIN --no-tui -i /tmp/t.md -f markdown --scrub email -o /tmp/out.json - grep -q '"text"' /tmp/out.json - $BIN --no-tui -i /tmp/t.md -f markdown -o /tmp/out.csv - head -1 /tmp/out.csv | grep -q 'chunk_index' - $BIN --no-tui -i /tmp/t.md -f markdown -o /tmp/out.arrow - test -s /tmp/out.arrow - # JSON input - printf '{"name":"BitVanes","tags":["a","b"]}' > /tmp/t.json - $BIN --no-tui -i /tmp/t.json -o /tmp/outj.json - grep -q 'BitVanes' /tmp/outj.json - # Directory walk picks up all supported extensions - mkdir -p /tmp/docs && cp /tmp/t.md /tmp/docs/ && cp /tmp/t.json /tmp/docs/ - $BIN --no-tui -i /tmp/docs/ -o /tmp/dir.json - grep -q '"chunk_index"' /tmp/dir.json + $BIN --no-tui | head -1 # help when no input + mkdir -p /tmp/docs && printf '# Hello\n\nWorld.\n' > /tmp/docs/t.md + # markdown -> JSON / CSV / Arrow + $BIN --no-tui -i /tmp/docs/t.md -f markdown -o /tmp/o.json && grep -q text /tmp/o.json + $BIN --no-tui -i /tmp/docs/t.md -o /tmp/o.csv && grep -q chunk_index /tmp/o.csv + $BIN --no-tui -i /tmp/docs/t.md -o /tmp/o.arrow && test -s /tmp/o.arrow + # dedup keys in JSON + grep -q chunk_hash /tmp/o.json + # glob + $BIN --no-tui --glob "/tmp/docs/**/*.md" -o /tmp/g.json && grep -q chunk_index /tmp/g.json + # manifest idempotency + $BIN --no-tui -i /tmp/docs --manifest /tmp/m.json -o /tmp/r1.json + $BIN --no-tui -i /tmp/docs --manifest /tmp/m.json -o /tmp/r2.json 2>&1 | grep -q "Nothing new" diff --git a/AGENTS.md b/AGENTS.md index bebb607..97d23a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,8 @@ cat /tmp/out.json ## Architecture -Headless CLI that links `bitvanes-core` via git dependency (tag `v0.1.0`). +Headless CLI that links `bitvanes-core` via path dependency (local dev) or +git tag `v0.3.0` (release). Entry point: `src/main.rs` (clap arg parsing → dispatch to headless or TUI). Headless: `src/headless.rs` (directory scan → rayon parallel → output). @@ -34,7 +35,8 @@ Output formats: Arrow IPC, CSV, JSON (all from `bitvanes-core`'s `arrow_io`). ## Dependency on core ```toml -bitvanes-core = { git = "https://github.com/BitVanes/core.git", tag = "v0.1.1", features = ["ipc", "csv", "cli-pdf", "parallel"] } +bitvanes-core = { path = "../core/crates/core", features = ["ipc", "csv", "cli-pdf", "parallel"] } +# For release: git = "https://github.com/BitVanes/core.git", tag = "v0.3.0" ``` The CLI is distributed as a prebuilt binary via GitHub Releases (not published @@ -42,11 +44,18 @@ to crates.io), so it links core via a git tag. Only `bitvanes-core` is published to crates.io. To bump the core version, update the tag here and in Cargo.toml. +## Features + +`default = []`. The `embed` cargo feature enables `--embed` (on-device +embeddings via ONNX Runtime). It is **off by default** because linking the +prebuilt `ort` static lib needs glibc ≥ 2.38, which the Linux release runner +(ubuntu-22.04) lacks; build locally with `cargo build --features embed`. + ## Release ```bash -git tag v0.1.0 -git push origin v0.1.0 +git tag v0.3.0 +git push origin v0.3.0 ``` The release workflow (.github/workflows/release.yml) builds binaries for: diff --git a/Cargo.lock b/Cargo.lock index 00a6dfd..92829de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,7 +25,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -38,6 +38,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -122,6 +123,27 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" + [[package]] name = "arrow" version = "59.0.0" @@ -199,7 +221,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "half", "lexical-core", @@ -324,18 +346,40 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit-set" version = "0.8.0" @@ -359,34 +403,59 @@ checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "bitvanes-cli" -version = "0.1.1" +version = "0.4.0" dependencies = [ "bitvanes-core", + "blake3", "clap", "crossterm", + "glob", "indicatif", "ratatui", "rayon", "serde", "serde_json", + "toml", "walkdir", ] [[package]] name = "bitvanes-core" -version = "0.1.1" -source = "git+https://github.com/BitVanes/core.git?tag=v0.1.1#450ea5f5cfae74266e33f8f730f6b29a37609cc0" +version = "0.4.0" dependencies = [ "arrow", + "blake3", + "calamine", + "memmap2", + "ndarray", + "ort", "pdf-extract", "pulldown-cmark", + "quick-xml", "rayon", "regex", + "rtf-parser", "scraper", "serde", "serde_json", "thiserror", "tiktoken-rs", + "tokenizers", + "zip 2.4.2", +] + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", ] [[package]] @@ -430,12 +499,35 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +[[package]] +name = "calamine" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975084f43060e56343ffba7f9731fa52a7dcf2e1cd8e2459fd4c6bf4a1bff59" +dependencies = [ + "atoi_simd", + "byteorder", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip 8.6.0", +] + [[package]] name = "cassowary" version = "0.3.0" @@ -543,6 +635,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -563,6 +664,21 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "console" version = "0.15.11" @@ -596,6 +712,22 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -611,6 +743,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -655,7 +796,7 @@ dependencies = [ "crossterm_winapi", "mio", "parking_lot", - "rustix", + "rustix 0.38.44", "signal-hook", "signal-hook-mio", "winapi", @@ -730,6 +871,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "daachorse" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2" + [[package]] name = "darling" version = "0.20.11" @@ -765,6 +912,73 @@ dependencies = [ "syn", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + +[[package]] +name = "der" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -796,6 +1010,17 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dtoa" version = "1.0.11" @@ -863,6 +1088,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "euclid" version = "0.20.14" @@ -883,6 +1114,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fast-float2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + [[package]] name = "fastrand" version = "2.4.1" @@ -913,6 +1150,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -927,6 +1165,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "futures-core" version = "0.3.32" @@ -993,6 +1246,25 @@ dependencies = [ "wasip2", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "gloo-utils" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037fcb07216cb3a30f7292bd0176b050b7b9a052ba830ef7d5d65f6dc64ba58e" +dependencies = [ + "js-sys", + "serde", + "serde_json", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "half" version = "2.7.1" @@ -1028,6 +1300,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "html5ever" version = "0.39.0" @@ -1038,6 +1316,22 @@ dependencies = [ "markup5ever", ] +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1138,6 +1432,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1236,6 +1539,12 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "lock_api" version = "0.4.14" @@ -1268,7 +1577,7 @@ dependencies = [ "itoa", "log", "md-5", - "nom", + "nom 8.0.0", "nom_locate", "rand", "rangemap", @@ -1288,6 +1597,28 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + [[package]] name = "markup5ever" version = "0.39.0" @@ -1299,6 +1630,16 @@ dependencies = [ "web_atoms", ] +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "md-5" version = "0.10.6" @@ -1315,6 +1656,21 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1338,36 +1694,100 @@ dependencies = [ ] [[package]] -name = "new_debug_unreachable" -version = "1.0.6" +name = "monostate" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] [[package]] -name = "nom" -version = "8.0.0" +name = "monostate-impl" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ - "memchr", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "nom_locate" -version = "5.0.0" +name = "native-tls" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" dependencies = [ - "bytecount", - "memchr", - "nom", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", ] [[package]] -name = "num-bigint" -version = "0.4.6" +name = "ndarray" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom_locate" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" +dependencies = [ + "bytecount", + "memchr", + "nom 8.0.0", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -1419,6 +1839,73 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1465,6 +1952,21 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "phf" version = "0.13.1" @@ -1524,6 +2026,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "pom" version = "1.1.0" @@ -1536,6 +2044,15 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postscript" version = "0.14.1" @@ -1585,6 +2102,16 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quote" version = "1.0.46" @@ -1643,11 +2170,11 @@ checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ "bitflags", "cassowary", - "compact_str", + "compact_str 0.8.2", "crossterm", "indoc", "instability", - "itertools", + "itertools 0.13.0", "lru", "paste", "strum", @@ -1656,6 +2183,12 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -1666,6 +2199,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.13.0" @@ -1714,6 +2258,17 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rtf-parser" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3147b4eb521eae5e29b781bdc30ab98bbaed784bfcb8376cda60ba4b2e0e3d" +dependencies = [ + "serde", + "tsify", + "wasm-bindgen", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -1738,10 +2293,32 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.15", "windows-sys 0.59.0", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -1763,6 +2340,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1784,6 +2370,29 @@ dependencies = [ "tendril", ] +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "selectors" version = "0.38.0" @@ -1839,6 +2448,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_derive_internals" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e578a843d40b4189a4d66bba51d7684f57da5bd7c304c64e14bd63efbef49509" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_json" version = "1.0.150" @@ -1852,6 +2472,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "servo_arc" version = "0.4.3" @@ -1868,7 +2497,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1933,6 +2562,29 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2019,6 +2671,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "tendril" version = "0.5.0" @@ -2056,7 +2721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bstr", "fancy-regex", "lazy_static", @@ -2088,6 +2753,124 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112" +dependencies = [ + "ahash", + "compact_str 0.9.1", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex", + "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tsify" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b26cf145f2f3b9ff84e182c448eaf05468e247f148cf3d2a7d67d78ff023a0" +dependencies = [ + "gloo-utils", + "serde", + "serde_json", + "tsify-macros", + "wasm-bindgen", +] + +[[package]] +name = "tsify-macros" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a94b0f0954b3e59bfc2c246b4c8574390d94a4ad4ad246aaf2fb07d7dfd3b47" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + [[package]] name = "ttf-parser" version = "0.25.1" @@ -2103,6 +2886,12 @@ dependencies = [ "pom", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -2136,6 +2925,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" @@ -2154,7 +2952,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" dependencies = [ - "itertools", + "itertools 0.13.0", "unicode-segmentation", "unicode-width 0.1.14", ] @@ -2171,18 +2969,66 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "der", + "log", + "native-tls", + "percent-encoding", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", +] + [[package]] name = "utf-8" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2259,6 +3105,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" @@ -2281,6 +3137,15 @@ dependencies = [ "string_cache_codegen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "weezl" version = "0.1.12" @@ -2459,6 +3324,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.46.0" @@ -2485,8 +3359,63 @@ dependencies = [ "syn", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 991eef7..aa06683 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bitvanes-cli" -version = "0.1.1" +version = "0.4.0" edition = "2024" rust-version = "1.85" license = "MIT OR Apache-2.0" @@ -15,16 +15,26 @@ readme = "README.md" name = "bitvanes" path = "src/main.rs" +[features] +default = [] +# Enable `--embed` (on-device embeddings via ONNX Runtime). Off by default: +# linking the prebuilt `ort` static lib needs glibc >= 2.38, which the Linux +# release runner (ubuntu-22.04) lacks. Build with `--features embed` locally. +embed = ["bitvanes-core/embeddings"] + [dependencies] -bitvanes-core = { git = "https://github.com/BitVanes/core.git", tag = "v0.1.1", features = ["ipc", "csv", "cli-pdf", "parallel"] } +bitvanes-core = { path = "../core/crates/core", features = ["ipc", "csv", "cli-pdf", "parallel", "office", "mmap"] } +blake3 = "1" clap = { version = "4", features = ["derive"] } crossterm = "0.28" +glob = "0.3" +indicatif = "0.17" ratatui = "0.29" rayon = "1" -walkdir = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" -indicatif = "0.17" +toml = "0.8" +walkdir = "2" [profile.release] opt-level = 3 diff --git a/README.md b/README.md index 0282cb8..351a904 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,39 @@ Processing runs on a background thread, so the UI stays responsive on large files (a spinner shows progress). CLI flags (`--format`, `--max-tokens`, `--tokenizer`, `--scrub`, `--config`) are honoured when launching the TUI. +### Ingestion at scale + +The CLI is built for automated, team-scale ingestion: + +```bash +# Glob instead of a single path +bitvanes --no-tui --glob "docs/**/*.md" -o chunks.json + +# Idempotent incremental runs: skip files already in the manifest +bitvanes --no-tui -i ./ingest/ --manifest .bitvanes-manifest.json -o chunks.json + +# Hot-folder daemon: keep watching for new/changed files and process them +bitvanes --no-tui -i ./ingest/ --watch --manifest .bitvanes-manifest.json --poll-interval 10 + +# Cap parallelism +bitvanes --no-tui -i ./docs/ --jobs 4 -o chunks.json +``` + +JSON output carries dedup keys for downstream pipelines: `source_hash` +(blake3 of the source file) and `chunk_hash` (blake3 of the chunk text). + +### On-device embeddings (optional) + +Build with the `embed` feature to enable `--embed`, which fills the Arrow +`embedding` column with real vectors (requires glibc ≥ 2.38 to link ONNX +Runtime, so it is off in the prebuilt release binaries): + +```bash +cargo build --release --features embed +bitvanes --no-tui -i ./docs/ --embed model.onnx --embed-tokenizer tokenizer.json \ + --embed-dim 384 -o chunks.arrow +``` + ### Profile replay (from web app) Export a profile from the BitVanes web app, then replay it identically: @@ -103,6 +136,9 @@ Pipeline: -t, --tokenizer cl100k_base | o200k_base | r50k_base | ... -m, --max-tokens Max tokens per chunk --scrub Comma-separated PII patterns + --min-confidence Min confidence [0.0–1.0] for scrubbing (default: 0.0) + --anchor-window Contextual anchor half-window in words (default: 7) + --toml bitvanes.toml config (layered below CLI flags) Output: -o, --output .arrow | .csv | .json | - (stdout) @@ -125,19 +161,24 @@ Output: | HTML | `.html` | `scraper` (html5ever) | | JSON | `.json` | Structural (one chunk per object/leaf) | | PDF | `.pdf` | `pdf-extract` (native, text-layer only) | +| **DOCX** | `.docx` | ZIP + quick-xml (headings, tables, lists) | +| **PPTX** | `.pptx` | Slide-by-slide text extraction | +| **XLSX** | `.xlsx` | `calamine` (memory-guarded for large sheets) | +| **EPUB** | `.epub` | OPF spine + HtmlParser per chapter | +| **RTF** | `.rtf` | `rtf-parser` | Format is auto-detected from file extension. Override with `--format`. -Scanned/image-only PDFs have no extractable text layer and are reported -as invalid input (OCR is out of scope; the web app uses PDF.js). ## Features +- **10 document formats**: Markdown, Text, HTML, JSON, PDF, DOCX, PPTX, XLSX, EPUB, RTF +- **8 PII patterns**: email, SSN, phone, credit card (Luhn), routing number + (ABA), AWS keys, GitHub PATs, JWTs — with confidence scoring + anchor windows - **6 OpenAI tokenizers**: cl100k_base, o200k_base, r50k_base, p50k_base, p50k_edit, o200k_harmony (all embedded at compile time) -- **7 PII patterns**: email, SSN, phone, credit card (Luhn), AWS keys, - GitHub PATs, JWTs -- **Structural context**: heading ancestry preserved per chunk -- **Parallel processing**: rayon multi-core for directory batch processing +- **Parallel processing**: rayon multi-core batch + intra-doc parallel regex +- **Memory-mapped I/O**: zero-copy reads for files > 1 MB +- **Streaming output**: Arrow IPC to stdout for pipe-friendly ETL - **Profile replay**: byte-for-byte identical output to the web app ## Build from source @@ -150,8 +191,8 @@ cargo build --release ``` The CLI depends on [`bitvanes-core`](https://github.com/BitVanes/core) via a -git dependency (tag `v0.1.1`, with the `ipc`, `csv`, `cli-pdf`, and -`parallel` features). No manual checkout of the core repo needed. +git dependency (tag `v0.4.0`, with the `ipc`, `csv`, `cli-pdf`, `office`, +`mmap`, and `parallel` features). No manual checkout of the core repo needed. ## License diff --git a/src/headless.rs b/src/headless.rs index 926046f..853af51 100644 --- a/src/headless.rs +++ b/src/headless.rs @@ -1,16 +1,22 @@ -//! Headless processing: directory scanning, parallel chunking, output. +//! Headless processing: file collection (walkdir + glob), content hashing, +//! idempotent manifests, watch mode, parallel chunking, and output. +use std::collections::HashSet; use std::fs; use std::path::{Path, PathBuf}; +use std::time::Duration; -use indicatif::{ProgressBar, ProgressStyle}; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use rayon::prelude::*; -use serde::Serialize; +use serde::{Deserialize, Serialize}; +use std::io::Write; use walkdir::WalkDir; -use bitvanes_core::arrow_io::batch::chunks_to_batch; +use bitvanes_core::arrow_io::batch::{chunks_to_batch, chunks_to_batch_with_embeddings}; +use bitvanes_core::arrow_io::ipc::IpcStream; use bitvanes_core::chunk::chunk_document; use bitvanes_core::parse::parse_bytes; +use bitvanes_core::pipeline::attach_metadata; use bitvanes_core::scrub::scrub_document; use bitvanes_core::{ BuiltInPattern, ChunkSpec, DocumentFormat, PipelineConfig, ScrubProfile, TokenizerKind, @@ -18,116 +24,396 @@ use bitvanes_core::{ use crate::Cli; +const SUPPORTED_EXTS: &[&str] = &[ + "md", "markdown", "txt", "html", "htm", "pdf", "json", "docx", "pptx", "xlsx", "epub", "rtf", +]; + /// Entry point for headless mode. pub fn run(cli: &Cli) -> Result<(), Box> { - let input = cli - .input - .as_ref() - .ok_or("no input specified — use --input ")?; - - // 1. Build pipeline config from profile + CLI flags. let config = build_config(cli)?; eprintln!( - "Config: format={:?} tokenizer={:?} max_tokens={}", - config.format, config.chunk.tokenizer, config.chunk.max_tokens + "Config: format={:?} tokenizer={:?} max_tokens={} strategy={:?}", + config.format, config.chunk.tokenizer, config.chunk.max_tokens, config.chunk.strategy ); - // 2. Collect input files. - let files = collect_files(input, &config, cli)?; + let mut manifest = Manifest::load_or_default(cli.manifest.as_deref())?; + let embedder = build_embedder(cli)?; + + if cli.watch { + run_watch(cli, &config, &mut manifest, embedder.as_deref())?; + } else { + run_once(cli, &config, &mut manifest, embedder.as_deref(), true)?; + } + Ok(()) +} + +/// Processes a single batch of new/changed files and writes output. +/// Returns the number of files actually processed (post-manifest filter). +#[allow(clippy::too_many_arguments)] +fn run_once( + cli: &Cli, + config: &PipelineConfig, + manifest: &mut Manifest, + embedder: Option<&dyn bitvanes_core::Embedder>, + write_output: bool, +) -> Result> { + let files = collect_files(cli, config)?; if files.is_empty() { - return Err("no supported files found in the input path".into()); + eprintln!("No supported files found."); + return Ok(0); } - eprintln!("Found {} files", files.len()); - // 3. Process in parallel. - let pb = ProgressBar::new(files.len() as u64); - pb.set_style( - ProgressStyle::with_template("Processing {wide_bar} {pos}/{len}") + // Hash + manifest filter (sequential: cheap, and mutates the set). + let force = cli.force; + let mut pending: Vec<(PathBuf, String, u64)> = Vec::new(); + let mut skipped = 0usize; + for path in files { + let bytes = match fs::read(&path) { + Ok(b) => b, + Err(e) => { + eprintln!(" ⚠ could not read {}: {e}", path.display()); + continue; + } + }; + let hash = hex_hash(&bytes); + if !force && manifest.contains(&hash) { + skipped += 1; + continue; + } + let size = bytes.len() as u64; + pending.push((path, hash.clone(), size)); + manifest.insert(hash); + } + if skipped > 0 { + eprintln!("Skipped {skipped} file(s) already in manifest (--force to reprocess)."); + } + if pending.is_empty() { + if write_output { + eprintln!("Nothing new to process."); + } + return Ok(0); + } + eprintln!("Processing {} file(s)...", pending.len()); + + // Two-level progress: files + bytes. + let total_bytes: u64 = pending.iter().map(|(_, _, sz)| *sz).sum(); + let mp = MultiProgress::new(); + let pb_files = mp.add(ProgressBar::new(pending.len() as u64)); + let pb_bytes = mp.add(ProgressBar::new(total_bytes)); + pb_files.set_style( + ProgressStyle::with_template(" Files {wide_bar} {pos}/{len}") + .unwrap() + .progress_chars("█░"), + ); + pb_bytes.set_style( + ProgressStyle::with_template(" Bytes {wide_bar} {bytes}/{total_bytes} ({bytes_per_sec})") .unwrap() .progress_chars("█░"), ); - let all_chunks: Vec = files + // Process files in parallel, keeping per-file grouping for streaming output. + let per_file: Vec<(PathBuf, String, Vec)> = pending .par_iter() - .flat_map(|path| { - let chunks = process_file(path, &config); - pb.inc(1); - chunks + .map(|(path, source_hash, size)| { + let chunks = process_file(path, config); + pb_files.inc(1); + pb_bytes.inc(*size); + (path.clone(), source_hash.clone(), chunks) }) .collect(); + pb_files.finish_and_clear(); + pb_bytes.finish_and_clear(); + + // Persist the manifest now that we've committed to processing these files. + manifest.save(cli.manifest.as_deref())?; - pb.finish_and_clear(); + let total_chunks: usize = per_file.iter().map(|(_, _, c)| c.len()).sum(); + if total_chunks == 0 { + eprintln!("\nNo chunks generated. Check that inputs are valid UTF-8 text."); + return Ok(pending.len()); + } - // 4. Print stats. - let total_tokens: u64 = all_chunks.iter().map(|c| c.token_count as u64).sum(); - eprintln!(); - eprintln!("Results:"); - eprintln!(" Files processed: {}", files.len()); - eprintln!(" Chunks generated: {}", all_chunks.len()); - eprintln!(" Total tokens: {}", total_tokens); - if !all_chunks.is_empty() { + if write_output { + let output = cli.output.as_deref().unwrap_or("output.json"); + let ext = Path::new(output) + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("json"); + + if ext == "arrow" { + // Streaming: write one RecordBatch per file directly to disk/stdout. + write_streaming_arrow(&per_file, output, embedder)?; + } else { + // Buffered: JSON/CSV (less memory-intensive, single batch). + let processed: Vec = per_file + .iter() + .flat_map(|(_, hash, chunks)| { + chunks + .iter() + .map(move |spec| ProcessedChunk { + spec: spec.clone(), + source_hash: hash.clone(), + }) + .collect::>() + }) + .collect(); + write_output_inner(&processed, output, embedder)?; + } eprintln!( - " Avg tokens/chunk: {}", - total_tokens / all_chunks.len() as u64 + "\n{} chunk(s) from {} file(s) written to {output}", + total_chunks, + pending.len() ); } + Ok(pending.len()) +} + +/// Watch loop: repeatedly scan for new/changed files, process, persist. +fn run_watch( + cli: &Cli, + config: &PipelineConfig, + manifest: &mut Manifest, + embedder: Option<&dyn bitvanes_core::Embedder>, +) -> Result<(), Box> { + eprintln!( + "Watching for new files every {}s. Press Ctrl-C to stop.", + cli.poll_interval + ); + loop { + let processed = run_once(cli, config, manifest, embedder, false)?; + if processed > 0 { + eprintln!("[watch] processed {processed} new file(s)."); + } + std::thread::sleep(Duration::from_secs(cli.poll_interval.max(1))); + } +} + +/// A processed chunk paired with its source file's content hash. +struct ProcessedChunk { + spec: ChunkSpec, + source_hash: String, +} + +/// Processes a single file through parse → scrub → chunk. +fn process_file(path: &Path, config: &PipelineConfig) -> Vec { + let bytes = match fs::read(path) { + Ok(b) => b, + Err(e) => { + eprintln!(" ⚠ could not read {}: {e}", path.display()); + return Vec::new(); + } + }; + + let cfg = PipelineConfig { + source_label: Some(path.display().to_string()), + ..config.clone() + }; + let cfg = infer_format(path, cfg); + + match parse_bytes(&bytes, &cfg).and_then(|doc| { + let (scrubbed, offset_map, findings) = scrub_document(doc, &cfg.scrub)?; + let mut chunks = chunk_document(&scrubbed, &cfg.chunk, cfg.source_label.as_deref())?; + attach_metadata(&mut chunks, &findings, &offset_map); + Ok(chunks) + }) { + Ok(chunks) => chunks, + Err(e) => { + eprintln!(" ⚠ failed to process {}: {e}", path.display()); + Vec::new() + } + } +} + +// --------------------------------------------------------------------------- +// Output +// --------------------------------------------------------------------------- + +/// Writes chunks to the output file (TUI path; no hashes, no embeddings). +pub fn write_output(chunks: &[ChunkSpec], output: &str) -> Result<(), Box> { + let processed: Vec = chunks + .iter() + .map(|spec| ProcessedChunk { + spec: spec.clone(), + source_hash: String::new(), + }) + .collect(); + write_output_inner(&processed, output, None) +} + +/// Writes processed chunks in the format implied by the output extension. +/// Adds `source_hash`/`chunk_hash` to JSON, and fills the embedding column +/// for Arrow/CSV when an embedder is supplied. +fn write_output_inner( + processed: &[ProcessedChunk], + output: &str, + embedder: Option<&dyn bitvanes_core::Embedder>, +) -> Result<(), Box> { + let ext = Path::new(output) + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("json"); + + let chunks: Vec = processed.iter().map(|p| p.spec.clone()).collect(); + let source_hashes: Vec = processed.iter().map(|p| p.source_hash.clone()).collect(); + + // Embeddings (optional). All chunk texts in one batch. + let embeddings: Option<(Vec>, usize)> = match embedder { + Some(em) => { + let texts: Vec<&str> = chunks.iter().map(|c| c.text.as_str()).collect(); + let vecs = em.embed(&texts)?; + Some((vecs, em.dim())) + } + None => None, + }; - if all_chunks.is_empty() { - eprintln!("\nNo chunks generated. Check that your input files are valid UTF-8 text."); - return Ok(()); + match ext { + "arrow" => { + let ipc_bytes = match &embeddings { + Some((vecs, dim)) => { + let batch = chunks_to_batch_with_embeddings(&chunks, vecs, *dim)?; + bitvanes_core::arrow_io::ipc::write_ipc_stream(&batch)? + } + None => { + let batch = chunks_to_batch(&chunks)?; + bitvanes_core::arrow_io::ipc::write_ipc_stream(&batch)? + } + }; + write_bytes(output, &ipc_bytes)?; + } + "csv" => { + let batch = match &embeddings { + Some((vecs, dim)) => chunks_to_batch_with_embeddings(&chunks, vecs, *dim)?, + None => chunks_to_batch(&chunks)?, + }; + let csv_text = bitvanes_core::arrow_io::csv::write_csv(&batch)?; + write_text(output, &csv_text)?; + } + _ => { + // JSON (default). Carries hashes; embeddings are omitted (use Arrow). + let rows: Vec = chunks + .iter() + .zip(&source_hashes) + .map(|(c, src)| JsonChunk::from_chunk(c, src)) + .collect(); + let json = serde_json::to_string_pretty(&rows)?; + write_text(output, &json)?; + } } + Ok(()) +} - // 5. Write output. - let output = cli.output.as_deref().unwrap_or("output.json"); - write_output(&all_chunks, output)?; - eprintln!("\nOutput written to {}", output); +fn write_bytes(output: &str, bytes: &[u8]) -> Result<(), Box> { + if output == "-" { + use std::io::Write; + std::io::stdout().write_all(bytes)?; + } else { + fs::write(output, bytes)?; + } + Ok(()) +} +fn write_text(output: &str, text: &str) -> Result<(), Box> { + if output == "-" { + print!("{text}"); + } else { + fs::write(output, text)?; + } Ok(()) } -/// Builds a `PipelineConfig` from a profile JSON and/or CLI flags. -pub(crate) fn build_config(cli: &Cli) -> Result> { - let mut config = if let Some(path) = &cli.config { - let json = fs::read_to_string(path) - .map_err(|e| format!("could not read config {}: {e}", path.display()))?; - serde_json::from_str(&json)? +/// Streams one Arrow `RecordBatch` per file directly to the output sink +/// (file or stdout). Memory holds only one file's batch at a time — flat +/// regardless of batch size. Enables true pipe streaming: +/// `bitvanes parse ./docs --format arrow -o - | vector-db-ingest`. +fn write_streaming_arrow( + per_file: &[(std::path::PathBuf, String, Vec)], + output: &str, + embedder: Option<&dyn bitvanes_core::Embedder>, +) -> Result<(), Box> { + // Determine the schema (with or without embeddings). + let schema = if let Some(em) = embedder { + bitvanes_core::arrow_io::output_schema_with_dim(em.dim()) } else { - PipelineConfig::default() + bitvanes_core::arrow_io::output_schema() }; - // CLI flags override profile values. - if let Some(fmt) = &cli.format { - config.format = parse_format(fmt)?; - } - if let Some(tok) = &cli.tokenizer { - config.chunk.tokenizer = parse_tokenizer(tok)?; + // Open the writer: stdout for "-", file otherwise. + let writer: Box = if output == "-" { + Box::new(std::io::BufWriter::new(std::io::stdout())) + } else { + Box::new(std::io::BufWriter::new(std::fs::File::create(output)?)) + }; + + let mut stream = IpcStream::try_new(writer, &schema)?; + + for (_, _hash, chunks) in per_file { + if chunks.is_empty() { + continue; + } + let batch = if let Some(em) = embedder { + let texts: Vec<&str> = chunks.iter().map(|c| c.text.as_str()).collect(); + let vecs = em.embed(&texts)?; + chunks_to_batch_with_embeddings(chunks, &vecs, em.dim())? + } else { + chunks_to_batch(chunks)? + }; + stream.write(&batch)?; } - if let Some(mt) = cli.max_tokens { - config.chunk.max_tokens = mt; + stream.finish()?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// File collection (walkdir + glob) +// --------------------------------------------------------------------------- + +fn collect_files( + cli: &Cli, + config: &PipelineConfig, +) -> Result, Box> { + let mut out = Vec::new(); + + if let Some(input) = &cli.input { + out.extend(collect_input(input, cli, config)?); } - if let Some(scrub) = &cli.scrub { - config.scrub = parse_scrub(scrub); + for pattern in &cli.glob { + for entry in glob::glob(pattern).map_err(|e| format!("bad glob '{pattern}': {e}"))? { + match entry { + Ok(p) => { + if let Some(canon) = canonical_if_supported(&p) { + if canon.is_file() && is_supported_ext(&canon) { + out.push(canon); + } else if canon.is_dir() { + out.extend(walk_dir(&canon, cli, config)); + } + } + } + Err(e) => eprintln!(" ⚠ glob read error: {e}"), + } + } } - Ok(config) + // Dedupe (the same file may match --input and a --glob). + let mut seen = HashSet::new(); + out.retain(|p| seen.insert(p.clone())); + Ok(out) } -/// Collects files from a path (single file or recursive directory walk). -fn collect_files( +fn collect_input( input: &Path, - config: &PipelineConfig, cli: &Cli, + config: &PipelineConfig, ) -> Result, Box> { if input.is_file() { - return Ok(vec![input.canonicalize()?]); + return Ok(vec![canonical_or(input)]); } - if !input.is_dir() { return Err(format!("{} is not a file or directory", input.display()).into()); } + Ok(walk_dir(input, cli, config)) +} - // Determine which extensions to look for. If format is explicitly given on CLI, - // restrict to those extensions. Otherwise, collect all supported formats. +fn walk_dir(root: &Path, cli: &Cli, config: &PipelineConfig) -> Vec { let extensions: &[&str] = if cli.format.is_some() { match config.format { DocumentFormat::Markdown => &["md", "markdown"], @@ -135,12 +421,18 @@ fn collect_files( DocumentFormat::Html => &["html", "htm"], DocumentFormat::Pdf => &["pdf"], DocumentFormat::Json => &["json"], + DocumentFormat::Docx => &["docx"], + DocumentFormat::Pptx => &["pptx"], + DocumentFormat::Xlsx => &["xlsx"], + DocumentFormat::Epub => &["epub"], + DocumentFormat::Rtf => &["rtf"], + _ => SUPPORTED_EXTS, } } else { - &["md", "markdown", "txt", "html", "htm", "pdf", "json"] + SUPPORTED_EXTS }; - let files: Vec = WalkDir::new(input) + WalkDir::new(root) .into_iter() .filter_map(|e| e.ok()) .filter(|e| e.file_type().is_file()) @@ -151,41 +443,149 @@ fn collect_files( .map(|ext| extensions.contains(&ext.to_lowercase().as_str())) .unwrap_or(false) }) - .map(|e| e.into_path()) - .collect(); + .map(|e| canonical_or(e.path())) + .collect() +} - Ok(files) +fn is_supported_ext(path: &Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .map(|e| SUPPORTED_EXTS.contains(&e.to_lowercase().as_str())) + .unwrap_or(false) } -/// Processes a single file through the full pipeline. -fn process_file(path: &Path, config: &PipelineConfig) -> Vec { - let bytes = match fs::read(path) { - Ok(b) => b, - Err(e) => { - eprintln!(" ⚠ Could not read {}: {}", path.display(), e); - return Vec::new(); +/// canonicalize where possible (falls back to the path as-is on platforms +/// where the file is not yet statable through canonicalize). +fn canonical_or(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + +fn canonical_if_supported(path: &Path) -> Option { + Some(path.canonicalize().unwrap_or_else(|_| path.to_path_buf())) +} + +// --------------------------------------------------------------------------- +// Manifest (idempotency) +// --------------------------------------------------------------------------- + +#[derive(Serialize, Deserialize, Default)] +struct Manifest { + processed: Vec, +} + +impl Manifest { + fn load_or_default(path: Option<&Path>) -> Result> { + match path { + Some(p) if p.exists() => { + let text = fs::read_to_string(p)?; + let m: Manifest = serde_json::from_str(&text)?; + Ok(m) + } + _ => Ok(Manifest::default()), } - }; + } - // Override source_label with the file path. - let cfg = PipelineConfig { - source_label: Some(path.display().to_string()), - ..config.clone() - }; + fn contains(&self, hash: &str) -> bool { + self.processed.iter().any(|h| h == hash) + } - // Try to infer format from extension if the config format doesn't match. - let cfg = infer_format(path, cfg); + fn insert(&mut self, hash: String) { + if !self.contains(&hash) { + self.processed.push(hash); + } + } - match parse_bytes(&bytes, &cfg) - .and_then(|doc| scrub_document(doc, &cfg.scrub).map(|(d, _)| d)) - .and_then(|doc| chunk_document(&doc, &cfg.chunk, cfg.source_label.as_deref())) - { - Ok(chunks) => chunks, - Err(e) => { - eprintln!(" ⚠ Failed to process {}: {}", path.display(), e); - Vec::new() + fn save(&self, path: Option<&Path>) -> Result<(), Box> { + if let Some(p) = path { + fs::write(p, serde_json::to_string_pretty(self)?)?; } + Ok(()) + } + + #[cfg(test)] + fn len(&self) -> usize { + self.processed.len() + } +} + +/// Blake3 hash of bytes, lower-case hex. +fn hex_hash(bytes: &[u8]) -> String { + let h = blake3::hash(bytes); + h.to_hex().to_string() +} + +// --------------------------------------------------------------------------- +// Config + embedding construction +// --------------------------------------------------------------------------- + +/// Builds a `PipelineConfig` from layered sources (highest precedence last): +/// built-in defaults → `bitvanes.toml` → `--config` JSON → CLI flags. +pub(crate) fn build_config(cli: &Cli) -> Result> { + // Layer 1: TOML config (if specified). + let mut config = if let Some(path) = &cli.toml { + let toml_str = fs::read_to_string(path) + .map_err(|e| format!("could not read toml config {}: {e}", path.display()))?; + toml::from_str(&toml_str)? + } else { + PipelineConfig::default() + }; + + // Layer 2: JSON profile (web-app export), overrides TOML. + if let Some(path) = &cli.config { + let json = fs::read_to_string(path) + .map_err(|e| format!("could not read config {}: {e}", path.display()))?; + let json_cfg: PipelineConfig = serde_json::from_str(&json)?; + config = json_cfg; + } + + // Layer 3: CLI flags (highest precedence). + if let Some(fmt) = &cli.format { + config.format = parse_format(fmt)?; + } + if let Some(tok) = &cli.tokenizer { + config.chunk.tokenizer = parse_tokenizer(tok)?; + } + if let Some(mt) = cli.max_tokens { + config.chunk.max_tokens = mt; + } + if let Some(scrub) = &cli.scrub { + config.scrub = parse_scrub(scrub); + } + if let Some(mc) = cli.min_confidence { + config.scrub.min_confidence = mc; + } + if let Some(aw) = cli.anchor_window { + config.scrub.anchor_window = aw; + } + if let Some(exclude) = &cli.exclude_pii { + config.scrub.report_only = exclude.split(',').map(|s| s.trim().to_string()).collect(); + } + + Ok(config) +} + +#[cfg(feature = "embed")] +fn build_embedder( + cli: &Cli, +) -> Result>, Box> { + if let (Some(model), Some(tok)) = (cli.embed.as_deref(), cli.embed_tokenizer.as_deref()) { + let em = bitvanes_core::OrtEmbedder::new(model, tok, cli.embed_dim, cli.embed_max_len)?; + eprintln!( + "Embeddings on: model={} dim={} max_len={}", + model.display(), + cli.embed_dim, + cli.embed_max_len + ); + return Ok(Some(Box::new(em))); } + Ok(None) +} + +#[cfg(not(feature = "embed"))] +fn build_embedder( + _cli: &Cli, +) -> Result>, Box> { + Ok(None) } /// Overrides the format based on file extension so each file is parsed by @@ -201,53 +601,16 @@ pub(crate) fn infer_format(path: &Path, mut cfg: PipelineConfig) -> PipelineConf Some("html") | Some("htm") => cfg.format = DocumentFormat::Html, Some("pdf") => cfg.format = DocumentFormat::Pdf, Some("json") => cfg.format = DocumentFormat::Json, + Some("docx") => cfg.format = DocumentFormat::Docx, + Some("pptx") => cfg.format = DocumentFormat::Pptx, + Some("xlsx") => cfg.format = DocumentFormat::Xlsx, + Some("epub") => cfg.format = DocumentFormat::Epub, + Some("rtf") => cfg.format = DocumentFormat::Rtf, _ => {} } cfg } -/// Writes chunks to the output file in the specified format. -pub fn write_output(chunks: &[ChunkSpec], output: &str) -> Result<(), Box> { - let ext = Path::new(output) - .extension() - .and_then(|e| e.to_str()) - .unwrap_or("json"); - - match ext { - "arrow" => { - let batch = chunks_to_batch(chunks)?; - let ipc_bytes = bitvanes_core::arrow_io::ipc::write_ipc_stream(&batch)?; - if output == "-" { - use std::io::Write; - std::io::stdout().write_all(&ipc_bytes)?; - } else { - fs::write(output, ipc_bytes)?; - } - } - "csv" => { - let batch = chunks_to_batch(chunks)?; - let csv_text = bitvanes_core::arrow_io::csv::write_csv(&batch)?; - if output == "-" { - print!("{csv_text}"); - } else { - fs::write(output, csv_text)?; - } - } - _ => { - // JSON output (default). - let json_chunks: Vec = chunks.iter().map(JsonChunk::from).collect(); - let json = serde_json::to_string_pretty(&json_chunks)?; - if output == "-" { - println!("{json}"); - } else { - fs::write(output, json)?; - } - } - } - - Ok(()) -} - // --- Helpers --- fn parse_format(s: &str) -> Result> { @@ -273,30 +636,63 @@ fn parse_scrub(s: &str) -> ScrubProfile { .collect(); ScrubProfile { patterns, - custom: vec![], + ..ScrubProfile::default() } } -/// JSON-serializable chunk wrapper. +/// JSON-serializable chunk wrapper with dedup hashes + PII metadata. #[derive(Serialize)] struct JsonChunk { chunk_index: u32, + chunk_id: String, text: String, token_count: u16, source_path: String, heading_path: Vec, section_kind: String, + /// PII findings overlapping this chunk (offsets into original text). + pii: Vec, + /// Blake3 of the source file's bytes (empty in the TUI path). + source_hash: String, + /// Blake3 of this chunk's text. + chunk_hash: String, } -impl From<&ChunkSpec> for JsonChunk { - fn from(c: &ChunkSpec) -> Self { +/// JSON-serializable PII finding. +#[derive(Serialize)] +struct JsonPiiFinding { + entity: String, + offset_start: u32, + offset_end: u32, + confidence: f32, + anchors_hit: Vec, +} + +impl JsonChunk { + fn from_chunk(c: &ChunkSpec, source_hash: &str) -> Self { + let chunk_hash = hex_hash(c.text.as_bytes()); + let pii = c + .pii + .iter() + .map(|f| JsonPiiFinding { + entity: f.entity.clone(), + offset_start: f.offset_start, + offset_end: f.offset_end, + confidence: f.confidence, + anchors_hit: f.anchors_hit.clone(), + }) + .collect(); Self { chunk_index: c.chunk_index, + chunk_id: c.chunk_id.clone(), text: c.text.clone(), token_count: c.token_count, source_path: c.source_path.clone(), heading_path: c.heading_path.clone(), section_kind: format!("{:?}", c.section_kind).to_lowercase(), + pii, + source_hash: source_hash.to_string(), + chunk_hash, } } } @@ -304,108 +700,139 @@ impl From<&ChunkSpec> for JsonChunk { #[cfg(test)] mod tests { use super::*; - use bitvanes_core::SectionKind; - - fn chunk(text: &str) -> ChunkSpec { - ChunkSpec { - chunk_index: 0, - text: text.to_string(), - token_count: 1, - source_path: "t.md".to_string(), - heading_path: vec![], - section_kind: SectionKind::Paragraph, - char_offset_start: 0, - char_offset_end: text.len() as u32, - } - } #[test] - fn parse_format_recognizes_every_supported_format() { - assert_eq!(parse_format("markdown").unwrap(), DocumentFormat::Markdown); - assert_eq!(parse_format("md").unwrap(), DocumentFormat::Markdown); - assert_eq!(parse_format("text").unwrap(), DocumentFormat::Text); - assert_eq!(parse_format("HTML").unwrap(), DocumentFormat::Html); - assert_eq!(parse_format("json").unwrap(), DocumentFormat::Json); - assert_eq!(parse_format("pdf").unwrap(), DocumentFormat::Pdf); - assert!(parse_format("docx").is_err()); + fn hex_hash_is_stable_and_distinct() { + let a = hex_hash(b"hello"); + let b = hex_hash(b"hello"); + let c = hex_hash(b"world"); + assert_eq!(a, b); + assert_ne!(a, c); + assert_eq!(a.len(), 64); // blake3 hex = 64 chars } #[test] - fn parse_tokenizer_accepts_all_variants_case_insensitively() { - for s in ["cl100k_base", "O200K_BASE", "r50k_base"] { - assert!(parse_tokenizer(s).is_ok(), "{s} should parse"); - } - assert!(parse_tokenizer("nope").is_err()); - } - - #[test] - fn parse_scrub_collects_known_patterns() { - let profile = parse_scrub("email,ssn, credit_card ,, unknown"); - assert_eq!(profile.patterns.len(), 3); - assert!(profile.patterns.contains(&BuiltInPattern::Email)); - assert!(profile.patterns.contains(&BuiltInPattern::CreditCard)); - assert!(profile.custom.is_empty()); + fn manifest_round_trips_and_dedups() { + let tmp = format!( + "/tmp/bv-manifest-{}.json", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let path = PathBuf::from(&tmp); + let mut m = Manifest::default(); + m.insert("aaa".to_string()); + m.insert("aaa".to_string()); // dup ignored + m.insert("bbb".to_string()); + assert_eq!(m.len(), 2); + assert!(m.contains("aaa")); + m.save(Some(&path)).unwrap(); + + let loaded = Manifest::load_or_default(Some(&path)).unwrap(); + assert_eq!(loaded.len(), 2); + assert!(loaded.contains("bbb")); + let _ = fs::remove_file(&path); } #[test] - fn infer_format_routes_by_extension() { + fn infer_format_covers_all_extensions() { let base = PipelineConfig::default(); assert_eq!( - infer_format(std::path::Path::new("/x/a.md"), base.clone()).format, + infer_format(Path::new("/x/a.md"), base.clone()).format, DocumentFormat::Markdown ); assert_eq!( - infer_format(std::path::Path::new("/x/a.json"), base.clone()).format, + infer_format(Path::new("/x/a.json"), base.clone()).format, DocumentFormat::Json ); assert_eq!( - infer_format(std::path::Path::new("/x/a.pdf"), base.clone()).format, + infer_format(Path::new("/x/a.pdf"), base.clone()).format, DocumentFormat::Pdf ); assert_eq!( - infer_format(std::path::Path::new("/x/a.htm"), base.clone()).format, - DocumentFormat::Html - ); - // Unknown extension keeps the configured format. - assert_eq!( - infer_format(std::path::Path::new("/x/a.xyz"), base).format, + infer_format(Path::new("/x/a.xyz"), base).format, DocumentFormat::Markdown ); } #[test] - fn write_output_emits_all_three_formats() { - let id = nanos(); - let chunks = vec![chunk("hello world")]; - - let json_path = format!("/tmp/bv-pub-{id}.json"); - write_output(&chunks, &json_path).unwrap(); - assert!( - std::fs::read_to_string(&json_path) + fn parse_scrub_filters_unknowns() { + let p = parse_scrub("email, garbage ,ssn"); + assert_eq!(p.patterns.len(), 2); + assert!(p.patterns.contains(&BuiltInPattern::Email)); + assert!(p.patterns.contains(&BuiltInPattern::Ssn)); + } + + #[test] + fn json_chunk_carries_hashes() { + let spec = ChunkSpec { + chunk_index: 0, + chunk_id: "test".to_string(), + text: "sample text".to_string(), + token_count: 2, + source_path: "t.md".to_string(), + heading_path: vec![], + section_kind: bitvanes_core::SectionKind::Paragraph, + char_offset_start: 0, + char_offset_end: 11, + pii: vec![], + }; + let jc = JsonChunk::from_chunk(&spec, "deadbeef"); + assert_eq!(jc.source_hash, "deadbeef"); + assert_eq!(jc.chunk_hash, hex_hash(b"sample text")); + } + + #[test] + fn collect_files_dedupes_overlap() { + // Write two identical-path refs via a real dir to exercise dedup. + let dir = format!( + "/tmp/bv-collect-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) .unwrap() - .contains("hello world") + .as_nanos() ); - - let csv_path = format!("/tmp/bv-pub-{id}.csv"); - write_output(&chunks, &csv_path).unwrap(); - let csv = std::fs::read_to_string(&csv_path).unwrap(); - assert!(csv.lines().next().unwrap().contains("chunk_index")); - assert!(csv.contains("hello world")); - - let arrow_path = format!("/tmp/bv-pub-{id}.arrow"); - write_output(&chunks, &arrow_path).unwrap(); - let bytes = std::fs::read(&arrow_path).unwrap(); - // Arrow IPC stream begins with the continuation marker 0xFFFFFFFF. - assert_eq!(&bytes[..4], &[0xFF, 0xFF, 0xFF, 0xFF]); - let _ = std::fs::remove_file(&json_path); - let _ = std::fs::remove_file(&csv_path); - let _ = std::fs::remove_file(&arrow_path); - } - - fn nanos() -> u128 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() + fs::create_dir_all(&dir).unwrap(); + fs::write(format!("{}/a.md", dir), "# hi").unwrap(); + let cli = Cli { + input: Some(PathBuf::from(&dir)), + glob: vec![format!("{}/a.md", dir)], + config: None, + no_tui: true, + format: None, + tokenizer: None, + max_tokens: None, + scrub: None, + exclude_pii: None, + init: false, + min_confidence: None, + anchor_window: None, + toml: None, + output: None, + jobs: None, + manifest: None, + watch: false, + poll_interval: 5, + force: false, + #[cfg(feature = "embed")] + embed: None, + #[cfg(feature = "embed")] + embed_tokenizer: None, + #[cfg(feature = "embed")] + embed_dim: 384, + #[cfg(feature = "embed")] + embed_max_len: 256, + }; + let files = collect_files(&cli, &PipelineConfig::default()).unwrap(); + let count = files + .iter() + .filter(|p| p.extension().and_then(|e| e.to_str()) == Some("md")) + .count(); + assert_eq!( + count, 1, + "a.md must appear once despite --input + --glob overlap" + ); + let _ = fs::remove_dir_all(&dir); } } diff --git a/src/main.rs b/src/main.rs index 6607311..8e0ef02 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,7 @@ use std::path::PathBuf; use std::process::ExitCode; /// Zero-trust ETL for AI/RAG — chunk documents for vector databases. -#[derive(Parser)] +#[derive(Parser, Debug)] #[command( name = "bitvanes", version, @@ -21,67 +21,198 @@ use std::process::ExitCode; Output formats: Arrow IPC (.arrow), CSV (.csv), JSON (.json).\n\ Supports profile replay from the web app." )] -struct Cli { +pub struct Cli { /// Input file or directory (recursive scan). #[arg(short, long, value_name = "PATH")] - input: Option, + pub input: Option, + + /// Glob pattern(s) instead of --input (may be repeated). e.g. "docs/**/*.md". + #[arg(long, value_name = "PATTERN")] + pub glob: Vec, /// Pipeline profile JSON (exported from the web app). #[arg(short, long, value_name = "FILE")] - config: Option, + pub config: Option, - /// Headless mode — no TUI, just process and output. (Default if --input is given.) + /// Headless mode — no TUI, just process and output. (Default if --input/--glob is given.) #[arg(long)] - no_tui: bool, + pub no_tui: bool, /// Document format. Inferred from file extension if omitted. #[arg(short, long, value_name = "FORMAT")] - format: Option, + pub format: Option, /// BPE tokenizer for chunk boundary calculation. #[arg(short, long, value_name = "NAME")] - tokenizer: Option, + pub tokenizer: Option, /// Maximum tokens per chunk. #[arg(short = 'm', long, value_name = "N")] - max_tokens: Option, + pub max_tokens: Option, /// PII patterns to scrub (comma-separated). E.g. "email,ssn,credit_card". #[arg(long, value_name = "PATTERNS")] - scrub: Option, + pub scrub: Option, + + /// PII patterns to detect but NOT scrub (comma-separated). The finding + /// is recorded in pii_metadata but the original text passes through + /// unchanged. Useful for audit-only pipelines. E.g. "email,ssn". + #[arg(long, value_name = "PATTERNS")] + pub exclude_pii: Option, + + /// Write a bitvanes.toml template to the current directory and exit. + #[arg(long)] + pub init: bool, + + /// Minimum confidence [0.0–1.0] for a PII candidate to be scrubbed and + /// reported. Candidates below this threshold are silently dropped. + /// Default: 0.0 (keep all that pass algorithmic verification). + #[arg(long, value_name = "FLOAT")] + pub min_confidence: Option, + + /// Half-window size (in words) for the contextual anchor scan around + /// each PII candidate. Default: 7. Set to 0 to disable anchor boosting. + #[arg(long, value_name = "N")] + pub anchor_window: Option, + + /// TOML config file (bitvanes.toml). Layered on top of --config (JSON) + /// but below CLI flags. See `bitvanes init` for a template. + #[arg(long, value_name = "FILE")] + pub toml: Option, /// Output file. Supports .arrow, .csv, .json extensions. /// Use "-" for stdout (JSON only). #[arg(short, long, value_name = "FILE")] - output: Option, + pub output: Option, + + /// Limit parallelism to N file-processing threads (default: all cores). + #[arg(long, value_name = "N")] + pub jobs: Option, + + /// Persist processed file hashes to this JSON manifest and skip files + /// already present in it (idempotent / incremental runs). + #[arg(long, value_name = "FILE")] + pub manifest: Option, + + /// Watch mode: after the initial pass, keep scanning for new/changed + /// files and process them until interrupted (Ctrl-C). Implies a manifest + /// in memory when --manifest is not given. + #[arg(long)] + pub watch: bool, + + /// Seconds between watch scans (default: 5). + #[arg(long, value_name = "SECS", default_value_t = 5)] + pub poll_interval: u64, + + /// Re-process files even if their hash is in the manifest. + #[arg(long)] + pub force: bool, + + // --- Embeddings (on-device; requires `--features embed` at build time) --- + /// ONNX model file for on-device embeddings (enables the embedding column). + #[cfg(feature = "embed")] + #[arg(long, value_name = "MODEL.onnx", requires = "embed_tokenizer")] + pub embed: Option, + + /// Tokenizer.json paired with --embed. + #[cfg(feature = "embed")] + #[arg(long, value_name = "FILE")] + pub embed_tokenizer: Option, + + /// Embedding dimension of the model (default: 384, MiniLM-L6-v2). + #[cfg(feature = "embed")] + #[arg(long, value_name = "N", default_value_t = 384)] + pub embed_dim: usize, + + /// Max sequence length for the embedder (default: 256). + #[cfg(feature = "embed")] + #[arg(long, value_name = "N", default_value_t = 256)] + pub embed_max_len: usize, } fn main() -> ExitCode { let cli = Cli::parse(); - // If --input is provided, run headless. Otherwise launch the TUI. - if cli.input.is_some() || cli.no_tui { - if cli.input.is_none() { - // `--no-tui` without an input has nothing to do; show help - // instead of failing with a confusing "no input" error. - let _ = Cli::command().print_help(); - println!(); - return ExitCode::SUCCESS; - } - match headless::run(&cli) { - Ok(()) => ExitCode::SUCCESS, + if cli.init { + return match write_default_config() { + Ok(()) => { + println!("Wrote bitvanes.toml to the current directory."); + ExitCode::SUCCESS + } Err(e) => { - eprintln!("Error: {e}"); + eprintln!("Error writing config: {e}"); ExitCode::FAILURE } + }; + } + + // Cap rayon parallelism if requested (must precede any rayon use). + if let Some(n) = cli.jobs { + if let Err(e) = rayon::ThreadPoolBuilder::new() + .num_threads(n.max(1)) + .build_global() + { + eprintln!("Warning: could not set thread count to {n}: {e}"); } - } else { - match tui::run(&cli) { + } + + let headless = cli.input.is_some() || !cli.glob.is_empty() || cli.no_tui; + + if !headless { + return match tui::run(&cli) { Ok(()) => ExitCode::SUCCESS, Err(e) => { eprintln!("Error: {e}"); ExitCode::FAILURE } + }; + } + + if cli.input.is_none() && cli.glob.is_empty() { + // `--no-tui` with nothing to read: show help. + let _ = Cli::command().print_help(); + println!(); + return ExitCode::SUCCESS; + } + + match headless::run(&cli) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("Error: {e}"); + ExitCode::FAILURE } } } + +/// Default `bitvanes.toml` template written by `--init`. +const CONFIG_TEMPLATE: &str = r#"# bitvanes.toml — pipeline configuration for the BitVanes CLI. +# Layered below CLI flags and --config (JSON profile) but above built-in defaults. +# Generated by `bitvanes --init`. + +[format] +# markdown | text | html | json | pdf | docx | pptx | xlsx | epub | rtf +format = "markdown" + +[scrub] +# Built-in patterns: email, ssn, phone, credit_card, routing_number, aws_key, github_pat, jwt +patterns = ["email"] +# Entity slugs to detect but NOT scrub (audit-only): report_only = ["ssn"] +# Contextual anchor half-window in words (0 disables boosting): +anchor_window = 7 +# Minimum confidence [0.0–1.0] for a candidate to be scrubbed: +min_confidence = 0.0 + +[[scrub.custom]] +regex = "\\bPROJECT-\\d+\\b" +replacement = "[PROJECT]" + +[chunk] +max_tokens = 512 +overlap_tokens = 0 +# cl100k_base | o200k_base | r50k_base | p50k_base | p50k_edit | o200k_harmony +tokenizer = "cl100k_base" +"#; + +fn write_default_config() -> std::io::Result<()> { + std::fs::write("bitvanes.toml", CONFIG_TEMPLATE) +} diff --git a/src/tui/app.rs b/src/tui/app.rs index f42d8e7..922442e 100644 --- a/src/tui/app.rs +++ b/src/tui/app.rs @@ -446,10 +446,12 @@ fn run_pipeline_for_files(paths: &[PathBuf], base_config: &PipelineConfig) -> Pr }; let cfg = crate::headless::infer_format(path, cfg); - match parse_bytes(&bytes, &cfg) - .and_then(|doc| scrub_document(doc, &cfg.scrub).map(|(d, _)| d)) - .and_then(|doc| chunk_document(&doc, &cfg.chunk, cfg.source_label.as_deref())) - { + match parse_bytes(&bytes, &cfg).and_then(|doc| { + let (scrubbed, map, findings) = scrub_document(doc, &cfg.scrub)?; + let mut c = chunk_document(&scrubbed, &cfg.chunk, cfg.source_label.as_deref())?; + bitvanes_core::pipeline::attach_metadata(&mut c, &findings, &map); + Ok(c) + }) { Ok(c) => chunks.extend(c), Err(e) => error = Some(format!("failed {}: {e}", path.display())), } @@ -533,6 +535,7 @@ mod tests { let mut app = AppState::new(&cli_with(&["--no-tui"])); app.chunks = vec![ChunkSpec { chunk_index: 0, + chunk_id: "test".to_string(), text: "sample".to_string(), token_count: 1, source_path: "t.md".to_string(), @@ -540,6 +543,7 @@ mod tests { section_kind: bitvanes_core::SectionKind::Paragraph, char_offset_start: 0, char_offset_end: 6, + pii: vec![], }]; let out = format!( "/tmp/bitvanes-save-{}.json",