Add IPFS storage backend support

.env.example

@@ -12,6 +12,7 @@ AZURITE_VERSION=latest
 JAEGER_VERSION=latest
 OTEL_VERSION=latest
 PROMETHEUS_VERSION=latest
+KUBO_VERSION=latest
 
 # Expose the services to outside network
 MAP_HOST_LOCALSTACK=0.0.0.0
@@ -24,3 +25,4 @@ MAP_HOST_GRAFANA=0.0.0.0
 MAP_HOST_JAEGER=0.0.0.0
 MAP_HOST_OTEL=0.0.0.0
 MAP_HOST_PROMETHEUS=0.0.0.0
+MAP_HOST_KUBO=0.0.0.0

docker-compose.yml

@@ -224,10 +224,33 @@ services:
       - all
       - gcp-pubsub
 
+  kubo:
+    image: ipfs/kubo:${KUBO_VERSION:-v0.32.1}
+    container_name: kubo
+    # Offline, local-only node for development and tests: the `test` profile
+    # strips the public bootstrap peers and `--offline` prevents any outbound
+    # network activity. Remove both to join the public IPFS network.
+    command: ["daemon", "--migrate=true", "--offline"]
+    environment:
+      IPFS_PROFILE: test
+    ports:
+      - "${MAP_HOST_KUBO:-127.0.0.1}:5001:5001" # RPC API
+      - "${MAP_HOST_KUBO:-127.0.0.1}:8082:8080" # Gateway
+    profiles:
+      - ipfs
+    volumes:
+      - kubo_data:/data/ipfs
+    healthcheck:
+      test: ["CMD", "ipfs", "id"]
+      interval: 1s
+      timeout: 5s
+      retries: 100
+
 volumes:
   azurite_data:
   fake_gcs_server_data:
   grafana_conf:
   grafana_data:
+  kubo_data:
   localstack_data:
   postgres_data:

docs/configuration/storage-config.md

@@ -5,11 +5,12 @@ sidebar_position: 2
 
 ## Supported Storage Providers
 
-Quickwit currently supports four types of storage providers:
+Quickwit currently supports five types of storage providers:
 - Amazon S3 and S3-compatible (Garage, MinIO, ...)
 - Azure Blob Storage
 - Local file storage*
 - Google Cloud Storage (native API)
+- IPFS (InterPlanetary File System, via the RPC API of a node such as [Kubo](https://github.com/ipfs/kubo))
 
 ## Storage URIs
 
@@ -18,6 +19,7 @@ Storage URIs refer to different storage providers identified by a URI "protocol"
 - `azure://` for Azure Blob Storage
 - `file://` for local file systems
 - `gs://` for Google Cloud Storage
+- `ipfs://` for IPFS (content-addressed storage)
 
 In general, you can use a storage URI or a file path anywhere you would intuitively expect a file path. For instance:
 - when setting the `index_uri` of an index to specify the storage provider and location;
@@ -127,6 +129,45 @@ storage:
     access_key: your-azure-access-key
 ```
 
+### IPFS storage configuration
+
+Quickwit stores splits in the MFS (Mutable File System) of an IPFS node through its RPC API. MFS presents content-addressed, chunked DAGs as regular files supporting ranged reads, which matches the byte-range access pattern Quickwit uses to open splits. Every split written this way has a CID and is addressable, pinnable, and exportable (e.g. as a CAR file) from the wider IPFS network.
+
+An `ipfs://` URI maps to an MFS directory on the configured node: `ipfs://quickwit-indexes/my-index` reads and writes under the MFS path `/quickwit-indexes/my-index`.
+
+| Property | Description | Default value |
+| --- | --- | --- |
+| `api_endpoint` | Base URL of the IPFS node RPC API. | `http://127.0.0.1:5001` |
+| `chunker` | Chunker applied when writing splits. Larger fixed-size chunks reduce the number of blocks a ranged read spans. | `size-1048576` |
+| `raw_leaves` | Use raw leaf blocks (no UnixFS envelope around data blocks). | `true` |
+| `request_timeout_secs` | Request timeout in seconds for individual RPC calls. | `30` |
+
+#### Environment variables
+
+| Env variable | Description |
+| --- | --- |
+| `QW_IPFS_API_ENDPOINT` | Base URL of the IPFS node RPC API. |
+
+Example of a storage configuration for IPFS in YAML format:
+
+```yaml
+storage:
+  ipfs:
+    api_endpoint: http://127.0.0.1:5001
+```
+
+:::note
+The IPFS backend covers index data (splits). Keep the metastore on PostgreSQL or a file-backed URI: metastore state is small, mutable, and frequently rewritten, which is a poor fit for content addressing.
+:::
+
+:::note
+Deleting a split unlinks it from MFS but only reclaims disk space once the IPFS node runs garbage collection. Kubo does not GC by default: run the daemon with `--enable-gc` or schedule `ipfs repo gc`, especially on indexes with frequent merges.
+:::
+
+:::caution
+The IPFS storage backend requires a Quickwit binary compiled with the `ipfs` feature enabled (`--features quickwit-storage/ipfs`). Release binaries built from the default release feature set include it.
+:::
+
 ## Storage configuration examples for various object storage providers
 
 ### Garage

docs/internals/ipfs-storage.md

@@ -0,0 +1,101 @@
+# IPFS storage backend
+
+The `ipfs` feature of `quickwit-storage` adds an `ipfs://` storage backend that
+stores splits on an IPFS node, making every split content-addressed: each split
+file has a CID and can be pinned, replicated, and exported (e.g. as a CAR file)
+across the IPFS network.
+
+## Design
+
+### Why MFS
+
+Quickwit's read path is built around `Storage::get_slice(path, byte_range)`:
+one tail read fetches the split footer + hotcache, and subsequent small ranged
+reads are mostly served from the hotcache. Raw IPLD blocks are chunk-addressed,
+not byte-addressed, so something must translate byte ranges into block fetches.
+
+Rather than reimplementing that translation, the backend delegates it to the
+node's MFS (Mutable File System, the `/api/v0/files/*` RPC surface of
+[Kubo](https://github.com/ipfs/kubo)):
+
+- `files/write` chunks the payload into a UnixFS DAG and links it at an MFS
+  path. The DAG is the content-addressed artifact; the MFS path is the mutable
+  name Quickwit uses (`<index-uri>/<split-id>.split`).
+- `files/read?offset=&count=` resolves a byte range to the covering blocks
+  using the UnixFS index — the exact "contiguous byte range over a set of
+  blocks" view that `get_slice` needs.
+- `files/stat` returns the file size and its CID.
+
+### Mapping
+
+| Quickwit | IPFS |
+| --- | --- |
+| `ipfs://indexes/my-index` (index URI) | MFS directory `/indexes/my-index` |
+| `Storage::put` | `files/write` to `<path>.temp`, then `files/mv` into place (atomic publish: readers never see partial splits) |
+| `Storage::get_slice` | `files/read` with `offset`/`count` |
+| `Storage::get_all` / `copy_to` | `files/stat` (size) + streamed `files/read`, with a length-enforcing reader that fails on truncation |
+| `Storage::delete` | `files/rm --force` (missing entries are not an error) |
+| `Storage::file_num_bytes` | `files/stat` |
+| split CID | `IpfsStorage::file_cid(path)` (`files/stat`) |
+
+### Chunking and read amplification
+
+The dominant latency cost is the first read per split (footer + hotcache tail
+range). The default chunker is `size-1048576` (1 MiB fixed-size chunks) with
+raw leaves, so a typical tail read spans a small number of blocks that a local
+node serves in one RPC round-trip. Both knobs are exposed in
+`IpfsStorageConfig` (`chunker`, `raw_leaves`).
+
+### What stays conventional
+
+- **Metastore.** Metastore state is small, mutable, and rewritten constantly —
+  the opposite of content-addressed data. Keep it on PostgreSQL or a file
+  backend. `StorageResolver` and `MetastoreResolver` are independent, so
+  `index_uri: ipfs://...` + `metastore_uri: postgres://...` composes with no
+  coupling.
+- **Split cache.** The searcher split cache still materializes hot splits on
+  local disk; IPFS replaces the *origin* storage, not the local cache.
+- **Split identity.** A split's CID alone is not sufficient to open it: the
+  metastore row carries `footer_offsets`. Verifiable replication of an index
+  needs the split CIDs *plus* a metastore snapshot.
+
+## Operational notes
+
+- The backend talks to a single node's RPC API (`QW_IPFS_API_ENDPOINT` or
+  `storage.ipfs.api_endpoint`, default `http://127.0.0.1:5001`). Run the node
+  colocated with indexers/searchers; the IPFS network is the replication
+  layer, not the query path.
+- The RPC API grants full control of the node. Never expose port 5001 beyond
+  localhost / a private network.
+- The dev `kubo` service in `docker-compose.yml` runs with the `test` profile
+  and `--offline`: no bootstrap peers, no outbound traffic. Remove both to
+  join the public IPFS network — an explicit opt-in, since published CIDs
+  become discoverable.
+- Writes are staged to `<path>.temp` and moved into place; a crash can leave
+  `.temp` entries behind. They are overwritten (truncate) on retry and can be
+  garbage-collected safely.
+- **Deletes free disk space only after IPFS garbage collection.** The backend
+  does not use pins: files are GC-protected by being reachable from the MFS
+  root. `Storage::delete` (`files/rm`) unlinks the entry, which makes its
+  blocks GC-*eligible*, not deleted. Kubo does not GC by default, so on
+  merge-heavy indexes the blockstore grows until `ipfs repo gc` runs — run the
+  daemon with `--enable-gc` (or GC periodically) to reclaim space from deleted
+  and merged-away splits. Conversely, a split whose CID was explicitly pinned
+  (e.g. for replication) survives GC after deletion in Quickwit, by design.
+
+## Testing
+
+Unit tests: `cargo test -p quickwit-storage --features ipfs,testsuite ipfs`.
+
+Integration suite (requires a running node, e.g.
+`docker compose up -d kubo`):
+
+```bash
+cargo test -p quickwit-storage \
+  --features ipfs,integration-testsuite,ci-test \
+  --test ipfs_storage
+```
+
+This runs the generic `storage_test_suite` (put/get/slice/stream/delete/bulk
+delete/size/exists) plus single-part and 15 MB multi-part uploads and a CID
+round-trip against the node.

quickwit/.dockerignore

@@ -0,0 +1,6 @@
+target/
+**/target/
+.git/
+node_modules/
+quickwit-ui/node_modules/
+build.log

quickwit/quickwit-cli/Cargo.toml

@@ -92,6 +92,7 @@ jemalloc-profiled = [
   "quickwit-serve/jemalloc-profiled"
 ]
 ci-test = []
+ipfs = ["quickwit-storage/ipfs"]
 pprof = ["quickwit-serve/pprof"]
 openssl-support = ["openssl-probe"]
 # metrics here refers to adding support for metrics ingestion within Quickwit.
@@ -114,6 +115,7 @@ release-feature-set = [
   "quickwit-serve/lambda",
   "quickwit-storage/azure",
   "quickwit-storage/gcs",
+  "quickwit-storage/ipfs",
   "quickwit-metastore/postgres",
 ]
 release-feature-vendored-set = [
@@ -129,6 +131,7 @@ release-feature-vendored-set = [
   "quickwit-serve/lambda",
   "quickwit-storage/azure",
   "quickwit-storage/gcs",
+  "quickwit-storage/ipfs",
   "quickwit-metastore/postgres",
 ]
 release-macos-feature-vendored-set = [
@@ -143,6 +146,7 @@ release-macos-feature-vendored-set = [
   "quickwit-serve/lambda",
   "quickwit-storage/azure",
   "quickwit-storage/gcs",
+  "quickwit-storage/ipfs",
   "quickwit-metastore/postgres",
 ]
 release-jemalloc-profiled = [

quickwit/quickwit-common/src/uri.rs

@@ -37,6 +37,7 @@ pub enum Protocol {
     Ram = 6,
     S3 = 7,
     Google = 8,
+    Ipfs = 9,
 }
 
 impl Protocol {
@@ -50,6 +51,7 @@ impl Protocol {
             Protocol::Ram => "ram",
             Protocol::S3 => "s3",
             Protocol::Google => "gs",
+            Protocol::Ipfs => "ipfs",
         }
     }
 
@@ -62,7 +64,10 @@ impl Protocol {
     }
 
     pub fn is_object_storage(&self) -> bool {
-        matches!(&self, Protocol::Azure | Protocol::S3 | Protocol::Google)
+        matches!(
+            &self,
+            Protocol::Azure | Protocol::S3 | Protocol::Google | Protocol::Ipfs
+        )
     }
 
     pub fn is_database(&self) -> bool {
@@ -89,6 +94,7 @@ impl FromStr for Protocol {
             "ram" => Ok(Protocol::Ram),
             "s3" => Ok(Protocol::S3),
             "gs" => Ok(Protocol::Google),
+            "ipfs" => Ok(Protocol::Ipfs),
             _ => bail!("unknown URI protocol `{protocol}`"),
         }
     }
@@ -183,6 +189,9 @@ impl Uri {
         if protocol == Protocol::Google && path.components().count() < 2 {
             return None;
         }
+        if protocol == Protocol::Ipfs && path.components().count() < 2 {
+            return None;
+        }
         let parent_path = path.parent()?;
 
         Some(Self {
@@ -211,6 +220,9 @@ impl Uri {
         if self.protocol() == Protocol::Google && path.components().count() < 2 {
             return None;
         }
+        if self.protocol() == Protocol::Ipfs && path.components().count() < 2 {
+            return None;
+        }
         path.file_name().map(Path::new)
     }
 
@@ -508,6 +520,10 @@ mod tests {
             Uri::for_test("gs://bucket/key").protocol(),
             Protocol::Google
         );
+        assert_eq!(
+            Uri::for_test("ipfs://indexes/key").protocol(),
+            Protocol::Ipfs
+        );
         assert_eq!(
             Uri::for_test("postgres://localhost:5432/metastore").protocol(),
             Protocol::PostgreSQL
@@ -661,6 +677,16 @@ mod tests {
             Uri::for_test("gs://bucket/foo/bar/").parent().unwrap(),
             "gs://bucket/foo"
         );
+        assert!(Uri::for_test("ipfs://indexes").parent().is_none());
+        assert!(Uri::for_test("ipfs://indexes/").parent().is_none());
+        assert_eq!(
+            Uri::for_test("ipfs://indexes/foo").parent().unwrap(),
+            "ipfs://indexes"
+        );
+        assert_eq!(
+            Uri::for_test("ipfs://indexes/foo/bar").parent().unwrap(),
+            "ipfs://indexes/foo"
+        );
     }
 
     #[test]
@@ -733,6 +759,12 @@ mod tests {
             Uri::for_test("gs://bucket/foo/").file_name().unwrap(),
             Path::new("foo"),
         );
+        assert!(Uri::for_test("ipfs://indexes").file_name().is_none());
+        assert!(Uri::for_test("ipfs://indexes/").file_name().is_none());
+        assert_eq!(
+            Uri::for_test("ipfs://indexes/foo").file_name().unwrap(),
+            Path::new("foo"),
+        );
     }
 
     #[test]

quickwit/quickwit-config/src/lib.rs

@@ -83,8 +83,8 @@ pub use crate::serde_utils::HumanDuration;
 use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_8, VersionedSourceConfig};
 pub use crate::storage_config::{
     AzureStorageConfig, ChecksumAlgorithm, FileStorageConfig, GoogleCloudStorageConfig,
-    RamStorageConfig, S3StorageConfig, StorageBackend, StorageBackendFlavor, StorageConfig,
-    StorageConfigs,
+    IpfsStorageConfig, RamStorageConfig, S3StorageConfig, StorageBackend, StorageBackendFlavor,
+    StorageConfig, StorageConfigs,
 };
 
 /// Returns true if the ingest API v2 is enabled.