serve: add shared S3 hash-cache tier for multi-instance deployments

MODULE.bazel

@@ -58,10 +58,38 @@ maven.install(
         "org.apache.commons:commons-pool2:2.11.1",
         "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2",
         "org.mockito.kotlin:mockito-kotlin:5.4.0",
+        "software.amazon.awssdk:url-connection-client:2.46.7",
     ],
     fail_if_repin_required = True,
     lock_file = "//:maven_install.json",
 )
+
+# AWS SDK for the `serve` S3 hash-cache backend. The Netty/Apache HTTP clients are
+# excluded (the service explicitly uses the lightweight url-connection-client above)
+# to keep the deploy jar lean. `sts` is runtime-only: DefaultCredentialsProvider needs
+# it to assume the pod's IAM role via web identity (IRSA) when running on EKS.
+maven.artifact(
+    name = "bazel_diff_maven",
+    artifact = "s3",
+    exclusions = [
+        "software.amazon.awssdk:netty-nio-client",
+        "software.amazon.awssdk:apache-client",
+        "software.amazon.awssdk:apache5-client",
+    ],
+    group = "software.amazon.awssdk",
+    version = "2.46.7",
+)
+maven.artifact(
+    name = "bazel_diff_maven",
+    artifact = "sts",
+    exclusions = [
+        "software.amazon.awssdk:netty-nio-client",
+        "software.amazon.awssdk:apache-client",
+        "software.amazon.awssdk:apache5-client",
+    ],
+    group = "software.amazon.awssdk",
+    version = "2.46.7",
+)
 use_repo(
     maven,
     bazel_diff_maven = "bazel_diff_maven",

README.md

@@ -206,11 +206,41 @@ curl 'http://localhost:8080/metrics'
   "ready": true,
   "gitEngine": "subprocess",
   "trackDeps": false,
-  "cache": {"directory": "/var/cache/bazel-diff", "entries": 128, "sizeBytes": 4823913, "sizeHuman": "4.6 MB"},
+  "cache": {"directory": "/var/cache/bazel-diff", "remote": "s3://my-bucket/bazel-diff/", "entries": 128, "sizeBytes": 4823913, "sizeHuman": "4.6 MB"},
   "jvm": {"usedBytes": 123456789, "maxBytes": 2147483648}
 }
 ```
 
+### Shared S3 cache for multi-instance deployments
+
+A single instance caches hashes on local disk only. When you run several replicas behind a load
+balancer (e.g. a Kubernetes Deployment behind a Service, with the readiness probe on `/health`),
+give them a shared S3 cache tier so a revision is cold-hashed once fleet-wide instead of once per
+pod:
+
+```bash
+bazel-diff serve \
+  --workspacePath /path/to/workspace-clone \
+  --cacheDir /var/cache/bazel-diff \
+  --s3Bucket my-hash-cache-bucket \
+  --s3Prefix bazel-diff/my-repo
+```
+
+With `--s3Bucket` set the cache becomes two-tiered: reads check local disk first and fall back to
+the bucket (backfilling local disk on a hit), and every generated entry is published to both, so
+any replica can serve a revision another replica already hashed. Credentials and region resolve
+through the standard AWS default provider chains (environment variables, profile, IRSA web
+identity on EKS, IMDS), or pin the region with `--s3Region`. `--s3Endpoint` plus
+`--s3ForcePathStyle` point the client at an S3-compatible store (MinIO, LocalStack) for local
+testing.
+
+S3 errors never fail a request: a failed read is treated as a cache miss (the revision is
+regenerated — slower, but correct) and a failed write leaves the entry local-only, so an S3 outage
+degrades throughput rather than availability. Concurrent replicas racing to hash the same
+revision are also harmless — entries are deterministic per key, so last-write-wins over identical
+content. The `--cacheMax*` pruning flags bound the *local* tier only; bound the bucket with an S3
+lifecycle policy instead.
+
 Notes and current limitations:
 
 * Distance metrics (`/impacted_targets_with_distances`) require the dependency-edge graph, which is
@@ -234,10 +264,8 @@ Notes and current limitations:
   sweeper enforces the limits once at startup and then every `--cachePruneInterval` (default `1h`),
   evicting least-recently-used entries first — a cache hit refreshes an entry's recency, so revisions
   under active query are not expired out from under live traffic. With no `--cacheMax*` flag set the
-  cache is never pruned (the previous behavior). The cache layer is pluggable behind a byte-oriented
-  interface so a remote backend (e.g. S3) can be added without touching callers; such a backend
-  manages its own retention (e.g. a bucket lifecycle policy), and the in-process `--cacheMax*` flags
-  do not apply to it.
+  cache is never pruned (the previous behavior). The `--cacheMax*` flags always bound the local-disk
+  tier only; the shared S3 tier (see above) manages its own retention via a bucket lifecycle policy.
 * Query-affecting flags (`--useCquery`, `--fineGrainedHashExternalRepos`, etc.) mirror
   `generate-hashes`, and are folded into the cache key so a server started with different flags never
   serves another configuration's cached hashes.
@@ -248,8 +276,8 @@ Notes and current limitations:
   shared-base full-hash cache is not reused (each distinct changed-set re-hashes the base), but each
   such hash is cheaper because it skips reading unchanged files. The extra entries are bounded by the
   same LRU `--cacheMax*` pruning as everything else.
-* Containerization, multi-instance deployment manifests, and remote cache backends are not yet
-  included.
+* Containerization and multi-instance deployment manifests are not yet included; the shared S3
+  cache tier above is the building block for running replicas behind a load balancer.
 
 <!-- BEGIN_SECTION: cli-help -->
 ## CLI Interface
@@ -508,9 +536,9 @@ Command-line utility to analyze the state of the bazel build graph
 
 ```terminal
 Usage: bazel-diff serve [-hkvV] [--[no-]excludeExternalTargets]
-                        [--no-initial-fetch] [--[no-]trackDeps] [--[no-]
-                        useCquery] [-b=<bazelPath>] --cacheDir=<cacheDir>
-                        [--cacheMaxAge=<cacheMaxAge>]
+                        [--no-initial-fetch] [--[no-]s3ForcePathStyle] [--[no-]
+                        trackDeps] [--[no-]useCquery] [-b=<bazelPath>]
+                        --cacheDir=<cacheDir> [--cacheMaxAge=<cacheMaxAge>]
                         [--cacheMaxEntries=<cacheMaxEntries>]
                         [--cacheMaxSize=<cacheMaxSize>]
                         [--cachePruneInterval=<cachePruneInterval>]
@@ -519,7 +547,9 @@ Usage: bazel-diff serve [-hkvV] [--[no-]excludeExternalTargets]
                         [--fineGrainedHashExternalReposFile=<fineGrainedHashExte
                         rnalReposFile>] [--gitPath=<gitPath>] [--port=<port>]
                         [--requestTimeout=<requestTimeoutSeconds>]
-                        [-s=<seedFilepaths>] -w=<workspacePath>
+                        [-s=<seedFilepaths>] [--s3Bucket=<s3Bucket>]
+                        [--s3Endpoint=<s3Endpoint>] [--s3Prefix=<s3Prefix>]
+                        [--s3Region=<s3Region>] -w=<workspacePath>
                         [-co=<bazelCommandOptions>]...
                         [--cqueryCommandOptions=<cqueryCommandOptions>]...
                         [--fineGrainedHashExternalRepos=<fineGrainedHashExternal
@@ -602,6 +632,27 @@ targets between two git revisions, caching generated hashes per commit SHA.
   -s, --seed-filepaths=<seedFilepaths>
                             A text file with a newline separated list of
                               filepaths used as a SHA256 seed for all targets.
+      --s3Bucket=<s3Bucket> S3 bucket used as a shared hash cache behind the
+                              local --cacheDir tier. Generated hashes are
+                              published to the bucket and local cache misses
+                              fall back to it, so replicas behind a load
+                              balancer share one cache. Credentials and region
+                              come from the AWS default provider chains (env
+                              vars, profile, IRSA, IMDS). Unset (the default)
+                              means local-disk caching only.
+      --s3Endpoint=<s3Endpoint>
+                            Custom S3 endpoint URL for S3-compatible stores
+                              (MinIO, LocalStack). Usually combined with
+                              --s3ForcePathStyle.
+      --[no-]s3ForcePathStyle
+                            Use path-style S3 addressing (bucket in the URL
+                              path, not the hostname), required by most
+                              S3-compatible stores. Defaults to false.
+      --s3Prefix=<s3Prefix> Key prefix for cache objects in --s3Bucket, e.g.
+                              'bazel-diff/my-repo'. Defaults to no prefix
+                              (objects at the bucket root).
+      --s3Region=<s3Region> AWS region of --s3Bucket. Defaults to the SDK's
+                              default region chain (env vars, profile, IMDS).
       -so, --bazelStartupOptions=<bazelStartupOptions>
                             Additional space separated Bazel client startup
                               options used when invoking Bazel

cli/BUILD

@@ -30,6 +30,11 @@ kt_jvm_library(
     name = "cli-lib",
     srcs = glob(["src/main/kotlin/**/*.kt"]),
     resources = [":version_file"],
+    # sts is runtime-only: the SDK's DefaultCredentialsProvider loads it reflectively to
+    # assume a pod IAM role via web identity (IRSA) for the serve S3 cache backend.
+    runtime_deps = [
+        "@bazel_diff_maven//:software_amazon_awssdk_sts",
+    ],
     deps = [
         "//proto:analysis_v2_java_proto",
         "//proto:build_java_proto",
@@ -39,6 +44,10 @@ kt_jvm_library(
         "@bazel_diff_maven//:io_insert_koin_koin_core_jvm",
         "@bazel_diff_maven//:org_apache_commons_commons_pool2",
         "@bazel_diff_maven//:org_jetbrains_kotlinx_kotlinx_coroutines_core_jvm",
+        "@bazel_diff_maven//:software_amazon_awssdk_regions",
+        "@bazel_diff_maven//:software_amazon_awssdk_s3",
+        "@bazel_diff_maven//:software_amazon_awssdk_sdk_core",
+        "@bazel_diff_maven//:software_amazon_awssdk_url_connection_client",
     ],
 )
 
@@ -212,7 +221,9 @@ kt_jvm_test(
 
 kt_jvm_test(
     name = "E2ETest",
-    timeout = "long",
+    # "eternal": the suite runs ~1150s on CI's macos-latest x bazel 8.x cell,
+    # right at the "long" ceiling, so slow-runner jitter was timing it out.
+    timeout = "eternal",
     data = [":workspaces"],
     test_class = "com.bazel_diff.e2e.E2ETest",
     runtime_deps = [":cli-test-lib"],
@@ -242,6 +253,21 @@ kt_jvm_test(
     runtime_deps = [":cli-test-lib"],
 )
 
+kt_jvm_test(
+    name = "S3HashCacheStorageTest",
+    jvm_flags = [
+        "-Dnet.bytebuddy.experimental=true",
+    ],
+    test_class = "com.bazel_diff.server.S3HashCacheStorageTest",
+    runtime_deps = [":cli-test-lib"],
+)
+
+kt_jvm_test(
+    name = "TieredHashCacheStorageTest",
+    test_class = "com.bazel_diff.server.TieredHashCacheStorageTest",
+    runtime_deps = [":cli-test-lib"],
+)
+
 kt_jvm_test(
     name = "CachePrunerTest",
     test_class = "com.bazel_diff.server.CachePrunerTest",
@@ -302,6 +328,8 @@ kt_jvm_library(
         "@bazel_diff_maven//:io_insert_koin_koin_test_jvm",
         "@bazel_diff_maven//:junit_junit",
         "@bazel_diff_maven//:org_mockito_kotlin_mockito_kotlin",
+        "@bazel_diff_maven//:software_amazon_awssdk_s3",
+        "@bazel_diff_maven//:software_amazon_awssdk_sdk_core",
     ],
 )
 

cli/src/main/kotlin/com/bazel_diff/cli/ServeCommand.kt

@@ -22,7 +22,10 @@ import com.bazel_diff.server.LocalDiskHashCacheStorage
 import com.bazel_diff.server.MetricsService
 import com.bazel_diff.server.ProcessGitClient
 import com.bazel_diff.server.PrunableHashCacheStorage
+import com.bazel_diff.server.S3HashCacheStorage
+import com.bazel_diff.server.TieredHashCacheStorage
 import java.io.File
+import java.net.URI
 import java.nio.file.Path
 import java.time.Duration
 import java.util.concurrent.Callable
@@ -141,6 +144,51 @@ class ServeCommand : Callable<Int> {
       defaultValue = "1h")
   var cachePruneInterval: Duration = Duration.ofHours(1)
 
+  @CommandLine.Option(
+      names = ["--s3Bucket"],
+      description =
+          [
+              "S3 bucket used as a shared hash cache behind the local --cacheDir tier. Generated " +
+                  "hashes are published to the bucket and local cache misses fall back to it, so " +
+                  "replicas behind a load balancer share one cache. Credentials and region come " +
+                  "from the AWS default provider chains (env vars, profile, IRSA, IMDS). Unset " +
+                  "(the default) means local-disk caching only."])
+  var s3Bucket: String? = null
+
+  @CommandLine.Option(
+      names = ["--s3Prefix"],
+      description =
+          [
+              "Key prefix for cache objects in --s3Bucket, e.g. 'bazel-diff/my-repo'. Defaults to " +
+                  "no prefix (objects at the bucket root)."],
+      defaultValue = "")
+  var s3Prefix: String = ""
+
+  @CommandLine.Option(
+      names = ["--s3Region"],
+      description =
+          [
+              "AWS region of --s3Bucket. Defaults to the SDK's default region chain (env vars, " +
+                  "profile, IMDS)."])
+  var s3Region: String? = null
+
+  @CommandLine.Option(
+      names = ["--s3Endpoint"],
+      description =
+          [
+              "Custom S3 endpoint URL for S3-compatible stores (MinIO, LocalStack). Usually " +
+                  "combined with --s3ForcePathStyle."])
+  var s3Endpoint: URI? = null
+
+  @CommandLine.Option(
+      names = ["--s3ForcePathStyle"],
+      negatable = true,
+      description =
+          [
+              "Use path-style S3 addressing (bucket in the URL path, not the hostname), required " +
+                  "by most S3-compatible stores. Defaults to false."])
+  var s3ForcePathStyle = false
+
   @CommandLine.Option(
       names = ["--warmupRevision"],
       description =
@@ -284,14 +332,37 @@ class ServeCommand : Callable<Int> {
     }
 
     return try {
-      val server = buildAndStartServer(createGitClient(), LocalDiskHashCacheStorage(cacheDir))
+      val server = buildAndStartServer(createGitClient(), createStorage())
       awaitShutdown(server)
       CommandLine.ExitCode.OK
     } finally {
       stopKoin()
     }
   }
 
+  /**
+   * Builds the cache storage stack: the local `--cacheDir` tier, backed by the shared [remote] tier
+   * when one is configured. [remote] is injectable for tests; the default builds the S3 tier from
+   * the `--s3*` flags (null -- local-only -- when `--s3Bucket` is unset).
+   */
+  fun createStorage(remote: HashCacheStorage? = createS3Storage()): HashCacheStorage {
+    val local = LocalDiskHashCacheStorage(cacheDir)
+    return if (remote == null) local else TieredHashCacheStorage(local, remote)
+  }
+
+  /** The S3 remote cache tier from the `--s3*` flags, or null when `--s3Bucket` is unset. */
+  fun createS3Storage(): S3HashCacheStorage? =
+      s3Bucket?.let { bucket ->
+        S3HashCacheStorage(
+            S3HashCacheStorage.buildClient(s3Region, s3Endpoint, s3ForcePathStyle),
+            bucket,
+            s3Prefix)
+      }
+
+  /** Human-readable location of the shared remote cache tier for `/metrics`; null when unset. */
+  fun remoteCacheLocation(): String? =
+      s3Bucket?.let { "s3://$it/${S3HashCacheStorage.normalizePrefix(s3Prefix)}" }
+
   /**
    * Builds the [GitClient]. Git fetch/checkout operations shell out to the `git` binary at
    * [gitPath], so a `git` binary must be available on the host.
@@ -324,6 +395,7 @@ class ServeCommand : Callable<Int> {
             gitEngine = "subprocess",
             trackDeps = trackDeps,
             cacheDir = cacheDir.toString(),
+            remoteCache = remoteCacheLocation(),
             storage = storage,
             readiness = { ready.get() },
         )

cli/src/main/kotlin/com/bazel_diff/server/MetricsService.kt

@@ -18,10 +18,13 @@ data class ServerMetrics(
 
 /**
  * Footprint of the hash cache. [entries]/[sizeBytes]/[sizeHuman] are null when the backend does not
- * cheaply report its size (e.g. a remote store) -- see [MeasurableHashCacheStorage].
+ * cheaply report its size (e.g. a remote store) -- see [MeasurableHashCacheStorage]. [remote] is
+ * the shared remote tier's location (e.g. `s3://bucket/prefix/`) when one is configured, null for a
+ * local-only cache; the size fields always describe the local tier only.
  */
 data class CacheMetrics(
     val directory: String?,
+    val remote: String?,
     val entries: Long?,
     val sizeBytes: Long?,
     val sizeHuman: String?,
@@ -57,6 +60,7 @@ class MetricsService(
     private val trackDeps: Boolean,
     private val cacheDir: String,
     private val storage: HashCacheStorage,
+    private val remoteCache: String? = null,
     private val readiness: () -> Boolean,
     private val clock: () -> Long = System::currentTimeMillis,
 ) : MetricsProvider {
@@ -72,6 +76,7 @@ class MetricsService(
         cache =
             CacheMetrics(
                 directory = cacheDir,
+                remote = remoteCache,
                 entries = cacheStats?.entryCount,
                 sizeBytes = cacheStats?.totalBytes,
                 sizeHuman = cacheStats?.totalBytes?.let(::humanReadableBytes),