From aebbf7b89ec54bd5b38087eeb052f9766ef5ed05 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Fri, 14 Aug 2026 22:33:32 -0700 Subject: [PATCH 01/12] ADFA-5153: Decode Content rows against the shared Brotli dictionary WebServer now always decompresses brotli content server-side rather than ever passing compressed bytes through to the client -- sidesteps needing WebView-side dictionary support entirely, since the client never sees compressed bytes. It loads CompressionDictionary once at startup, and again on the debug-DB swap, and attaches it via brotli4j's attachDictionary before decoding -- falling back to plain decode if the table doesn't exist (a database that predates the dictionary migration). Confirmed cross-tool compatibility empirically: content compressed by OfflineDocumentationTools' brotli-CLI pipeline decodes byte-for-byte correctly via brotli4j's attachDictionary, and the same in-memory dictionary buffer is safe to reuse across many decode calls (WebServer holds one for its whole lifetime). BrotliDictionaryDecodeTest embeds those real cross-tool-produced fixtures as permanent regression coverage. Also adds testImplementation(libs.brotli4j.linux.x64): JVM unit tests exercising brotli4j's real native decoder had no native lib to load at all before this and would fail with UnsatisfiedLinkError -- a pre-existing gap, not introduced by this change, just never hit until now. docs/documentation-database.md updated for CompressionDictionary and WebServer's always-decompress behavior. --- app/build.gradle.kts | 5 + .../androidide/localWebServer/WebServer.kt | 65 +++++-- .../BrotliDictionaryDecodeTest.kt | 171 ++++++++++++++++++ docs/documentation-database.md | 5 +- 4 files changed, 229 insertions(+), 17 deletions(-) create mode 100644 app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 2f4fdf7ddc..7e33f57d32 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -337,6 +337,11 @@ dependencies { // brotli4j implementation(libs.brotli4j) + // JVM unit tests (e.g. BrotliDictionaryDecodeTest) run brotli4j's real native decoder, not an + // Android target -- without a desktop native on the test classpath, Brotli4jLoader has nothing + // to load and every such test fails with UnsatisfiedLinkError. Only linux-x64 is added since + // that's the only platform this project's dev machines/CI actually run JVM tests on. + testImplementation(libs.brotli4j.linux.x64) implementation(libs.common.markwon.core) implementation(libs.common.markwon.linkify) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index 0b76b64d2d..ec9fa6fd08 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -25,6 +25,7 @@ import java.net.InetSocketAddress import java.net.ServerSocket import java.net.Socket import java.net.URLDecoder +import java.nio.ByteBuffer import java.sql.Date import java.text.SimpleDateFormat import java.util.Locale @@ -76,6 +77,12 @@ class WebServer( private lateinit var serverSocket: ServerSocket private lateinit var database: SQLiteDatabase private var databaseTimestamp: Long = -1 + // The shared dictionary Content's brotli-compressed rows are compressed against (see + // ADFA-5153). Reloaded whenever `database` is (re)opened -- including the debug-override + // swap below -- since a different database file may have trained its own dictionary. + // Null (no dictionary attached, plain-brotli decode) if this database predates the + // dictionary-compression migration -- CompressionDictionary won't exist yet. + private var compressionDictionary: ByteBuffer? = null private val log = LoggerFactory.getLogger(WebServer::class.java) private val debugEnabled: Boolean = File(config.debugEnablePath).exists() @@ -85,8 +92,6 @@ class WebServer( // Frozen at startup; restart the server to pick up a change. private val clearCacheEnabled: Boolean = File(config.clearCacheEnablePath).exists() - private val encodingHeader: String = "Accept-Encoding" - private val brotliCompression: String = "br" private val pebbleEngine = PebbleEngine.Builder().loader(StringLoader()).build() private val templateCache = ConcurrentHashMap() private val gson: Gson = @@ -130,6 +135,37 @@ class WebServer( } } + /** + * Loads the shared Brotli dictionary Content's compressed rows are compressed against (see + * ADFA-5153) into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct + * buffer, a heap-backed one throws `IllegalArgumentException`. Returns null (logged once) + * if `CompressionDictionary` doesn't exist -- a database that predates the dictionary + * migration -- so callers fall back to plain, dictionary-free brotli decode. + */ + private fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { + val tableExists = + db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", + null, + ).use { it.moveToFirst() } + if (!tableExists) { + log.warn("CompressionDictionary table not found; decoding brotli content without a dictionary.") + return null + } + + return db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> + if (!cursor.moveToFirst()) { + log.warn("CompressionDictionary table is empty; decoding brotli content without a dictionary.") + return null + } + val bytes = cursor.getBlob(0) + ByteBuffer.allocateDirect(bytes.size).apply { + put(bytes) + flip() + } + } + } + /** * Stops the server by closing the listening socket. Safe to call from any thread. * Causes [start]'s accept loop to exit. If [start] hasn't bound the socket yet -- @@ -173,6 +209,7 @@ class WebServer( log.error("Cannot open database: {}", e.message) return } + compressionDictionary = loadCompressionDictionary(database) // NEW FEATURE: Log database metadata when debug is enabled if (debugEnabled) logDatabaseLastChanged() @@ -284,8 +321,6 @@ class WebServer( val writer = PrintWriter(output, true) if (debugEnabled) log.debug(" writer is {}.", writer) - var brotliSupported = false // assume nothing - // Read the request method line, it is always the first line of the request var requestLine = readLineFromStream(input) if (requestLine == null) { @@ -317,7 +352,6 @@ class WebServer( headers[requestLine.substring(0, colon).trim().lowercase()] = requestLine.substring(colon + 1).trim() } } - brotliSupported = headers["accept-encoding"]?.contains(brotliCompression) == true // Playground endpoint: POST only, handled before GET-only check if (false && path == "playground/execute") { @@ -337,6 +371,7 @@ class WebServer( database.close() database = SQLiteDatabase.openDatabase(config.debugDatabasePath, null, SQLiteDatabase.OPEN_READONLY) databaseTimestamp = debugDatabaseTimestamp + compressionDictionary = loadCompressionDictionary(database) } // Handle the special "pr" endpoint with highest priority @@ -406,15 +441,16 @@ class WebServer( dbContent = combined.toByteArray() } - // If a document is stored in brotli form and the client doesn't support that encoding - // decompress and send that to the client. - // Pebble templates have to be in string form so the retrieved database content may need to be - // decompressed. - if (compression == "brotli" && (!brotliSupported || templateId > 0)) { - dbContent = BrotliInputStream(ByteArrayInputStream(dbContent)).use { it.readBytes() } + // Content is compressed at rest with brotli, against the shared dictionary loaded + // into compressionDictionary (see ADFA-5153) -- this server always decompresses + // before responding, so it never needs to negotiate Content-Encoding with the client. + if (compression == "brotli") { + dbContent = + BrotliInputStream(ByteArrayInputStream(dbContent)).use { stream -> + compressionDictionary?.let { stream.attachDictionary(it) } + stream.readBytes() + } compression = "none" - } else if (compression == "brotli") { - compression = "br" } // If the file is associated with a template, instantiate that template and send the result to the client @@ -425,7 +461,6 @@ class WebServer( writer.println("HTTP/1.1 200 OK") writer.println("Content-Type: $dbMimeType") writer.println("Content-Length: ${dbContent.size}") - if (compression != "none") writer.println("Content-Encoding: $compression") writer.println("Connection: close") writer.println() writer.flush() @@ -446,7 +481,7 @@ class WebServer( * @param dbContent JSON bytes that will be parsed and supplied as the template context. * @param path The request/content path associated with this template (used for diagnostic/logging purposes). * @param dbMimeType The MIME type of the stored content (used for diagnostic/logging purposes). - * @param compression The compression label of the stored content (e.g., "br", "none") (used for diagnostic/logging purposes). + * @param compression The compression label of the stored content (always "none" by this point, since decompression already happened) (used for diagnostic/logging purposes). * @return The rendered template encoded as UTF-8 bytes. * @throws Exception If the template ID is not found, is duplicated in the database, or if template lookup/instantiation fails. */ diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt new file mode 100644 index 0000000000..916c28fa62 --- /dev/null +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -0,0 +1,171 @@ +package com.itsaky.androidide.localWebServer + +import com.aayushatharva.brotli4j.Brotli4jLoader +import com.aayushatharva.brotli4j.decoder.BrotliInputStream +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertThrows +import org.junit.BeforeClass +import org.junit.Test +import java.io.ByteArrayInputStream +import java.nio.ByteBuffer +import java.util.Base64 + +// Regression coverage for ADFA-5153: documentation.db's Content rows are Brotli-compressed +// against a shared dictionary trained by OfflineDocumentationTools' zstd/brotli CLI pipeline +// (see populate_db.py's DictionaryCompressor), not by brotli4j itself. These fixtures were +// produced by that exact pipeline, so this test is what protects the cross-tool contract: a +// brotli4j upgrade (or native lib change) that silently broke compatibility with the CLI-produced +// wire format would otherwise only surface as garbled content on-device. +private fun decodeBase64ToDirectBuffer(base64: String): ByteBuffer { + val bytes = Base64.getDecoder().decode(base64) + return ByteBuffer.allocateDirect(bytes.size).apply { + put(bytes) + flip() + } +} + +class BrotliDictionaryDecodeTest { + companion object { + // Unlike on-device (where ToolsManager/AssetsInstallationHelper already load it before + // WebServer ever runs), nothing loads brotli4j's native lib in a plain JVM unit test -- + // without this, every test below fails with UnsatisfiedLinkError instead of exercising + // real decode behavior. + @JvmStatic + @BeforeClass + fun loadNativeLibrary() { + Brotli4jLoader.ensureAvailability() + } + } + + // A ~3.3 KB zstd fast-cover dictionary trained on synthetic doc-page-like text, and a small + // payload Brotli-compressed against it via the `brotli` CLI's `-D` flag (OfflineDocumentationTools' + // actual encode path) -- see ADFA-5153. + private val dictionaryBase64 = + "N6Qw7OTyEGgfENCSpAP//////49QsrssRMqWGsnNSkLy/zfL/Ef3/zMAADhYoPCcRptTLgAEQIEAAMAS" + + "pykQlqZI41QGmTEGEAIAAAAAAAAAAAAAAABkXQEAAAAAAAAAAAAAAAAAAAABAAAABAAAAAgAAABhY2Ug" + + "dG9jLWVsZW1lbnQgZG9jcy1zaWRlYmFyIGludGVyZmFjZSB2YWwgZnVuIG9iamxlbWVudCB0b2MtZWxl" + + "bWVudCBrb3RsaW4gb3ZlcnJpZGUgdG9jLWVsZW1lbnQgb3ZlciBrb3RsaW4ga290bGluIHZhciBkb2Nz" + + "LXNpZGViYXIgdmFsIGNvbXBhbmlvbiBjb21wZSBmdW4gcGFnZS5wZWIga290bGluIGZ1biB2YXIgb2Jq" + + "ZWN0IHRlbXBsYXRlIGRvY24gdmFyIHRlbXBsYXRlIGludGVyZmFjZSBjb21wYW5pb24gcGFnZS5wZWIg" + + "dmFyIGlua290bGluIENvbnRlbnQtVHlwZSBkb2NzLXNpZGViYXIgbmF2IGludGVyZmFjZSBjb20gdG9j" + + "LWVsZW1lbnQgY29tcGFuaW9uIG9iamVjdCBpbnRlcmZhY2Uga290bGluIGRvY2RlYmFyIG5hdiB0b2Mt" + + "ZWxlbWVudCBDb250ZW50LVR5cGUgdGVtcGxhdGUgdmFyIGNsbiBzaWRlYmFyIHNpZGViYXIgdG9jLWVs" + + "ZW1lbnQgb2JqZWN0IGNvbXBhbmlvbiBpbnRycmlkZSB0b2MtZWxlbWVudCBmdW4gY2xhc3MgdGVtcGxh" + + "dGUgaW50ZXJmYWNlIGRvYyB0b2MtZWxlbWVudCBmdW4gdG9jLWVsZW1lbnQgdmFsIG9iamVjdCBvYmpl" + + "Y3QgdG9jYmplY3QgbmF2IGZ1biBzaWRlYmFyIG92ZXJyaWRlIG9iamVjdCBmdW4gdmFsIG92ZXJhdGUg" + + "aW50ZXJmYWNlIHZhciB0ZW1wbGF0ZSB0ZW1wbGF0ZSB2YXIgb2JqZWN0IGtvdGUga290bGluIG92ZXJy" + + "aWRlIHBhZ2UucGViIG92ZXJyaWRlIGZ1biBjbGFzcyB2YXIgaW50ZXJmYWNlIGNsYXNzIHRlbXBsYXRl" + + "IHNpZGViYXIgZnVuIHBhZ2UucGViIGRvY3NlIENvbnRlbnQtVHlwZSBpbnRlcmZhY2UgdGVtcGxhdGUg" + + "aW50ZXJmYWNlIHZhciB0ZWVudC1UeXBlIENvbnRlbnQtVHlwZSBvYmplY3QgcGFnZS5wZWIgdGVtcGxh" + + "dGUgb3ZlZW50LVR5cGUgb3ZlcnJpZGUgQ29udGVudC1UeXBlIHBhZ2UucGViIGNsYXNzIHNpZGVyIHRv" + + "Yy1lbGVtZW50IHZhciBzaWRlYmFyIG5hdiBmdW4gY2xhc3Mga290bGluIHBhZyBvdmVycmlkZSBpbnRl" + + "cmZhY2UgbmF2IHZhciBvdmVycmlkZSBjb21wYW5pb24gcGFnY2xhc3MgdmFsIGNsYXNzIENvbnRlbnQt" + + "VHlwZSBkb2NzLXNpZGViYXIgbmF2IGNvbXAgZnVuIHRlbXBsYXRlIHBhZ2UucGViIGNsYXNzIG5hdiBw" + + "YWdlLnBlYiBuYXYgQ29udCBjb21wYW5pb24gb3ZlcnJpZGUgdGVtcGxhdGUga290bGluIHNpZGViYXIg" + + "dmFyIHBhdmFsIG5hdiBjbGFzcyBmdW4gb3ZlcnJpZGUgaW50ZXJmYWNlIGludGVyZmFjZSBrb3RudGVu" + + "dC1UeXBlIENvbnRlbnQtVHlwZSBjbGFzcyBvYmplY3QgcGFnZS5wZWIgQ29udGJhciBzaWRlYmFyIHBh" + + "Z2UucGViIHZhbCBDb250ZW50LVR5cGUgdGVtcGxhdGUgdmFsbCBjb21wYW5pb24gZnVuIGRvY3Mtc2lk" + + "ZWJhciBjbGFzcyB0b2MtZWxlbWVudCBDb25kZWJhciB2YWwgZG9jcy1zaWRlYmFyIHZhciBDb250ZW50" + + "LVR5cGUgY2xhc3MgcGFnZXVuIHNpZGViYXIgQ29udGVudC1UeXBlIHZhbCBvYmplY3QgdGVtcGxhdGUg" + + "bmF2IG92ZmFjZSBDb250ZW50LVR5cGUgcGFnZS5wZWIga290bGluIGZ1biBvdmVycmlkZSB2YXJuaW9u" + + "IENvbnRlbnQtVHlwZSBrb3RsaW4gbmF2IHRvYy1lbGVtZW50IG9iamVjdCBvYmF2IG92ZXJyaWRlIHRv" + + "Yy1lbGVtZW50IHZhbCB2YWwgbmF2IG5hdiBvYmplY3QgcGFnbGluIGZ1biB2YWwgY2xhc3MgaW50ZXJm" + + "YWNlIHRvYy1lbGVtZW50IHNpZGViYXIgY29hdGUgc2lkZWJhciB2YXIgQ29udGVudC1UeXBlIGNvbXBh" + + "bmlvbiB2YXIgZnVuIHNpZCBrb3RsaW4gZnVuIENvbnRlbnQtVHlwZSBpbnRlcmZhY2UgdG9jLWVsZW1l" + + "bnQgZnVuYWdlLnBlYiB0ZW1wbGF0ZSBjb21wYW5pb24gdmFyIG92ZXJyaWRlIGtvdGxpbiBuYXZpbnRl" + + "cmZhY2UgZnVuIGludGVyZmFjZSBvYmplY3QgdGVtcGxhdGUgY2xhc3MgZG9jc2xpbiB0ZW1wbGF0ZSB0" + + "b2MtZWxlbWVudCB0b2MtZWxlbWVudCBuYXYga290bGluIGRvbmlvbiB0ZW1wbGF0ZSBvYmplY3QgY2xh" + + "c3Mgb2JqZWN0IENvbnRlbnQtVHlwZSBmdW5lY3QgY2xhc3MgY2xhc3MgdG9jLWVsZW1lbnQgY2xhc3Mg" + + "bmF2IHRlbXBsYXRlIENvbiBuYXYgdGVtcGxhdGUgZnVuIG5hdiBzaWRlYmFyIG92ZXJyaWRlIHZhbCBm" + + "dW4gdmFsZW50IGNsYXNzIHZhbCB2YXIgb2JqZWN0IGNsYXNzIGZ1biBrb3RsaW4gdmFsIGludGVvbXBh" + + "bmlvbiBjbGFzcyBrb3RsaW4gZnVuIGRvY3Mtc2lkZWJhciBrb3RsaW4gQ29udG4gZG9jcy1zaWRlYmFy" + + "IHRvYy1lbGVtZW50IG9iamVjdCB2YWwgbmF2IG5hdiBzaWRlciBDb250ZW50LVR5cGUgbmF2IHBhZ2Uu" + + "cGViIG5hdiBjbGFzcyBvdmVycmlkZSBzaWRpZGViYXIgb2JqZWN0IHNpZGViYXIgdmFsIG5hdiBpbnRl" + + "cmZhY2Ugb2JqZWN0IGRvYyBpbnRlcmZhY2Ugb3ZlcnJpZGUgcGFnZS5wZWIgb3ZlcnJpZGUgb3ZlcnJp" + + "ZGUgY2xhb2NzLXNpZGViYXIgY2xhc3MgY29tcGFuaW9uIGtvdGxpbiB0b2MtZWxlbWVudCBpbnQucGVi" + + "IHRvYy1lbGVtZW50IGNvbXBhbmlvbiBzaWRlYmFyIGRvY3Mtc2lkZWJhciBuYW1lbnQgcGFnZS5wZWIg" + + "dmFsIGtvdGxpbiBvYmplY3QgdmFyIHZhciBvYmplY3QgdGVtYWwgcGFnZS5wZWIgdmFyIHRvYy1lbGVt" + + "ZW50IHRlbXBsYXRlIHBhZ2UucGViIHNpZGVuYXYgcGFnZS5wZWIgdmFyIGtvdGxpbiBpbnRlcmZhY2Ug" + + "c2lkZWJhciB2YXIgY29tcGUga290bGluIGNsYXNzIHZhbCBzaWRlYmFyIHBhZ2UucGViIGludGVyZmFj" + + "ZSBwYWdlZ2UucGViIGNvbXBhbmlvbiBuYXYgb2JqZWN0IGNsYXNzIENvbnRlbnQtVHlwZSB0b2NiYXIg" + + "b3ZlcnJpZGUgdGVtcGxhdGUgdmFyIHNpZGViYXIga290bGluIGZ1biB2YXIgQ25pb24gdmFsIHBhZ2Uu" + + "cGViIGZ1biB0ZW1wbGF0ZSB0b2MtZWxlbWVudCB2YWwgY29tbnRlcmZhY2UgdmFsIGNsYXNzIGNvbXBh" + + "bmlvbiBzaWRlYmFyIHRlbXBsYXRlIGludGV2YWwgdGVtcGxhdGUgdGVtcGxhdGUgb2JqZWN0IG5hdiBk" + + "b2NzLXNpZGViYXIgc2lkZWUgY29tcGFuaW9uIG9iamVjdCBvdmVycmlkZSBmdW4gZnVuIGNvbXBhbmlv" + + "biB0b2MtVHlwZSBvdmVycmlkZSBuYXYgdmFsIHRvYy1lbGVtZW50IGtvdGxpbiB2YXIgbmF2IHBudC1U" + + "eXBlIHZhciBkb2NzLXNpZGViYXIgQ29udGVudC1UeXBlIHNpZGViYXIgcGFnZWViYXIgdmFsIHBhZ2Uu" + + "cGViIG9iamVjdCBmdW4gcGFnZS5wZWIgcGFnZS5wZWIgZG9jbiBvdmVycmlkZSBkb2NzLXNpZGViYXIg" + + "b2JqZWN0IGludGVyZmFjZSBjbGFzcyBrb3RhciB0ZW1wbGF0ZSB2YXIga290bGluIGNvbXBhbmlvbiBk" + + "b2NzLXNpZGViYXIgZnVuICB0b2MtZWxlbWVudCBkb2NzLXNpZGViYXIgaW50ZXJmYWNlIENvbnRlbnQt" + + "VHlwZSBj" + + private val compressedBase64 = + "H6AEIBypU5+7WdgVm1yEUcQuEA0twSdtb3qRIOfy83EJ6BCu9aGiz72LjySb9TQmV4wATYW9JhfwdjwI" + + "woRvurJjIaNH/hC6U59+QaiVFTX9XajztuGO9hS2C2GJEnZn+6vh0spFMR6RDFwzXTjCHWzxThsHAcW2" + + "9ev+Wau/71qnhgYFy8JNHS3F87DOOc02MhMXA9ZP9Ti9LOWqrKld7hlsgT8bDn888jGY1CPGtwU=" + + private val expectedBase64 = + "dmFsIG92ZXJyaWRlIGZ1biB2YXIgaW50ZXJmYWNlIHNpZGViYXIgaW50ZXJmYWNlIHNpZGViYXIgb2Jq" + + "ZWN0IGNsYXNzIGZ1biBDb250ZW50LVR5cGUgcGFnZS5wZWIgZnVuIHNpZGViYXIgaW50ZXJmYWNlIG92" + + "ZXJyaWRlIHNpZGViYXIgb3ZlcnJpZGUgZG9jcy1zaWRlYmFyIGtvdGxpbiBDb250ZW50LVR5cGUgdG9j" + + "LWVsZW1lbnQgb2JqZWN0IG92ZXJyaWRlIGNvbXBhbmlvbiBrb3RsaW4gZG9jcy1zaWRlYmFyIGtvdGxp" + + "biB2YWwgdG9jLWVsZW1lbnQgbmF2IGNvbXBhbmlvbiB2YXIgQ29udGVudC1UeXBlIG92ZXJyaWRlIGNs" + + "YXNzIGtvdGxpbiBuYXYgcGFnZS5wZWIgc2lkZWJhciBDb250ZW50LVR5cGUgb3ZlcnJpZGUgaW50ZXJm" + + "YWNlIHRvYy1lbGVtZW50IGludGVyZmFjZSBzaWRlYmFyIHNpZGViYXIgaW50ZXJmYWNlIG92ZXJyaWRl" + + "IHNpZGViYXIgc2lkZWJhciBmdW4gZG9jcy1zaWRlYmFyIHZhciB2YWwgY2xhc3MgZnVuIHBhZ2UucGVi" + + "IENvbnRlbnQtVHlwZSB2YWwgc2lkZWJhciB2YXIgaW50ZXJmYWNlIGNsYXNzIHRlbXBsYXRlIGludGVy" + + "ZmFjZSBmdW4gdG9jLWVsZW1lbnQgY2xhc3MgdmFsIHRlbXBsYXRlIHNpZGViYXIgY2xhc3MgbmF2IHNp" + + "ZGViYXIgdmFyIG9iamVjdCB2YXIgZG9jcy1zaWRlYmFyIHZhciBpbnRlcmZhY2UgdmFyIHRvYy1lbGVt" + + "ZW50IHRlbXBsYXRlIG9iamVjdCBjb21wYW5pb24ga290bGluIGNvbXBhbmlvbiBvdmVycmlkZSBpbnRl" + + "cmZhY2UgdmFsIG9iamVjdCB0ZW1wbGF0ZSBkb2NzLXNpZGViYXIgZG9jcy1zaWRlYmFyIGludGVyZmFj" + + "ZSBzaWRlYmFyIGRvY3Mtc2lkZWJhciBrb3RsaW4gdmFsIGZ1biBpbnRlcmZhY2UgdGVtcGxhdGUgaW50" + + "ZXJmYWNlIGludGVyZmFjZSBvdmVycmlkZSBkb2NzLXNpZGViYXIgc2lkZWJhciB2YWwgdmFsIG9iamVj" + + "dCBvYmplY3QgdGVtcGxhdGUgdmFsIGtvdGxpbiBuYXYgdGVtcGxhdGUgdGVtcGxhdGUgZnVuIHRvYy1l" + + "bGVtZW50IG92ZXJyaWRlIHRlbXBsYXRlIGludGVyZmFjZSB2YWwgb3ZlcnJpZGUgdmFyIHBhZ2UucGVi" + + "IHZhciBrb3RsaW4gdGVtcGxhdGUgdmFyIHRlbXBsYXRlIG5hdiBuYXYgdGVtcGxhdGUgQ29udGVudC1U" + + "eXBlIGtvdGxpbiB2YWwgaW50ZXJmYWNlIGRvY3Mtc2lkZWJhciBwYWdlLnBlYiBvYmplY3Qgb2JqZWN0" + + "IGZ1biBrb3RsaW4gc2lkZWJhciB2YXIgdGVtcGxhdGUgZG9jcy1zaWRlYmFy" + + @Test + fun `decodes CLI dictionary-compressed content correctly`() { + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val compressed = Base64.getDecoder().decode(compressedBase64) + val expected = Base64.getDecoder().decode(expectedBase64) + + val result = + BrotliInputStream(ByteArrayInputStream(compressed)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + + assertArrayEquals(expected, result) + } + + @Test + fun `the same dictionary buffer instance is safe to reuse across multiple decodes`() { + // WebServer holds one long-lived dictionary buffer across many requests -- + // this guards against a brotli4j change that mutates buffer position/limit + // state in a way that would break the second decode. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val compressed = Base64.getDecoder().decode(compressedBase64) + val expected = Base64.getDecoder().decode(expectedBase64) + + repeat(3) { + val result = + BrotliInputStream(ByteArrayInputStream(compressed)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + assertArrayEquals(expected, result) + } + } + + @Test + fun `decoding dictionary-compressed content without attaching a dictionary fails`() { + val compressed = Base64.getDecoder().decode(compressedBase64) + + assertThrows(Exception::class.java) { + BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() } + } + } +} diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 566703ad1b..168b2068c0 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -34,7 +34,7 @@ CREATE TABLE Content ( One row per file the web server can serve (HTML, CSS, JS, image, video, PDF, ...) — 30,000+ rows. Key points: - **`path`** is the lookup key (indexed via the `UNIQUE` constraint) and is what `WebServer` matches the HTTP request path against. Paths carry a short source prefix to avoid collisions between doc sets, e.g. `k/index.html` (Kotlin) vs `j/index.html` (Java). -- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. +- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every Brotli-compressed row is compressed against the single shared dictionary in `CompressionDictionary` (see below) — there is no plain, dictionary-free Brotli content left in this database (ADFA-5153 converted it all in one pass) and no per-row flag saying so, because a dictionary-compressed stream and a plain one are not distinguishable or interchangeable at decode time (verified empirically: attaching the wrong dictionary, or none, does not reliably fail loudly — it can silently decode to different bytes than were compressed). Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. - **`templateId`**: `0` (or unset) means `content` is legacy HTML with presentation baked in (the pre-CMS Release 0/1 format). A positive value means `content` is JSON *facts only*, rendered through the matching row in `Templates` (a Pebble template) — the ongoing move to a proper CMS that de-duplicates presentation across near-identical pages (e.g. `sin`/`cos` docs). - The `UNIQUE(path)` constraint rejects any duplicate `path`, regardless of `languageID` — a second language for an existing path isn't supported yet (only `EN-us` currently exists). Getting there needs an upstream schema change to composite uniqueness on `(path, languageID)` (see *Known rough edges* below). @@ -62,6 +62,7 @@ CREATE TABLE Tooltips ( ### Supporting tables +- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every `compression = 'brotli'` `Content` row is compressed against (see ADFA-5153). Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `WebServer` loads it once at startup (and again on the debug-DB swap) and attaches it via brotli4j's `attachDictionary` before decoding; a database predating this migration simply has no `CompressionDictionary` table, and `WebServer` falls back to plain (dictionary-free) decode. - **`Templates(id, name, content)`** — Pebble template source, keyed by id (and by `name` for well-known templates like `bookshelf`). Referenced by `Content.templateId`. - **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`). - **`LastChange(documentationSet, changeTime, who)`** — audit trail for edits made through `docdb-studio`; not shown to end users. `DatabaseVersionResolver` reads the `documentationSet = 'wholedb'` row to report the DB's build/edit stamp in debug logging, falling back to the most recent row of any set if `'wholedb'` is missing. @@ -80,7 +81,7 @@ All three sites below open the file with `SQLiteDatabase.openDatabase(..., OPEN_ AND C.path = ? ``` - then reassembles chunked blobs, decompresses Brotli when the client can't accept it (or when a Pebble template needs a string to render), and instantiates the template if `templateId > 0`. Also serves a Dynamic Bookshelf JSON payload (joining `Content`/`Bookshelf`/`BookCategories`, rendered through the `bookshelf` template) and debug-only HTML dumps at `/pr/db` (`LastChange`, last 20 rows) and `/pr/pr` (recent projects, from a *different* database). + then reassembles chunked blobs, always decompresses Brotli content (attaching `CompressionDictionary`'s bytes first, if loaded — see above) since this server never negotiates `Content-Encoding` with the client, and instantiates the template if `templateId > 0`. Also serves a Dynamic Bookshelf JSON payload (joining `Content`/`Bookshelf`/`BookCategories`, rendered through the `bookshelf` template) and debug-only HTML dumps at `/pr/db` (`LastChange`, last 20 rows) and `/pr/pr` (recent projects, from a *different* database). - **`idetooltips/.../ToolTipManager.kt`** — serves Tier 1/2. Looks up `Tooltips` joined to `TooltipCategories` by `(category, tag)`, then `TooltipButtons` for the Tier 3 links shown at the bottom. - **`plugin-manager/.../documentation/PluginDocumentationManager.kt`** (with `Tier3AssetWalker.kt`, and the `DocumentationExtension` contract in `plugin-api`) — lets plugins contribute their own help content into the same lookup paths. From d9b82afa1496bbec4cdbbfe7b7fdc3a9775edee6 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Fri, 14 Aug 2026 23:12:44 -0700 Subject: [PATCH 02/12] Apply spotlessApply formatting Co-Authored-By: Claude Sonnet 5 --- .../com/itsaky/androidide/localWebServer/WebServer.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index ec9fa6fd08..37a0746fea 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -77,6 +77,7 @@ class WebServer( private lateinit var serverSocket: ServerSocket private lateinit var database: SQLiteDatabase private var databaseTimestamp: Long = -1 + // The shared dictionary Content's brotli-compressed rows are compressed against (see // ADFA-5153). Reloaded whenever `database` is (re)opened -- including the debug-override // swap below -- since a different database file may have trained its own dictionary. @@ -144,10 +145,11 @@ class WebServer( */ private fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { val tableExists = - db.rawQuery( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", - null, - ).use { it.moveToFirst() } + db + .rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", + null, + ).use { it.moveToFirst() } if (!tableExists) { log.warn("CompressionDictionary table not found; decoding brotli content without a dictionary.") return null From bfb3baa870015075ef8837d6242b0c5611761872 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 16 Aug 2026 16:05:49 -0700 Subject: [PATCH 03/12] ADFA-5153: Narrow no-dictionary decode test to IOException CodeRabbit flagged this test as asserting an unsupported invariant, citing docs/documentation-database.md's claim that "wrong dictionary, or none" doesn't reliably fail loudly. Verified empirically that the two cases are actually distinct: a wrong dictionary decodes silently to incorrect bytes (its distances resolve into real, just wrong, bytes), but no dictionary at all reliably throws IOException, since distances into the dictionary region are out of bounds for any spec-compliant decoder. Narrowed the assertion from Exception to IOException and corrected the doc to describe both failure modes instead of conflating them. Co-Authored-By: Claude Sonnet 5 --- .../localWebServer/BrotliDictionaryDecodeTest.kt | 9 ++++++++- docs/documentation-database.md | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt index 916c28fa62..2a445dd522 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -7,6 +7,7 @@ import org.junit.Assert.assertThrows import org.junit.BeforeClass import org.junit.Test import java.io.ByteArrayInputStream +import java.io.IOException import java.nio.ByteBuffer import java.util.Base64 @@ -162,9 +163,15 @@ class BrotliDictionaryDecodeTest { @Test fun `decoding dictionary-compressed content without attaching a dictionary fails`() { + // Unlike a *wrong* dictionary (whose backward distances resolve into real, + // just incorrect, bytes -- silently wrong output, no error), decoding with + // no dictionary at all leaves distances that reach into the dictionary + // region out of bounds for any spec-compliant decoder, which must reject + // the stream as corrupt. Verified empirically: brotli4j throws IOException + // here, not an arbitrary Exception subtype. val compressed = Base64.getDecoder().decode(compressedBase64) - assertThrows(Exception::class.java) { + assertThrows(IOException::class.java) { BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() } } } diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 168b2068c0..8fda587afc 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -34,7 +34,7 @@ CREATE TABLE Content ( One row per file the web server can serve (HTML, CSS, JS, image, video, PDF, ...) — 30,000+ rows. Key points: - **`path`** is the lookup key (indexed via the `UNIQUE` constraint) and is what `WebServer` matches the HTTP request path against. Paths carry a short source prefix to avoid collisions between doc sets, e.g. `k/index.html` (Kotlin) vs `j/index.html` (Java). -- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every Brotli-compressed row is compressed against the single shared dictionary in `CompressionDictionary` (see below) — there is no plain, dictionary-free Brotli content left in this database (ADFA-5153 converted it all in one pass) and no per-row flag saying so, because a dictionary-compressed stream and a plain one are not distinguishable or interchangeable at decode time (verified empirically: attaching the wrong dictionary, or none, does not reliably fail loudly — it can silently decode to different bytes than were compressed). Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. +- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every Brotli-compressed row is compressed against the single shared dictionary in `CompressionDictionary` (see below) — there is no plain, dictionary-free Brotli content left in this database (ADFA-5153 converted it all in one pass) and no per-row flag saying so, because a dictionary-compressed stream and a plain one are not distinguishable or interchangeable at decode time. Verified empirically, the two failure modes differ: attaching the *wrong* dictionary decodes without error to different bytes than were compressed (its backward distances resolve into real, just incorrect, bytes) — but attaching *no* dictionary reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are then out of bounds for any spec-compliant decoder. Either way, don't rely on decode success/failure to distinguish dictionary-compressed content from plain — only the wrong-dictionary case is silent. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. - **`templateId`**: `0` (or unset) means `content` is legacy HTML with presentation baked in (the pre-CMS Release 0/1 format). A positive value means `content` is JSON *facts only*, rendered through the matching row in `Templates` (a Pebble template) — the ongoing move to a proper CMS that de-duplicates presentation across near-identical pages (e.g. `sin`/`cos` docs). - The `UNIQUE(path)` constraint rejects any duplicate `path`, regardless of `languageID` — a second language for an existing path isn't supported yet (only `EN-us` currently exists). Getting there needs an upstream schema change to composite uniqueness on `(path, languageID)` (see *Known rough edges* below). From 3465bffbc0dafd9b261487059503e493c9328cfa Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 16 Aug 2026 21:11:13 -0700 Subject: [PATCH 04/12] ADFA-5153: Address code-review findings on the dictionary compression PR Fixes 13 findings from a max-effort /code-review pass, most significant first: - Plugin-contributed Tier 3 docs (PluginDocumentationManager/BrotliCompressor) are plain brotli with no dictionary, but WebServer unconditionally attached the shared dictionary before decoding any brotli row -- every such page 500'd. Extracted decompressBrotli(): tries the dictionary first, falls back to a plain decode on IOException. Verified empirically that a dictionary attached to a stream compressed without one reliably throws rather than silently decoding wrong bytes, so this fallback never lets a real dictionary-compressed row slip through unnoticed. - loadCompressionDictionary() now wraps its whole body in one catch-all, matching DatabaseVersionResolver's existing pattern, instead of hand-anticipating individual failure cases. Fixes three related bugs this gap caused: a failed dictionary reload during the debug-DB swap left stale state with no retry; a dictionary-load failure at server startup aborted the entire server with no retry; a NULL dictionary blob threw an uncaught NPE. - Extracted switchToDatabase() so database/databaseTimestamp/ compressionDictionary/templateCache/bookshelfTemplateId are all swapped atomically in one place instead of duplicated across start() and the debug-swap block -- also fixes templateCache never being invalidated on a debug-DB swap, and a reopen-after-close ordering bug where a failed reopen left `database` referencing an already-closed handle. - Added test coverage for the previously-untested no-dictionary/plugin-content decode path. - Corrected docs/documentation-database.md's false "no dictionary-free content left" claim (contradicted by its own PluginDocumentationManager section) and the build.gradle.kts comment falsely claiming linux-x64 is the only platform this project's dev machines run JVM tests on. - Minor: deduped the byte[]->direct-ByteBuffer idiom, removed a stale Accept-Encoding comment on a header no longer read. Separately discovered (not caused by this PR, filed as ADFA-5168 instead of fixed here): :app:testV8DebugUnitTest is flaky (~50% of full-suite runs) due to Brotli4jLoader static state shared across one JVM test process between AssetsInstallationHelperTest's mockkStatic and BrotliDictionaryDecodeTest's real native load -- confirmed present on bfb3baa87 already, independent of any change in this commit. Co-Authored-By: Claude Sonnet 5 --- app/build.gradle.kts | 6 +- .../androidide/localWebServer/WebServer.kt | 150 +++++++++++++----- .../BrotliDictionaryDecodeTest.kt | 30 ++++ docs/documentation-database.md | 4 +- 4 files changed, 146 insertions(+), 44 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 7e33f57d32..f54647a119 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -339,8 +339,10 @@ dependencies { implementation(libs.brotli4j) // JVM unit tests (e.g. BrotliDictionaryDecodeTest) run brotli4j's real native decoder, not an // Android target -- without a desktop native on the test classpath, Brotli4jLoader has nothing - // to load and every such test fails with UnsatisfiedLinkError. Only linux-x64 is added since - // that's the only platform this project's dev machines/CI actually run JVM tests on. + // to load and every such test fails with UnsatisfiedLinkError. Only linux-x64 is added, matching + // this repo's CI runners (ubuntu-latest); a contributor running :app:test on macOS/Windows/arm64 + // needs the matching libs.brotli4j.* native added locally (see build-logic/plugins' build.gradle.kts + // for the OS/arch dispatch pattern) or to run the suite in CI/a Linux x64 environment instead. testImplementation(libs.brotli4j.linux.x64) implementation(libs.common.markwon.core) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index 37a0746fea..d043775f56 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -18,6 +18,7 @@ import org.slf4j.LoggerFactory import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.File +import java.io.IOException import java.io.InputStream import java.io.PrintWriter import java.io.StringWriter @@ -63,6 +64,16 @@ data class JavaExecutionResult( val timeoutLimit: Long, ) +/** + * Copies [bytes] into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct + * buffer, a heap-backed one throws `IllegalArgumentException`. + */ +internal fun toDirectByteBuffer(bytes: ByteArray): ByteBuffer = + ByteBuffer.allocateDirect(bytes.size).apply { + put(bytes) + flip() + } + class WebServer( private val config: ServerConfig, ) { @@ -137,35 +148,99 @@ class WebServer( } /** - * Loads the shared Brotli dictionary Content's compressed rows are compressed against (see - * ADFA-5153) into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct - * buffer, a heap-backed one throws `IllegalArgumentException`. Returns null (logged once) - * if `CompressionDictionary` doesn't exist -- a database that predates the dictionary - * migration -- so callers fall back to plain, dictionary-free brotli decode. + * Loads the shared Brotli dictionary most Content rows are compressed against (see ADFA-5153). + * Returns null (logged) on any failure to load one -- a database that predates the dictionary + * migration, a schema/row anomaly, or any other error -- so callers always fall back to plain, + * dictionary-free brotli decode rather than propagating the failure (see [decompressBrotli]). */ private fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { - val tableExists = - db - .rawQuery( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", - null, - ).use { it.moveToFirst() } - if (!tableExists) { - log.warn("CompressionDictionary table not found; decoding brotli content without a dictionary.") - return null + // Whole body wrapped in one catch-all (matching DatabaseVersionResolver.resolveDatabaseVersion's + // pattern) rather than hand-anticipating individual SQLiteExceptions: a caller-visible failure + // here must never abort start() or leave a stale dictionary un-retried after a debug-DB swap -- + // falling back to null (plain, dictionary-free decode) is always the safe choice. + return try { + val tableExists = + db + .rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", + null, + ).use { it.moveToFirst() } + if (!tableExists) { + log.warn("CompressionDictionary table not found; decoding brotli content without a dictionary.") + return null + } + + db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> + if (!cursor.moveToFirst()) { + log.warn("CompressionDictionary table is empty; decoding brotli content without a dictionary.") + return null + } + val bytes = cursor.getBlob(0) + if (bytes == null) { + log.warn("CompressionDictionary row has a NULL data column; decoding brotli content without a dictionary.") + return null + } + toDirectByteBuffer(bytes) + } + } catch (e: Exception) { + log.error("Could not load compression dictionary; decoding brotli content without a dictionary: {}", e.message) + null } + } - return db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> - if (!cursor.moveToFirst()) { - log.warn("CompressionDictionary table is empty; decoding brotli content without a dictionary.") - return null + /** + * Opens [path] as the active database, refreshing every piece of state that depends on which + * database file is active -- [databaseTimestamp], [compressionDictionary], and the per-database + * caches [bookshelfTemplateId]/[templateCache] -- as one atomic operation, so a request never + * observes a database swapped in without its matching dictionary/caches. Only closes the + * previous database once the new one has opened successfully, so a failed swap (this throws) + * leaves the previous, still-open database serving requests rather than leaving [database] + * referencing an already-closed handle. + */ + private fun switchToDatabase( + path: String, + timestamp: Long, + ) { + val newDatabase = SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY) + if (::database.isInitialized) { + try { + database.close() + } catch (e: Exception) { + log.error("Cannot close previous database: {}", e.message) } - val bytes = cursor.getBlob(0) - ByteBuffer.allocateDirect(bytes.size).apply { - put(bytes) - flip() + } + database = newDatabase + databaseTimestamp = timestamp + compressionDictionary = loadCompressionDictionary(database) + bookshelfTemplateId = -1 + templateCache.clear() + } + + /** + * Decompresses one Brotli-compressed Content row. Tries the shared dictionary first, since every + * ADFA-5153-migrated row requires it, then falls back to a plain decode for rows that were never + * dictionary-compressed: plugin-contributed Tier 3 docs (PluginDocumentationManager/BrotliCompressor + * compress with no dictionary) or any row served from a pre-migration database. Attaching a + * dictionary to a stream that wasn't compressed against one reliably fails to decode rather than + * silently producing wrong bytes (verified empirically -- see docs/documentation-database.md), so + * this ordering never lets a dictionary-compressed row fall through to the plain path by accident. + */ + private fun decompressBrotli(content: ByteArray): ByteArray { + val dictionary = compressionDictionary + if (dictionary != null) { + try { + return BrotliInputStream(ByteArrayInputStream(content)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + } catch (e: IOException) { + log.debug( + "Dictionary decode failed for a brotli row (likely dictionary-free plugin content); retrying without a dictionary: {}", + e.message, + ) } } + return BrotliInputStream(ByteArrayInputStream(content)).use { it.readBytes() } } /** @@ -203,15 +278,12 @@ class WebServer( config.experimentsEnablePath, ) - databaseTimestamp = getDatabaseTimestamp(config.databasePath) - try { - database = SQLiteDatabase.openDatabase(config.databasePath, null, SQLiteDatabase.OPEN_READONLY) + switchToDatabase(config.databasePath, getDatabaseTimestamp(config.databasePath)) } catch (e: Exception) { log.error("Cannot open database: {}", e.message) return } - compressionDictionary = loadCompressionDictionary(database) // NEW FEATURE: Log database metadata when debug is enabled if (debugEnabled) logDatabaseLastChanged() @@ -343,7 +415,7 @@ class WebServer( var path = parts[1].split("?")[0] // Discard any HTTP query parameters. path = path.substring(1) - // Read all headers until blank line (needed for Content-Length on POST and Accept-Encoding on GET) + // Read all headers until blank line (needed for Content-Length on POST) val headers = mutableMapOf() while (true) { requestLine = readLineFromStream(input) ?: break @@ -369,11 +441,11 @@ class WebServer( // if there is use that for our responses val debugDatabaseTimestamp = getDatabaseTimestamp(config.debugDatabasePath, true) if (debugDatabaseTimestamp > databaseTimestamp) { - bookshelfTemplateId = -1 - database.close() - database = SQLiteDatabase.openDatabase(config.debugDatabasePath, null, SQLiteDatabase.OPEN_READONLY) - databaseTimestamp = debugDatabaseTimestamp - compressionDictionary = loadCompressionDictionary(database) + try { + switchToDatabase(config.debugDatabasePath, debugDatabaseTimestamp) + } catch (e: Exception) { + log.error("Cannot swap to debug database '{}': {}", config.debugDatabasePath, e.message) + } } // Handle the special "pr" endpoint with highest priority @@ -443,15 +515,13 @@ class WebServer( dbContent = combined.toByteArray() } - // Content is compressed at rest with brotli, against the shared dictionary loaded - // into compressionDictionary (see ADFA-5153) -- this server always decompresses - // before responding, so it never needs to negotiate Content-Encoding with the client. + // Content is compressed at rest with brotli -- most rows against the shared dictionary + // loaded into compressionDictionary (see ADFA-5153), but plugin-contributed Tier 3 docs + // (PluginDocumentationManager/BrotliCompressor) are plain brotli with no dictionary. + // This server always decompresses before responding, so it never needs to negotiate + // Content-Encoding with the client. if (compression == "brotli") { - dbContent = - BrotliInputStream(ByteArrayInputStream(dbContent)).use { stream -> - compressionDictionary?.let { stream.attachDictionary(it) } - stream.readBytes() - } + dbContent = decompressBrotli(dbContent) compression = "none" } diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt index 2a445dd522..a5a9bd8735 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -2,13 +2,17 @@ package com.itsaky.androidide.localWebServer import com.aayushatharva.brotli4j.Brotli4jLoader import com.aayushatharva.brotli4j.decoder.BrotliInputStream +import com.aayushatharva.brotli4j.encoder.BrotliOutputStream +import com.aayushatharva.brotli4j.encoder.Encoder import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertThrows import org.junit.BeforeClass import org.junit.Test import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream import java.io.IOException import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets import java.util.Base64 // Regression coverage for ADFA-5153: documentation.db's Content rows are Brotli-compressed @@ -175,4 +179,30 @@ class BrotliDictionaryDecodeTest { BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() } } } + + @Test + fun `dictionary-free plugin content fails with a dictionary attached but decodes plain`() { + // Regression coverage for the WebServer.decompressBrotli fallback: plugin-contributed + // Tier 3 docs (PluginDocumentationManager/BrotliCompressor) are compressed with the same + // encoder params (quality 11, window 24) but no dictionary, coexisting in the same Content + // table as ADFA-5153-migrated, dictionary-compressed rows. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val plaintext = "plugin-contributed Tier 3 content, compressed with no dictionary" + val expected = plaintext.toByteArray(StandardCharsets.UTF_8) + val compressed = + ByteArrayOutputStream() + .apply { + BrotliOutputStream(this, Encoder.Parameters().setQuality(11).setWindow(24)).use { it.write(expected) } + }.toByteArray() + + assertThrows(IOException::class.java) { + BrotliInputStream(ByteArrayInputStream(compressed)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + } + + val plainResult = BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() } + assertArrayEquals(expected, plainResult) + } } diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 8fda587afc..33ea4df661 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -34,7 +34,7 @@ CREATE TABLE Content ( One row per file the web server can serve (HTML, CSS, JS, image, video, PDF, ...) — 30,000+ rows. Key points: - **`path`** is the lookup key (indexed via the `UNIQUE` constraint) and is what `WebServer` matches the HTTP request path against. Paths carry a short source prefix to avoid collisions between doc sets, e.g. `k/index.html` (Kotlin) vs `j/index.html` (Java). -- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every Brotli-compressed row is compressed against the single shared dictionary in `CompressionDictionary` (see below) — there is no plain, dictionary-free Brotli content left in this database (ADFA-5153 converted it all in one pass) and no per-row flag saying so, because a dictionary-compressed stream and a plain one are not distinguishable or interchangeable at decode time. Verified empirically, the two failure modes differ: attaching the *wrong* dictionary decodes without error to different bytes than were compressed (its backward distances resolve into real, just incorrect, bytes) — but attaching *no* dictionary reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are then out of bounds for any spec-compliant decoder. Either way, don't rely on decode success/failure to distinguish dictionary-compressed content from plain — only the wrong-dictionary case is silent. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. +- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every row shipped in this database is Brotli-compressed against the single shared dictionary in `CompressionDictionary` (see below), converted in one pass by ADFA-5153 — but plugin-contributed Tier 3 rows (`PluginDocumentationManager`/`BrotliCompressor`, see below) are plain, dictionary-free Brotli, and there is no per-row flag distinguishing the two, because a dictionary-compressed stream and a plain one are not distinguishable at decode time by inspection. They *are* distinguishable by attempting the decode: attaching the *wrong* dictionary decodes without error to different bytes than were compressed (its backward distances resolve into real, just incorrect, bytes) — but attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are then out of bounds for any spec-compliant decoder. `WebServer` relies on exactly this: it tries the dictionary first and falls back to a plain decode on `IOException`, which correctly handles both dictionary-compressed and plain rows — but never rely on decode success/failure to detect a *wrong* dictionary, since that case is silent. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. - **`templateId`**: `0` (or unset) means `content` is legacy HTML with presentation baked in (the pre-CMS Release 0/1 format). A positive value means `content` is JSON *facts only*, rendered through the matching row in `Templates` (a Pebble template) — the ongoing move to a proper CMS that de-duplicates presentation across near-identical pages (e.g. `sin`/`cos` docs). - The `UNIQUE(path)` constraint rejects any duplicate `path`, regardless of `languageID` — a second language for an existing path isn't supported yet (only `EN-us` currently exists). Getting there needs an upstream schema change to composite uniqueness on `(path, languageID)` (see *Known rough edges* below). @@ -62,7 +62,7 @@ CREATE TABLE Tooltips ( ### Supporting tables -- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every `compression = 'brotli'` `Content` row is compressed against (see ADFA-5153). Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `WebServer` loads it once at startup (and again on the debug-DB swap) and attaches it via brotli4j's `attachDictionary` before decoding; a database predating this migration simply has no `CompressionDictionary` table, and `WebServer` falls back to plain (dictionary-free) decode. +- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `WebServer` loads it once at startup (and again on the debug-DB swap) and, per row, tries decoding with it attached first via brotli4j's `attachDictionary`, falling back to a plain decode on failure — needed both for a database predating this migration (no `CompressionDictionary` table at all) and for plugin-contributed rows within an otherwise-migrated database (see `PluginDocumentationManager` below). - **`Templates(id, name, content)`** — Pebble template source, keyed by id (and by `name` for well-known templates like `bookshelf`). Referenced by `Content.templateId`. - **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`). - **`LastChange(documentationSet, changeTime, who)`** — audit trail for edits made through `docdb-studio`; not shown to end users. `DatabaseVersionResolver` reads the `documentationSet = 'wholedb'` row to report the DB's build/edit stamp in debug logging, falling back to the most recent row of any set if `'wholedb'` is missing. From e901f6ca3c97561a355c2956f4d0c55800872d5f Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 16 Aug 2026 21:19:10 -0700 Subject: [PATCH 05/12] ADFA-5153: Scope shared-dictionary claim to migrated brotli rows CodeRabbit caught a self-contradiction: line 34 already says non-Brotli content uses format-specific compression, but the prior wording said 'every row' is dictionary-compressed. Scoped to migrated Content rows with ContentTypes.compression = 'brotli'. Co-Authored-By: Claude Sonnet 5 --- docs/documentation-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 33ea4df661..9352a235b3 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -34,7 +34,7 @@ CREATE TABLE Content ( One row per file the web server can serve (HTML, CSS, JS, image, video, PDF, ...) — 30,000+ rows. Key points: - **`path`** is the lookup key (indexed via the `UNIQUE` constraint) and is what `WebServer` matches the HTTP request path against. Paths carry a short source prefix to avoid collisions between doc sets, e.g. `k/index.html` (Kotlin) vs `j/index.html` (Java). -- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every row shipped in this database is Brotli-compressed against the single shared dictionary in `CompressionDictionary` (see below), converted in one pass by ADFA-5153 — but plugin-contributed Tier 3 rows (`PluginDocumentationManager`/`BrotliCompressor`, see below) are plain, dictionary-free Brotli, and there is no per-row flag distinguishing the two, because a dictionary-compressed stream and a plain one are not distinguishable at decode time by inspection. They *are* distinguishable by attempting the decode: attaching the *wrong* dictionary decodes without error to different bytes than were compressed (its backward distances resolve into real, just incorrect, bytes) — but attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are then out of bounds for any spec-compliant decoder. `WebServer` relies on exactly this: it tries the dictionary first and falls back to a plain decode on `IOException`, which correctly handles both dictionary-compressed and plain rows — but never rely on decode success/failure to detect a *wrong* dictionary, since that case is silent. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. +- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every migrated `Content` row with `ContentTypes.compression = 'brotli'` is Brotli-compressed against the single shared dictionary in `CompressionDictionary` (see below), converted in one pass by ADFA-5153 — but plugin-contributed Tier 3 rows (`PluginDocumentationManager`/`BrotliCompressor`, see below) are plain, dictionary-free Brotli, and there is no per-row flag distinguishing the two, because a dictionary-compressed stream and a plain one are not distinguishable at decode time by inspection. They *are* distinguishable by attempting the decode: attaching the *wrong* dictionary decodes without error to different bytes than were compressed (its backward distances resolve into real, just incorrect, bytes) — but attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are then out of bounds for any spec-compliant decoder. `WebServer` relies on exactly this: it tries the dictionary first and falls back to a plain decode on `IOException`, which correctly handles both dictionary-compressed and plain rows — but never rely on decode success/failure to detect a *wrong* dictionary, since that case is silent. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. - **`templateId`**: `0` (or unset) means `content` is legacy HTML with presentation baked in (the pre-CMS Release 0/1 format). A positive value means `content` is JSON *facts only*, rendered through the matching row in `Templates` (a Pebble template) — the ongoing move to a proper CMS that de-duplicates presentation across near-identical pages (e.g. `sin`/`cos` docs). - The `UNIQUE(path)` constraint rejects any duplicate `path`, regardless of `languageID` — a second language for an existing path isn't supported yet (only `EN-us` currently exists). Getting there needs an upstream schema change to composite uniqueness on `(path, languageID)` (see *Known rough edges* below). From 61f2b5f5bf9b929673035c7dc01d58290442c5d7 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 16 Aug 2026 21:22:54 -0700 Subject: [PATCH 06/12] ADFA-5153: Add test proving the compression dictionary loads once Per ticket comment: verifies WebServer fetches CompressionDictionary only at startup and reuses the cached instance across every request, never re-querying it per-request. Drives 3 real HTTP requests over a socket against a mocked SQLiteDatabase and asserts the dictionary query fired exactly once while the Content query fired 3 times. Co-Authored-By: Claude Sonnet 5 --- .../localWebServer/WebServerTest.kt | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt index ef1e18de8f..bc6978c4fe 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt @@ -1,11 +1,13 @@ package com.itsaky.androidide.localWebServer +import android.database.Cursor import android.database.sqlite.SQLiteDatabase import android.net.TrafficStats import io.mockk.every import io.mockk.mockk import io.mockk.mockkStatic import io.mockk.unmockkAll +import io.mockk.verify import org.junit.After import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue @@ -102,6 +104,77 @@ class WebServerTest { assertPortIsFree(port) } + // ADFA-5153: the compression dictionary must be loaded once, at server startup, and + // stay cached in memory for every request thereafter -- never re-fetched per-request. + @Test + fun `compression dictionary is loaded once at startup and reused across every request`() { + val port = freePort() + + val dictionaryExistsCursor = + mockk(relaxed = true) { + every { moveToFirst() } returns true + } + val dictionaryDataCursor = + mockk(relaxed = true) { + every { moveToFirst() } returns true + every { getBlob(0) } returns "test-dictionary-bytes".toByteArray() + } + val contentCursor = + mockk(relaxed = true) { + every { count } returns 1 + every { moveToFirst() } returns true + every { getBlob(0) } returns "hello".toByteArray() + every { getString(1) } returns "text/plain" + every { getString(2) } returns "none" + every { getInt(3) } returns 0 + } + + val db = mockk(relaxed = true) + every { SQLiteDatabase.openDatabase(any(), isNull(), any()) } returns db + every { + db.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } returns dictionaryExistsCursor + every { + db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } returns dictionaryDataCursor + every { + db.rawQuery(match { it.contains("FROM Content") }, any()) + } returns contentCursor + + val server = WebServer(testConfig(port)) + val serverThread = Thread { server.start() }.apply { isDaemon = true } + serverThread.start() + try { + awaitPortBound(port) + + repeat(3) { sendRawGetRequestAndAwaitClose(port, "/some/path") } + + verify(exactly = 1) { + db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + } finally { + server.stop() + serverThread.join(2_000) + } + } + + // Blocks until the server closes the connection (every response sends "Connection: close"), + // so by the time this returns the server has fully finished processing this one request -- + // making repeated calls a reliable way to serialize several full request/response cycles. + private fun sendRawGetRequestAndAwaitClose( + port: Int, + path: String, + ) { + Socket().use { socket -> + socket.connect(InetSocketAddress("localhost", port), 2_000) + socket.getOutputStream().apply { + write("GET $path HTTP/1.1\r\n\r\n".toByteArray(Charsets.ISO_8859_1)) + flush() + } + socket.getInputStream().readBytes() + } + } + // Polls by attempting an actual TCP connect rather than sleeping a fixed // duration: as soon as WebServer's accept() loop is listening, the connect // succeeds, which is the readiness signal. (A bind-then-unbind probe was From fafcf48ebc0d0450f5e115d98f16a1409945bf36 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 16 Aug 2026 21:33:23 -0700 Subject: [PATCH 07/12] ADFA-5153: Reload compression dictionary per-request, not at swap time Moved loadCompressionDictionary() out of switchToDatabase() (called at startup and on the debug-DB swap) to right before the content fetch in handleClient(). A database swap can bring in a database with a different dictionary or none at all, so loading it right where it's consumed -- rather than caching it at swap time -- keeps it directly tied to whichever database is actually active when a request needs it. Updated the WebServerTest coverage added for the prior (now-reversed) "load once, cache for app lifetime" behavior: it now asserts zero dictionary queries before any request and one dictionary query per content fetch (3 requests -> 3 queries). Updated docs/comments to match. Co-Authored-By: Claude Sonnet 5 --- .../androidide/localWebServer/WebServer.kt | 28 +++++++++++-------- .../localWebServer/WebServerTest.kt | 17 ++++++++--- docs/documentation-database.md | 2 +- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index d043775f56..d5ddabe6b4 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -90,10 +90,11 @@ class WebServer( private var databaseTimestamp: Long = -1 // The shared dictionary Content's brotli-compressed rows are compressed against (see - // ADFA-5153). Reloaded whenever `database` is (re)opened -- including the debug-override - // swap below -- since a different database file may have trained its own dictionary. - // Null (no dictionary attached, plain-brotli decode) if this database predates the - // dictionary-compression migration -- CompressionDictionary won't exist yet. + // ADFA-5153). Reloaded fresh before every content fetch in handleClient() -- not cached + // across requests or tied to database-open/swap time -- since a database swap (including + // the debug-override one) can bring in a different database with its own different + // dictionary. Null (no dictionary attached, plain-brotli decode) if the active database + // predates the dictionary-compression migration -- CompressionDictionary won't exist yet. private var compressionDictionary: ByteBuffer? = null private val log = LoggerFactory.getLogger(WebServer::class.java) private val debugEnabled: Boolean = File(config.debugEnablePath).exists() @@ -190,12 +191,13 @@ class WebServer( /** * Opens [path] as the active database, refreshing every piece of state that depends on which - * database file is active -- [databaseTimestamp], [compressionDictionary], and the per-database - * caches [bookshelfTemplateId]/[templateCache] -- as one atomic operation, so a request never - * observes a database swapped in without its matching dictionary/caches. Only closes the - * previous database once the new one has opened successfully, so a failed swap (this throws) - * leaves the previous, still-open database serving requests rather than leaving [database] - * referencing an already-closed handle. + * database file is active -- [databaseTimestamp] and the per-database caches + * [bookshelfTemplateId]/[templateCache] -- as one atomic operation. Does *not* load + * [compressionDictionary]: a different database can have a different dictionary (or none), so + * that's instead loaded fresh right before each content fetch (see [handleClient]) rather than + * cached here at swap time. Only closes the previous database once the new one has opened + * successfully, so a failed swap (this throws) leaves the previous, still-open database + * serving requests rather than leaving [database] referencing an already-closed handle. */ private fun switchToDatabase( path: String, @@ -211,7 +213,6 @@ class WebServer( } database = newDatabase databaseTimestamp = timestamp - compressionDictionary = loadCompressionDictionary(database) bookshelfTemplateId = -1 templateCache.clear() } @@ -461,6 +462,11 @@ class WebServer( } } + // Reloaded fresh for every content fetch, rather than cached from database-open/swap time: + // a database swap (just above) can bring in a different dictionary (or none), and this is + // the one place that dictionary is actually consumed (see decompressBrotli). + compressionDictionary = loadCompressionDictionary(database) + // Database fetch val query = """ SELECT C.content, CT.value, CT.compression, C.templateId diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt index bc6978c4fe..c4b811e3a0 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt @@ -104,10 +104,11 @@ class WebServerTest { assertPortIsFree(port) } - // ADFA-5153: the compression dictionary must be loaded once, at server startup, and - // stay cached in memory for every request thereafter -- never re-fetched per-request. + // ADFA-5153: the compression dictionary is reloaded fresh before every content fetch, not + // cached from server-startup/database-swap time -- a swap can bring in a database with a + // different dictionary (or none), and this is the one place that dictionary is consumed. @Test - fun `compression dictionary is loaded once at startup and reused across every request`() { + fun `compression dictionary is reloaded before every content fetch, not cached from startup`() { val port = freePort() val dictionaryExistsCursor = @@ -147,9 +148,17 @@ class WebServerTest { try { awaitPortBound(port) + // Nothing fetches the dictionary merely from starting the server -- only a content + // fetch does, so before any request there should be no dictionary query at all yet. + verify(exactly = 0) { + db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + repeat(3) { sendRawGetRequestAndAwaitClose(port, "/some/path") } - verify(exactly = 1) { + // One dictionary load per content fetch, matching the 3 requests -- proving it's + // reloaded fresh each time rather than cached across requests. + verify(exactly = 3) { db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) } } finally { diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 9352a235b3..9defbd78fe 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -62,7 +62,7 @@ CREATE TABLE Tooltips ( ### Supporting tables -- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `WebServer` loads it once at startup (and again on the debug-DB swap) and, per row, tries decoding with it attached first via brotli4j's `attachDictionary`, falling back to a plain decode on failure — needed both for a database predating this migration (no `CompressionDictionary` table at all) and for plugin-contributed rows within an otherwise-migrated database (see `PluginDocumentationManager` below). +- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `WebServer` reloads it fresh before every content fetch (not cached from server-startup or database-swap time, since a swap can bring in a database with a different dictionary or none) and, per row, tries decoding with it attached first via brotli4j's `attachDictionary`, falling back to a plain decode on failure — needed both for a database predating this migration (no `CompressionDictionary` table at all) and for plugin-contributed rows within an otherwise-migrated database (see `PluginDocumentationManager` below). - **`Templates(id, name, content)`** — Pebble template source, keyed by id (and by `name` for well-known templates like `bookshelf`). Referenced by `Content.templateId`. - **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`). - **`LastChange(documentationSet, changeTime, who)`** — audit trail for edits made through `docdb-studio`; not shown to end users. `DatabaseVersionResolver` reads the `documentationSet = 'wholedb'` row to report the DB's build/edit stamp in debug logging, falling back to the most recent row of any set if `'wholedb'` is missing. From 3813981d99ad0bb437b8d1c5941173e14e5cc2b0 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 16 Aug 2026 21:39:11 -0700 Subject: [PATCH 08/12] ADFA-5153: Load compression dictionary lazily, once per database change Corrects the prior commit, which reloaded the dictionary on every single request instead of only when the database actually changes. Added compressionDictionaryStale, set by switchToDatabase() (startup and the debug-DB swap) instead of eagerly loading the dictionary there. The content-fetch site in handleClient() -- the one place the dictionary is actually consumed -- checks the flag and only loads when stale, clearing it once loaded. Net effect: loaded lazily (not merely from starting the server), but cached across every request against the same database, and reloaded exactly once when a swap brings in a database with a different dictionary (or none). Replaced the WebServerTest coverage accordingly: one test proves the dictionary loads on first use and stays cached across repeated requests against the same database; a second drives an actual debug-DB swap and proves it reloads exactly once for the new database, not on every subsequent request. Co-Authored-By: Claude Sonnet 5 --- .../androidide/localWebServer/WebServer.kt | 37 ++++--- .../localWebServer/WebServerTest.kt | 99 +++++++++++++++++-- docs/documentation-database.md | 2 +- 3 files changed, 118 insertions(+), 20 deletions(-) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index d5ddabe6b4..f54a24e62e 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -90,12 +90,19 @@ class WebServer( private var databaseTimestamp: Long = -1 // The shared dictionary Content's brotli-compressed rows are compressed against (see - // ADFA-5153). Reloaded fresh before every content fetch in handleClient() -- not cached - // across requests or tied to database-open/swap time -- since a database swap (including - // the debug-override one) can bring in a different database with its own different - // dictionary. Null (no dictionary attached, plain-brotli decode) if the active database - // predates the dictionary-compression migration -- CompressionDictionary won't exist yet. + // ADFA-5153). Lazily (re)loaded on demand, right before the first content fetch that needs + // it after `database` changes -- see compressionDictionaryStale -- rather than eagerly at + // database-open/swap time, but still cached (not reloaded per-request) once loaded for the + // currently active database. Null (no dictionary attached, plain-brotli decode) if the + // active database predates the dictionary-compression migration -- CompressionDictionary + // won't exist yet. private var compressionDictionary: ByteBuffer? = null + + // Set whenever `database` changes (see switchToDatabase); cleared once compressionDictionary + // has been (re)loaded for that database. Lets the dictionary stay lazily loaded -- only right + // before the first content fetch that actually needs it -- while still loading at most once + // per database change rather than once per request. + private var compressionDictionaryStale = true private val log = LoggerFactory.getLogger(WebServer::class.java) private val debugEnabled: Boolean = File(config.debugEnablePath).exists() @@ -193,9 +200,10 @@ class WebServer( * Opens [path] as the active database, refreshing every piece of state that depends on which * database file is active -- [databaseTimestamp] and the per-database caches * [bookshelfTemplateId]/[templateCache] -- as one atomic operation. Does *not* load - * [compressionDictionary]: a different database can have a different dictionary (or none), so - * that's instead loaded fresh right before each content fetch (see [handleClient]) rather than - * cached here at swap time. Only closes the previous database once the new one has opened + * [compressionDictionary] itself -- a different database can have a different dictionary (or + * none) -- it only marks [compressionDictionaryStale] so the next content fetch that needs it + * loads it lazily then (see [handleClient]), at most once per database change rather than + * once per request. Only closes the previous database once the new one has opened * successfully, so a failed swap (this throws) leaves the previous, still-open database * serving requests rather than leaving [database] referencing an already-closed handle. */ @@ -213,6 +221,7 @@ class WebServer( } database = newDatabase databaseTimestamp = timestamp + compressionDictionaryStale = true bookshelfTemplateId = -1 templateCache.clear() } @@ -462,10 +471,14 @@ class WebServer( } } - // Reloaded fresh for every content fetch, rather than cached from database-open/swap time: - // a database swap (just above) can bring in a different dictionary (or none), and this is - // the one place that dictionary is actually consumed (see decompressBrotli). - compressionDictionary = loadCompressionDictionary(database) + // Lazily (re)loaded here -- the one place the dictionary is actually consumed (see + // decompressBrotli) -- rather than eagerly at database-open/swap time, but only once per + // database change: a swap (just above) marks compressionDictionaryStale rather than + // reloading immediately, so this only hits the database again when that flag is set. + if (compressionDictionaryStale) { + compressionDictionary = loadCompressionDictionary(database) + compressionDictionaryStale = false + } // Database fetch val query = """ diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt index c4b811e3a0..81fec2bb15 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt @@ -13,6 +13,7 @@ import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test +import java.io.File import java.net.InetSocketAddress import java.net.ServerSocket import java.net.Socket @@ -104,11 +105,11 @@ class WebServerTest { assertPortIsFree(port) } - // ADFA-5153: the compression dictionary is reloaded fresh before every content fetch, not - // cached from server-startup/database-swap time -- a swap can bring in a database with a - // different dictionary (or none), and this is the one place that dictionary is consumed. + // ADFA-5153: the compression dictionary is loaded lazily -- not merely from starting the + // server -- but only once per database, cached across every subsequent request against that + // same database rather than re-fetched per-request. @Test - fun `compression dictionary is reloaded before every content fetch, not cached from startup`() { + fun `compression dictionary loads lazily on first use, once per database, not once per request`() { val port = freePort() val dictionaryExistsCursor = @@ -156,9 +157,9 @@ class WebServerTest { repeat(3) { sendRawGetRequestAndAwaitClose(port, "/some/path") } - // One dictionary load per content fetch, matching the 3 requests -- proving it's - // reloaded fresh each time rather than cached across requests. - verify(exactly = 3) { + // Exactly one dictionary load across all 3 requests against the same, unchanged + // database -- the first request's lazy load, cached for the other two. + verify(exactly = 1) { db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) } } finally { @@ -167,6 +168,90 @@ class WebServerTest { } } + // ADFA-5153: a database swap (the debug-DB override) must invalidate the cached dictionary -- + // the new database can have a different one, or none -- causing exactly one fresh reload on + // the first content fetch against the new database, not a reload on every later request too. + @Test + fun `database swap invalidates the cached dictionary, reloading it once for the new database`() { + val port = freePort() + val debugDbFile = File.createTempFile("webserver-test-debug", ".db") + debugDbFile.delete() // must not exist yet -- the first request should stay on the primary db + + fun contentCursorFor(marker: String) = + mockk(relaxed = true) { + every { count } returns 1 + every { moveToFirst() } returns true + every { getBlob(0) } returns marker.toByteArray() + every { getString(1) } returns "text/plain" + every { getString(2) } returns "none" + every { getInt(3) } returns 0 + } + + fun stubDatabase( + db: SQLiteDatabase, + dictionaryBytes: String, + ) { + every { + db.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } returns mockk(relaxed = true) { every { moveToFirst() } returns true } + every { + db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } returns + mockk(relaxed = true) { + every { moveToFirst() } returns true + every { getBlob(0) } returns dictionaryBytes.toByteArray() + } + every { + db.rawQuery(match { it.contains("FROM Content") }, any()) + } returns contentCursorFor(dictionaryBytes) + } + + val primaryDb = mockk(relaxed = true) + val debugDb = mockk(relaxed = true) + stubDatabase(primaryDb, "dict-primary") + stubDatabase(debugDb, "dict-debug") + + val config = testConfig(port).copy(debugDatabasePath = debugDbFile.absolutePath) + every { SQLiteDatabase.openDatabase(config.databasePath, isNull(), any()) } returns primaryDb + every { SQLiteDatabase.openDatabase(config.debugDatabasePath, isNull(), any()) } returns debugDb + + val server = WebServer(config) + val serverThread = Thread { server.start() }.apply { isDaemon = true } + serverThread.start() + try { + awaitPortBound(port) + + sendRawGetRequestAndAwaitClose(port, "/some/path") + verify(exactly = 1) { + primaryDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + verify(exactly = 0) { + debugDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + + // Now make the debug override newer than the primary database -- the swap check in + // handleClient() picks this up on the very next request. + debugDbFile.createNewFile() + debugDbFile.setLastModified(System.currentTimeMillis() + 60_000) + + repeat(2) { sendRawGetRequestAndAwaitClose(port, "/some/path") } + + // Exactly one reload for the new (debug) database, across both post-swap requests -- + // not zero (it must invalidate), not two (it must still cache after the first reload). + verify(exactly = 1) { + debugDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + // The primary database's dictionary is never touched again after the swap. + verify(exactly = 1) { + primaryDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + } finally { + server.stop() + serverThread.join(2_000) + debugDbFile.delete() + } + } + // Blocks until the server closes the connection (every response sends "Connection: close"), // so by the time this returns the server has fully finished processing this one request -- // making repeated calls a reliable way to serialize several full request/response cycles. diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 9defbd78fe..4c60cf2af1 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -62,7 +62,7 @@ CREATE TABLE Tooltips ( ### Supporting tables -- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `WebServer` reloads it fresh before every content fetch (not cached from server-startup or database-swap time, since a swap can bring in a database with a different dictionary or none) and, per row, tries decoding with it attached first via brotli4j's `attachDictionary`, falling back to a plain decode on failure — needed both for a database predating this migration (no `CompressionDictionary` table at all) and for plugin-contributed rows within an otherwise-migrated database (see `PluginDocumentationManager` below). +- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `WebServer` loads it lazily -- not merely from starting the server or swapping databases, but on the first content fetch that needs it after `database` changes -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Per row, it tries decoding with the dictionary attached first via brotli4j's `attachDictionary`, falling back to a plain decode on failure — needed both for a database predating this migration (no `CompressionDictionary` table at all) and for plugin-contributed rows within an otherwise-migrated database (see `PluginDocumentationManager` below). - **`Templates(id, name, content)`** — Pebble template source, keyed by id (and by `name` for well-known templates like `bookshelf`). Referenced by `Content.templateId`. - **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`). - **`LastChange(documentationSet, changeTime, who)`** — audit trail for edits made through `docdb-studio`; not shown to end users. `DatabaseVersionResolver` reads the `documentationSet = 'wholedb'` row to report the DB's build/edit stamp in debug logging, falling back to the most recent row of any set if `'wholedb'` is missing. From 568b21e109fcc5f13ad31bc13bca88bbadfff5a9 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 17 Aug 2026 16:04:24 -0700 Subject: [PATCH 09/12] ADFA-5153: Run the brotli tests on any host, and cover the buffer helper Review of PR #1677 found three things worth fixing. The test native was pinned to linux-x64, so :app:testV8DebugUnitTest failed with UnsatisfiedLinkError in @BeforeClass for anyone on macOS, Windows, or linux-arm64 - a comment documented the breakage rather than fixing it. Dispatch on the host's OS/arch instead, reusing the pattern already proven in build-logic/plugins' build.gradle.kts. All six natives are already in the version catalog. BrotliDictionaryDecodeTest allocated its own direct buffer, a byte-for-byte copy of production's toDirectByteBuffer, leaving the only code that builds the runtime dictionary buffer untested. The two agree today, so this is a regression risk rather than a live bug: attachDictionary reads the buffer's capacity and ignores position/limit, so a later over-allocation there (pooling, rounding, padding) would break every doc page on device while the suite stayed green. The test now calls the production helper, and that helper's KDoc records the exact-capacity requirement. loadCompressionDictionary validated a missing table, an empty table, and a NULL data column, but not a zero-length blob. That yields a 0-capacity buffer, which attachDictionary rejects, so every row would fail its dictionary decode, fall through to a plain decode that also fails, and return HTTP 500 - with nothing above DEBUG to explain it. Added to the same ladder so it gets the same one-line warning. Left alone: peak heap on the chunked PDFs (always-decompress holds the accumulator, its copy, and the output live at once) and the debug-DB swap retrying every request after a failure. Both are pre-existing design questions rather than regressions from this PR. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Y4t7fFLNPJq9EiXr9S9LxF --- app/build.gradle.kts | 46 +++++++++++++++++-- .../androidide/localWebServer/WebServer.kt | 10 ++++ .../BrotliDictionaryDecodeTest.kt | 13 ++---- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index f54647a119..b0d5d8f3f4 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -3,6 +3,7 @@ import com.itsaky.androidide.build.config.BuildConfig import com.itsaky.androidide.desugaring.utils.JavaIOReplacements.applyJavaIOReplacements import com.itsaky.androidide.plugins.AndroidIDEAssetsPlugin +import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform import org.json.JSONObject import java.io.BufferedOutputStream import java.io.ByteArrayInputStream @@ -209,6 +210,43 @@ configurations.configureEach { exclude(group = "com.google.auto.value", module = "auto-value") } +// brotli4j ships its native decoder as a per-OS/arch artifact, so the JVM unit tests need the one +// matching whoever is building. Mirrors build-logic/plugins' dispatch. +fun brotli4jNativeForHost(): Provider { + val arch = DefaultNativePlatform.getCurrentArchitecture() + return DefaultNativePlatform.getCurrentOperatingSystem().let { os -> + when { + os.isMacOsX -> { + when { + arch.isArm64 -> libs.brotli4j.osx.aarch64 + arch.isAmd64 -> libs.brotli4j.osx.x64 + else -> throw IllegalStateException("Unsupported OSX architecture: $arch") + } + } + + os.isWindows -> { + when { + arch.isArm64 -> libs.brotli4j.windows.aarch64 + arch.isAmd64 -> libs.brotli4j.windows.x64 + else -> throw IllegalStateException("Unsupported Windows architecture: $arch") + } + } + + os.isLinux -> { + when { + arch.isArm64 -> libs.brotli4j.linux.aarch64 + arch.isAmd64 -> libs.brotli4j.linux.x64 + else -> throw IllegalStateException("Unsupported Linux architecture: $arch") + } + } + + else -> { + throw IllegalStateException("Unsupported OS: $os") + } + } + } +} + dependencies { debugImplementation(libs.common.leakcanary) @@ -339,11 +377,9 @@ dependencies { implementation(libs.brotli4j) // JVM unit tests (e.g. BrotliDictionaryDecodeTest) run brotli4j's real native decoder, not an // Android target -- without a desktop native on the test classpath, Brotli4jLoader has nothing - // to load and every such test fails with UnsatisfiedLinkError. Only linux-x64 is added, matching - // this repo's CI runners (ubuntu-latest); a contributor running :app:test on macOS/Windows/arm64 - // needs the matching libs.brotli4j.* native added locally (see build-logic/plugins' build.gradle.kts - // for the OS/arch dispatch pattern) or to run the suite in CI/a Linux x64 environment instead. - testImplementation(libs.brotli4j.linux.x64) + // to load and every such test fails with UnsatisfiedLinkError. Pick the native for whoever is + // building, so the suite runs off a Linux x64 CI runner too (same dispatch as build-logic/plugins'). + testImplementation(brotli4jNativeForHost()) implementation(libs.common.markwon.core) implementation(libs.common.markwon.linkify) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index f54a24e62e..b30c5b1d06 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -67,6 +67,10 @@ data class JavaExecutionResult( /** * Copies [bytes] into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct * buffer, a heap-backed one throws `IllegalArgumentException`. + * + * The capacity must be exactly [bytes]`.size`: `attachDictionary` reads the whole capacity and + * ignores position/limit, so trailing slack from an over-allocated buffer is treated as dictionary + * content and every decode then fails with `IOException: corrupted input`. */ internal fun toDirectByteBuffer(bytes: ByteArray): ByteBuffer = ByteBuffer.allocateDirect(bytes.size).apply { @@ -188,6 +192,12 @@ class WebServer( log.warn("CompressionDictionary row has a NULL data column; decoding brotli content without a dictionary.") return null } + // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects -- + // every row's dictionary decode would then fail with nothing above DEBUG to say why. + if (bytes.isEmpty()) { + log.warn("CompressionDictionary row has an empty data column; decoding brotli content without a dictionary.") + return null + } toDirectByteBuffer(bytes) } } catch (e: Exception) { diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt index a5a9bd8735..6248057b2e 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -15,20 +15,17 @@ import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.util.Base64 +// Deliberately routed through production's toDirectByteBuffer rather than allocating here: +// attachDictionary reads the buffer's capacity, so an over-allocated buffer fails every decode. +// Duplicating the allocation would leave that helper untested and let the two drift apart. +private fun decodeBase64ToDirectBuffer(base64: String): ByteBuffer = toDirectByteBuffer(Base64.getDecoder().decode(base64)) + // Regression coverage for ADFA-5153: documentation.db's Content rows are Brotli-compressed // against a shared dictionary trained by OfflineDocumentationTools' zstd/brotli CLI pipeline // (see populate_db.py's DictionaryCompressor), not by brotli4j itself. These fixtures were // produced by that exact pipeline, so this test is what protects the cross-tool contract: a // brotli4j upgrade (or native lib change) that silently broke compatibility with the CLI-produced // wire format would otherwise only surface as garbled content on-device. -private fun decodeBase64ToDirectBuffer(base64: String): ByteBuffer { - val bytes = Base64.getDecoder().decode(base64) - return ByteBuffer.allocateDirect(bytes.size).apply { - put(bytes) - flip() - } -} - class BrotliDictionaryDecodeTest { companion object { // Unlike on-device (where ToolsManager/AssetsInstallationHelper already load it before From 5a94e24d7062a883f98f99e154a92029bfa2cd54 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 17 Aug 2026 16:10:53 -0700 Subject: [PATCH 10/12] ADFA-5153: Cut peak heap on chunked rows, and stop retrying a bad debug DB Two findings from the PR #1677 review that were deferred as design questions. Peak heap on the largest bundled PDFs. Serving a >1 MB row concatenated its chunks into a ByteArrayOutputStream and then called toByteArray(), so the doubling buffer and its full copy were both live alongside the decompressed output - roughly 35 MB transient for AndroidNotesForProfessionals.pdf (8.8 MB over 9 chunks), a plausible OOM on a low-heap device. The chunks now stay a list: brotli rows decode from a SequenceInputStream over them, and non-brotli rows are joined once into an exactly-sized array. That drops the two largest transients, leaving the compressed chunks and the decompressed output. Fully streaming the response would remove the last one too, but that means giving up Content-Length, so it is left alone. A failed debug-database swap left databaseTimestamp unadvanced, and the swap is checked per request - so a corrupt or unreadable debug DB newer than the primary was reopened on every single request, logging an ERROR each time. The failing timestamp is now remembered and skipped; a newer copy has a different timestamp and is retried, which is the case that matters, since replacing the file is how a developer fixes it. joinChunks and chunksAsStream are internal top-level functions next to toDirectByteBuffer so the tests exercise the real code, with three new cases: a compressed stream decodes identically when split at uneven chunk boundaries, joinChunks concatenates in order at an exact size, and a lone chunk comes back without a copy. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Y4t7fFLNPJq9EiXr9S9LxF --- .../androidide/localWebServer/WebServer.kt | 80 ++++++++++++++----- .../BrotliDictionaryDecodeTest.kt | 46 +++++++++++ 2 files changed, 108 insertions(+), 18 deletions(-) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index b30c5b1d06..319807e682 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -21,6 +21,7 @@ import java.io.File import java.io.IOException import java.io.InputStream import java.io.PrintWriter +import java.io.SequenceInputStream import java.io.StringWriter import java.net.InetSocketAddress import java.net.ServerSocket @@ -29,6 +30,7 @@ import java.net.URLDecoder import java.nio.ByteBuffer import java.sql.Date import java.text.SimpleDateFormat +import java.util.Collections import java.util.Locale import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit @@ -78,6 +80,31 @@ internal fun toDirectByteBuffer(bytes: ByteArray): ByteBuffer = flip() } +/** + * Reads [chunks] back to back as one stream, without concatenating them into a new array. + * Cheap to build twice, which the no-dictionary retry in `decompressBrotli` relies on. + */ +internal fun chunksAsStream(chunks: List): InputStream = + SequenceInputStream(Collections.enumeration(chunks.map { ByteArrayInputStream(it) })) + +/** + * Joins [chunks] into one exactly-sized array. A ByteArrayOutputStream would repeatedly double its + * buffer and then hand back a second full copy -- avoidable here since the total is known up front. + * Returns the sole element as-is when there is nothing to join. + */ +internal fun joinChunks(chunks: List): ByteArray { + if (chunks.size == 1) { + return chunks[0] + } + val joined = ByteArray(chunks.sumOf { it.size }) + var offset = 0 + for (chunk in chunks) { + chunk.copyInto(joined, offset) + offset += chunk.size + } + return joined +} + class WebServer( private val config: ServerConfig, ) { @@ -93,6 +120,12 @@ class WebServer( private lateinit var database: SQLiteDatabase private var databaseTimestamp: Long = -1 + // Timestamp of a debug database whose swap already failed, so a corrupt or unreadable one + // isn't reopened on every single request (it is checked per request). A newer copy has a + // different timestamp and is retried, which is the case that matters -- the developer + // replacing the file is exactly how they'd fix it. + private var failedDebugSwapTimestamp: Long = -1 + // The shared dictionary Content's brotli-compressed rows are compressed against (see // ADFA-5153). Lazily (re)loaded on demand, right before the first content fetch that needs // it after `database` changes -- see compressionDictionaryStale -- rather than eagerly at @@ -245,11 +278,11 @@ class WebServer( * silently producing wrong bytes (verified empirically -- see docs/documentation-database.md), so * this ordering never lets a dictionary-compressed row fall through to the plain path by accident. */ - private fun decompressBrotli(content: ByteArray): ByteArray { + private fun decompressBrotli(chunks: List): ByteArray { val dictionary = compressionDictionary if (dictionary != null) { try { - return BrotliInputStream(ByteArrayInputStream(content)).use { stream -> + return BrotliInputStream(chunksAsStream(chunks)).use { stream -> stream.attachDictionary(dictionary) stream.readBytes() } @@ -260,7 +293,7 @@ class WebServer( ) } } - return BrotliInputStream(ByteArrayInputStream(content)).use { it.readBytes() } + return BrotliInputStream(chunksAsStream(chunks)).use { it.readBytes() } } /** @@ -460,11 +493,17 @@ class WebServer( // check to see if there is a newer version of the documentation.db database on the sdcard // if there is use that for our responses val debugDatabaseTimestamp = getDatabaseTimestamp(config.debugDatabasePath, true) - if (debugDatabaseTimestamp > databaseTimestamp) { + if (debugDatabaseTimestamp > databaseTimestamp && debugDatabaseTimestamp != failedDebugSwapTimestamp) { try { switchToDatabase(config.debugDatabasePath, debugDatabaseTimestamp) + failedDebugSwapTimestamp = -1 } catch (e: Exception) { - log.error("Cannot swap to debug database '{}': {}", config.debugDatabasePath, e.message) + failedDebugSwapTimestamp = debugDatabaseTimestamp + log.error( + "Cannot swap to debug database '{}'; ignoring it until it changes: {}", + config.debugDatabasePath, + e.message, + ) } } @@ -515,24 +554,27 @@ class WebServer( } cursor.moveToFirst() - var dbContent = cursor.getBlob(0) + val firstChunk = cursor.getBlob(0) val dbMimeType = cursor.getString(1) var compression = cursor.getString(2) val templateId = cursor.getInt(3) - // Fragment handling for large content (> 1MB) - if (dbContent.size == contentChunkSize) { + // Fragment handling for large content (> 1MB). The chunks stay a list rather than + // being concatenated: the old accumulate-into-a-ByteArrayOutputStream-then-copy held + // the doubling buffer *and* its toByteArray() copy live alongside the decompressed + // output, roughly 35 MB transient for the largest bundled PDF (8.8 MB over 9 chunks). + val chunks = mutableListOf(firstChunk) + if (firstChunk.size == contentChunkSize) { val query2 = "SELECT content FROM Content WHERE path = ? AND languageId = 1" var fragmentNumber = 1 - val combined = ByteArrayOutputStream().apply { write(dbContent) } - var dbContent2 = dbContent - while (dbContent2.size == contentChunkSize) { + var nextChunk = firstChunk + while (nextChunk.size == contentChunkSize) { val path2 = "$path-$fragmentNumber" val cursor2 = database.rawQuery(query2, arrayOf(path2)) try { if (cursor2.moveToFirst()) { - dbContent2 = cursor2.getBlob(0) - combined.write(dbContent2) + nextChunk = cursor2.getBlob(0) + chunks.add(nextChunk) fragmentNumber++ } else { break @@ -541,7 +583,6 @@ class WebServer( cursor2.close() } } - dbContent = combined.toByteArray() } // Content is compressed at rest with brotli -- most rows against the shared dictionary @@ -549,10 +590,13 @@ class WebServer( // (PluginDocumentationManager/BrotliCompressor) are plain brotli with no dictionary. // This server always decompresses before responding, so it never needs to negotiate // Content-Encoding with the client. - if (compression == "brotli") { - dbContent = decompressBrotli(dbContent) - compression = "none" - } + var dbContent = + if (compression == "brotli") { + compression = "none" + decompressBrotli(chunks) + } else { + joinChunks(chunks) + } // If the file is associated with a template, instantiate that template and send the result to the client if (templateId > 0) { diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt index 6248057b2e..80a1ac152c 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -5,6 +5,8 @@ import com.aayushatharva.brotli4j.decoder.BrotliInputStream import com.aayushatharva.brotli4j.encoder.BrotliOutputStream import com.aayushatharva.brotli4j.encoder.Encoder import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertSame import org.junit.Assert.assertThrows import org.junit.BeforeClass import org.junit.Test @@ -202,4 +204,48 @@ class BrotliDictionaryDecodeTest { val plainResult = BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() } assertArrayEquals(expected, plainResult) } + + @Test + fun `content split across chunks decodes the same as one contiguous array`() { + // Rows over 1 MB are stored as several Content rows and were previously concatenated + // before decoding; they are now fed to the decoder as a stream over the chunk list, so + // a compressed stream must decode identically no matter where the chunk boundaries fall. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val compressed = Base64.getDecoder().decode(compressedBase64) + val expected = Base64.getDecoder().decode(expectedBase64) + + // Deliberately uneven, and not aligned to anything in the brotli stream. + val chunks = + listOf( + compressed.copyOfRange(0, 7), + compressed.copyOfRange(7, 8), + compressed.copyOfRange(8, compressed.size - 1), + compressed.copyOfRange(compressed.size - 1, compressed.size), + ) + + val result = + BrotliInputStream(chunksAsStream(chunks)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + + assertArrayEquals(expected, result) + } + + @Test + fun `joinChunks concatenates in order and sizes the result exactly`() { + val chunks = listOf(byteArrayOf(1, 2, 3), byteArrayOf(), byteArrayOf(4), byteArrayOf(5, 6)) + + val joined = joinChunks(chunks) + + assertArrayEquals(byteArrayOf(1, 2, 3, 4, 5, 6), joined) + assertEquals(6, joined.size) + } + + @Test + fun `joinChunks hands back a lone chunk without copying it`() { + val only = byteArrayOf(7, 8, 9) + + assertSame(only, joinChunks(listOf(only))) + } } From 9fbf2bb7863257f4742c0ccf34ee6ab8cc49bb1f Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 17 Aug 2026 17:29:26 -0700 Subject: [PATCH 11/12] ADFA-5153: Address CodeRabbit findings on the dictionary tests - Assert the sqlite_master existence-check query count alongside the data query in both dictionary tests, not just the data query -- a regression that re-ran only the existence check every request would otherwise pass unnoticed. - Set socket.soTimeout before reading the response in sendRawGetRequestAndAwaitClose, so a server that fails to close the connection fails the test instead of hanging the JVM indefinitely. Co-Authored-By: Claude Sonnet 5 --- .../localWebServer/WebServerTest.kt | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt index 81fec2bb15..835d3c0d5d 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt @@ -150,7 +150,11 @@ class WebServerTest { awaitPortBound(port) // Nothing fetches the dictionary merely from starting the server -- only a content - // fetch does, so before any request there should be no dictionary query at all yet. + // fetch does, so before any request there should be no dictionary query at all yet -- + // neither the sqlite_master existence check nor the data fetch. + verify(exactly = 0) { + db.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } verify(exactly = 0) { db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) } @@ -158,7 +162,13 @@ class WebServerTest { repeat(3) { sendRawGetRequestAndAwaitClose(port, "/some/path") } // Exactly one dictionary load across all 3 requests against the same, unchanged - // database -- the first request's lazy load, cached for the other two. + // database -- the first request's lazy load, cached for the other two. Both queries + // loadCompressionDictionary issues (the sqlite_master existence check, then the data + // fetch) must be checked, or a regression re-running just the existence check on + // every request would pass unnoticed. + verify(exactly = 1) { + db.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } verify(exactly = 1) { db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) } @@ -222,9 +232,15 @@ class WebServerTest { awaitPortBound(port) sendRawGetRequestAndAwaitClose(port, "/some/path") + verify(exactly = 1) { + primaryDb.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } verify(exactly = 1) { primaryDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) } + verify(exactly = 0) { + debugDb.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } verify(exactly = 0) { debugDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) } @@ -238,10 +254,17 @@ class WebServerTest { // Exactly one reload for the new (debug) database, across both post-swap requests -- // not zero (it must invalidate), not two (it must still cache after the first reload). + // Both queries loadCompressionDictionary issues must be checked (see the sibling test). + verify(exactly = 1) { + debugDb.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } verify(exactly = 1) { debugDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) } // The primary database's dictionary is never touched again after the swap. + verify(exactly = 1) { + primaryDb.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } verify(exactly = 1) { primaryDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) } @@ -261,6 +284,7 @@ class WebServerTest { ) { Socket().use { socket -> socket.connect(InetSocketAddress("localhost", port), 2_000) + socket.soTimeout = 2_000 socket.getOutputStream().apply { write("GET $path HTTP/1.1\r\n\r\n".toByteArray(Charsets.ISO_8859_1)) flush() From 3052217c6172ed4cf50ff3cd24b55279045fe37d Mon Sep 17 00:00:00 2001 From: David Schachter Date: Tue, 18 Aug 2026 08:20:50 -0700 Subject: [PATCH 12/12] ADFA-5153: Address jatezzz's review on PR #1677 (3 of 5 findings) - loadCompressionDictionary no longer swallows exceptions into "no dictionary." It only returns null for a definitive absence (missing table, empty table, null/empty blob); an unexpected SQLiteException now propagates to the call site, which leaves compressionDictionaryStale set so the next request retries instead of permanently caching a transient failure as "no dictionary" for the rest of the database's lifetime. - brotli4jNativeForHost() in app/build.gradle.kts no longer throws on an unrecognized host. That ran at configuration time, so throwing failed every task in the build -- including :app:assembleV8Debug, which needs no desktop native at all -- not just the JVM unit-test tasks that consume it. Degrades to a logged warning and no test native instead. - Softened the chunked-content comment's memory-savings claim: the decompressed output still goes through a comparable accumulate-then-copy in decompressBrotli's own readBytes() call, so the saving from keeping compressed chunks as a list is real but doesn't eliminate that separate transient the way the prior wording implied. The two remaining findings (dictionary-first decode's theoretical silent-wrong-bytes risk, and the resulting double-decode cost for dictionary-free rows) need a design discussion, not a quick fix -- see the PR thread reply. Co-Authored-By: Claude Sonnet 5 --- app/build.gradle.kts | 30 +++++-- .../androidide/localWebServer/WebServer.kt | 90 ++++++++++--------- 2 files changed, 70 insertions(+), 50 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index b0d5d8f3f4..9cca6903e1 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -211,16 +211,20 @@ configurations.configureEach { } // brotli4j ships its native decoder as a per-OS/arch artifact, so the JVM unit tests need the one -// matching whoever is building. Mirrors build-logic/plugins' dispatch. -fun brotli4jNativeForHost(): Provider { +// matching whoever is building. Mirrors build-logic/plugins' dispatch, but degrades to null on an +// unrecognized host instead of throwing: this runs at configuration time, so throwing would fail +// every task in the build -- including :app:assembleV8Debug, which needs no desktop native at all +// -- rather than only the JVM unit-test tasks that actually consume this dependency. +fun brotli4jNativeForHost(): Provider? { val arch = DefaultNativePlatform.getCurrentArchitecture() - return DefaultNativePlatform.getCurrentOperatingSystem().let { os -> + val os = DefaultNativePlatform.getCurrentOperatingSystem() + val native = when { os.isMacOsX -> { when { arch.isArm64 -> libs.brotli4j.osx.aarch64 arch.isAmd64 -> libs.brotli4j.osx.x64 - else -> throw IllegalStateException("Unsupported OSX architecture: $arch") + else -> null } } @@ -228,7 +232,7 @@ fun brotli4jNativeForHost(): Provider { when { arch.isArm64 -> libs.brotli4j.windows.aarch64 arch.isAmd64 -> libs.brotli4j.windows.x64 - else -> throw IllegalStateException("Unsupported Windows architecture: $arch") + else -> null } } @@ -236,15 +240,23 @@ fun brotli4jNativeForHost(): Provider { when { arch.isArm64 -> libs.brotli4j.linux.aarch64 arch.isAmd64 -> libs.brotli4j.linux.x64 - else -> throw IllegalStateException("Unsupported Linux architecture: $arch") + else -> null } } else -> { - throw IllegalStateException("Unsupported OS: $os") + null } } + if (native == null) { + logger.warn( + "brotli4j: no native decoder for {}/{} -- brotli4j-backed JVM unit tests " + + "(e.g. BrotliDictionaryDecodeTest) will fail with UnsatisfiedLinkError on this host.", + os, + arch, + ) } + return native } dependencies { @@ -379,7 +391,9 @@ dependencies { // Android target -- without a desktop native on the test classpath, Brotli4jLoader has nothing // to load and every such test fails with UnsatisfiedLinkError. Pick the native for whoever is // building, so the suite runs off a Linux x64 CI runner too (same dispatch as build-logic/plugins'). - testImplementation(brotli4jNativeForHost()) + // Null on an unrecognized host just means those specific tests fail there -- see + // brotli4jNativeForHost's own warning -- not that this whole build should refuse to configure. + brotli4jNativeForHost()?.let { testImplementation(it) } implementation(libs.common.markwon.core) implementation(libs.common.markwon.linkify) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index 319807e682..86e42350ff 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -194,48 +194,43 @@ class WebServer( /** * Loads the shared Brotli dictionary most Content rows are compressed against (see ADFA-5153). - * Returns null (logged) on any failure to load one -- a database that predates the dictionary - * migration, a schema/row anomaly, or any other error -- so callers always fall back to plain, - * dictionary-free brotli decode rather than propagating the failure (see [decompressBrotli]). + * Returns null (logged) when the database *definitively* has no dictionary -- predates the + * migration, or has an empty/anomalous `CompressionDictionary` row -- so callers fall back to + * plain, dictionary-free brotli decode (see [decompressBrotli]). Deliberately does *not* catch + * exceptions itself: an unexpected `SQLiteException`/IO failure is likely transient, and the + * caller (see [handleClient]) must not cache that as "no dictionary" the way it does a + * definitive absence, or a transient failure would permanently disable dictionary decoding + * for the rest of this database's lifetime. */ private fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { - // Whole body wrapped in one catch-all (matching DatabaseVersionResolver.resolveDatabaseVersion's - // pattern) rather than hand-anticipating individual SQLiteExceptions: a caller-visible failure - // here must never abort start() or leave a stale dictionary un-retried after a debug-DB swap -- - // falling back to null (plain, dictionary-free decode) is always the safe choice. - return try { - val tableExists = - db - .rawQuery( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", - null, - ).use { it.moveToFirst() } - if (!tableExists) { - log.warn("CompressionDictionary table not found; decoding brotli content without a dictionary.") + val tableExists = + db + .rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", + null, + ).use { it.moveToFirst() } + if (!tableExists) { + log.warn("CompressionDictionary table not found; decoding brotli content without a dictionary.") + return null + } + + return db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> + if (!cursor.moveToFirst()) { + log.warn("CompressionDictionary table is empty; decoding brotli content without a dictionary.") return null } - - db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> - if (!cursor.moveToFirst()) { - log.warn("CompressionDictionary table is empty; decoding brotli content without a dictionary.") - return null - } - val bytes = cursor.getBlob(0) - if (bytes == null) { - log.warn("CompressionDictionary row has a NULL data column; decoding brotli content without a dictionary.") - return null - } - // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects -- - // every row's dictionary decode would then fail with nothing above DEBUG to say why. - if (bytes.isEmpty()) { - log.warn("CompressionDictionary row has an empty data column; decoding brotli content without a dictionary.") - return null - } - toDirectByteBuffer(bytes) + val bytes = cursor.getBlob(0) + if (bytes == null) { + log.warn("CompressionDictionary row has a NULL data column; decoding brotli content without a dictionary.") + return null } - } catch (e: Exception) { - log.error("Could not load compression dictionary; decoding brotli content without a dictionary: {}", e.message) - null + // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects -- + // every row's dictionary decode would then fail with nothing above DEBUG to say why. + if (bytes.isEmpty()) { + log.warn("CompressionDictionary row has an empty data column; decoding brotli content without a dictionary.") + return null + } + toDirectByteBuffer(bytes) } } @@ -524,9 +519,16 @@ class WebServer( // decompressBrotli) -- rather than eagerly at database-open/swap time, but only once per // database change: a swap (just above) marks compressionDictionaryStale rather than // reloading immediately, so this only hits the database again when that flag is set. + // Only clears the flag on a clean load (definitive dictionary or definitive absence) -- + // an unexpected exception leaves it set so the next request retries, rather than caching + // a transient failure as "no dictionary" for the rest of this database's lifetime. if (compressionDictionaryStale) { - compressionDictionary = loadCompressionDictionary(database) - compressionDictionaryStale = false + try { + compressionDictionary = loadCompressionDictionary(database) + compressionDictionaryStale = false + } catch (e: Exception) { + log.error("Could not load compression dictionary; will retry on the next request: {}", e.message) + } } // Database fetch @@ -560,9 +562,13 @@ class WebServer( val templateId = cursor.getInt(3) // Fragment handling for large content (> 1MB). The chunks stay a list rather than - // being concatenated: the old accumulate-into-a-ByteArrayOutputStream-then-copy held - // the doubling buffer *and* its toByteArray() copy live alongside the decompressed - // output, roughly 35 MB transient for the largest bundled PDF (8.8 MB over 9 chunks). + // being eagerly concatenated: the old accumulate-into-a-ByteArrayOutputStream-then-copy + // held both the doubling buffer and its toByteArray() copy of the *compressed* chunks + // live at once, on top of the decompressed output that follows -- for the largest + // bundled PDF (8.8 MB over 9 chunks) that's a real, if partial, reduction: the + // decompressed output still goes through a comparable accumulate-then-copy in + // decompressBrotli's own readBytes() call, so the compressed-side saving here doesn't + // eliminate that separate transient. val chunks = mutableListOf(firstChunk) if (firstChunk.size == contentChunkSize) { val query2 = "SELECT content FROM Content WHERE path = ? AND languageId = 1"