diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 21c52a5fc2..acff8f5ee7 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -3,6 +3,7 @@ import com.itsaky.androidide.build.config.BuildConfig import com.itsaky.androidide.desugaring.utils.JavaIOReplacements.applyJavaIOReplacements import com.itsaky.androidide.plugins.AndroidIDEAssetsPlugin +import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform import org.json.JSONObject import java.io.BufferedOutputStream import java.io.ByteArrayInputStream @@ -214,6 +215,55 @@ configurations.configureEach { exclude(group = "com.google.auto.value", module = "auto-value") } +// brotli4j ships its native decoder as a per-OS/arch artifact, so the JVM unit tests need the one +// matching whoever is building. Mirrors build-logic/plugins' dispatch, but degrades to null on an +// unrecognized host instead of throwing: this runs at configuration time, so throwing would fail +// every task in the build -- including :app:assembleV8Debug, which needs no desktop native at all +// -- rather than only the JVM unit-test tasks that actually consume this dependency. +fun brotli4jNativeForHost(): Provider? { + val arch = DefaultNativePlatform.getCurrentArchitecture() + val os = DefaultNativePlatform.getCurrentOperatingSystem() + val native = + when { + os.isMacOsX -> { + when { + arch.isArm64 -> libs.brotli4j.osx.aarch64 + arch.isAmd64 -> libs.brotli4j.osx.x64 + else -> null + } + } + + os.isWindows -> { + when { + arch.isArm64 -> libs.brotli4j.windows.aarch64 + arch.isAmd64 -> libs.brotli4j.windows.x64 + else -> null + } + } + + os.isLinux -> { + when { + arch.isArm64 -> libs.brotli4j.linux.aarch64 + arch.isAmd64 -> libs.brotli4j.linux.x64 + else -> null + } + } + + else -> { + null + } + } + if (native == null) { + logger.warn( + "brotli4j: no native decoder for {}/{} -- brotli4j-backed JVM unit tests " + + "(e.g. BrotliDictionaryDecodeTest) will fail with UnsatisfiedLinkError on this host.", + os, + arch, + ) + } + return native +} + dependencies { debugImplementation(libs.common.leakcanary) @@ -353,6 +403,13 @@ dependencies { // brotli4j implementation(libs.brotli4j) + // JVM unit tests (e.g. BrotliDictionaryDecodeTest) run brotli4j's real native decoder, not an + // Android target -- without a desktop native on the test classpath, Brotli4jLoader has nothing + // to load and every such test fails with UnsatisfiedLinkError. Pick the native for whoever is + // building, so the suite runs off a Linux x64 CI runner too (same dispatch as build-logic/plugins'). + // Null on an unrecognized host just means those specific tests fail there -- see + // brotli4jNativeForHost's own warning -- not that this whole build should refuse to configure. + brotli4jNativeForHost()?.let { testImplementation(it) } implementation(libs.common.markwon.core) implementation(libs.common.markwon.linkify) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index 0b76b64d2d..86e42350ff 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -18,15 +18,19 @@ import org.slf4j.LoggerFactory import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.File +import java.io.IOException import java.io.InputStream import java.io.PrintWriter +import java.io.SequenceInputStream import java.io.StringWriter import java.net.InetSocketAddress import java.net.ServerSocket import java.net.Socket import java.net.URLDecoder +import java.nio.ByteBuffer import java.sql.Date import java.text.SimpleDateFormat +import java.util.Collections import java.util.Locale import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit @@ -62,6 +66,45 @@ data class JavaExecutionResult( val timeoutLimit: Long, ) +/** + * Copies [bytes] into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct + * buffer, a heap-backed one throws `IllegalArgumentException`. + * + * The capacity must be exactly [bytes]`.size`: `attachDictionary` reads the whole capacity and + * ignores position/limit, so trailing slack from an over-allocated buffer is treated as dictionary + * content and every decode then fails with `IOException: corrupted input`. + */ +internal fun toDirectByteBuffer(bytes: ByteArray): ByteBuffer = + ByteBuffer.allocateDirect(bytes.size).apply { + put(bytes) + flip() + } + +/** + * Reads [chunks] back to back as one stream, without concatenating them into a new array. + * Cheap to build twice, which the no-dictionary retry in `decompressBrotli` relies on. + */ +internal fun chunksAsStream(chunks: List): InputStream = + SequenceInputStream(Collections.enumeration(chunks.map { ByteArrayInputStream(it) })) + +/** + * Joins [chunks] into one exactly-sized array. A ByteArrayOutputStream would repeatedly double its + * buffer and then hand back a second full copy -- avoidable here since the total is known up front. + * Returns the sole element as-is when there is nothing to join. + */ +internal fun joinChunks(chunks: List): ByteArray { + if (chunks.size == 1) { + return chunks[0] + } + val joined = ByteArray(chunks.sumOf { it.size }) + var offset = 0 + for (chunk in chunks) { + chunk.copyInto(joined, offset) + offset += chunk.size + } + return joined +} + class WebServer( private val config: ServerConfig, ) { @@ -76,6 +119,27 @@ class WebServer( private lateinit var serverSocket: ServerSocket private lateinit var database: SQLiteDatabase private var databaseTimestamp: Long = -1 + + // Timestamp of a debug database whose swap already failed, so a corrupt or unreadable one + // isn't reopened on every single request (it is checked per request). A newer copy has a + // different timestamp and is retried, which is the case that matters -- the developer + // replacing the file is exactly how they'd fix it. + private var failedDebugSwapTimestamp: Long = -1 + + // The shared dictionary Content's brotli-compressed rows are compressed against (see + // ADFA-5153). Lazily (re)loaded on demand, right before the first content fetch that needs + // it after `database` changes -- see compressionDictionaryStale -- rather than eagerly at + // database-open/swap time, but still cached (not reloaded per-request) once loaded for the + // currently active database. Null (no dictionary attached, plain-brotli decode) if the + // active database predates the dictionary-compression migration -- CompressionDictionary + // won't exist yet. + private var compressionDictionary: ByteBuffer? = null + + // Set whenever `database` changes (see switchToDatabase); cleared once compressionDictionary + // has been (re)loaded for that database. Lets the dictionary stay lazily loaded -- only right + // before the first content fetch that actually needs it -- while still loading at most once + // per database change rather than once per request. + private var compressionDictionaryStale = true private val log = LoggerFactory.getLogger(WebServer::class.java) private val debugEnabled: Boolean = File(config.debugEnablePath).exists() @@ -85,8 +149,6 @@ class WebServer( // Frozen at startup; restart the server to pick up a change. private val clearCacheEnabled: Boolean = File(config.clearCacheEnablePath).exists() - private val encodingHeader: String = "Accept-Encoding" - private val brotliCompression: String = "br" private val pebbleEngine = PebbleEngine.Builder().loader(StringLoader()).build() private val templateCache = ConcurrentHashMap() private val gson: Gson = @@ -130,6 +192,105 @@ class WebServer( } } + /** + * Loads the shared Brotli dictionary most Content rows are compressed against (see ADFA-5153). + * Returns null (logged) when the database *definitively* has no dictionary -- predates the + * migration, or has an empty/anomalous `CompressionDictionary` row -- so callers fall back to + * plain, dictionary-free brotli decode (see [decompressBrotli]). Deliberately does *not* catch + * exceptions itself: an unexpected `SQLiteException`/IO failure is likely transient, and the + * caller (see [handleClient]) must not cache that as "no dictionary" the way it does a + * definitive absence, or a transient failure would permanently disable dictionary decoding + * for the rest of this database's lifetime. + */ + private fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { + val tableExists = + db + .rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", + null, + ).use { it.moveToFirst() } + if (!tableExists) { + log.warn("CompressionDictionary table not found; decoding brotli content without a dictionary.") + return null + } + + return db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> + if (!cursor.moveToFirst()) { + log.warn("CompressionDictionary table is empty; decoding brotli content without a dictionary.") + return null + } + val bytes = cursor.getBlob(0) + if (bytes == null) { + log.warn("CompressionDictionary row has a NULL data column; decoding brotli content without a dictionary.") + return null + } + // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects -- + // every row's dictionary decode would then fail with nothing above DEBUG to say why. + if (bytes.isEmpty()) { + log.warn("CompressionDictionary row has an empty data column; decoding brotli content without a dictionary.") + return null + } + toDirectByteBuffer(bytes) + } + } + + /** + * Opens [path] as the active database, refreshing every piece of state that depends on which + * database file is active -- [databaseTimestamp] and the per-database caches + * [bookshelfTemplateId]/[templateCache] -- as one atomic operation. Does *not* load + * [compressionDictionary] itself -- a different database can have a different dictionary (or + * none) -- it only marks [compressionDictionaryStale] so the next content fetch that needs it + * loads it lazily then (see [handleClient]), at most once per database change rather than + * once per request. Only closes the previous database once the new one has opened + * successfully, so a failed swap (this throws) leaves the previous, still-open database + * serving requests rather than leaving [database] referencing an already-closed handle. + */ + private fun switchToDatabase( + path: String, + timestamp: Long, + ) { + val newDatabase = SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY) + if (::database.isInitialized) { + try { + database.close() + } catch (e: Exception) { + log.error("Cannot close previous database: {}", e.message) + } + } + database = newDatabase + databaseTimestamp = timestamp + compressionDictionaryStale = true + bookshelfTemplateId = -1 + templateCache.clear() + } + + /** + * Decompresses one Brotli-compressed Content row. Tries the shared dictionary first, since every + * ADFA-5153-migrated row requires it, then falls back to a plain decode for rows that were never + * dictionary-compressed: plugin-contributed Tier 3 docs (PluginDocumentationManager/BrotliCompressor + * compress with no dictionary) or any row served from a pre-migration database. Attaching a + * dictionary to a stream that wasn't compressed against one reliably fails to decode rather than + * silently producing wrong bytes (verified empirically -- see docs/documentation-database.md), so + * this ordering never lets a dictionary-compressed row fall through to the plain path by accident. + */ + private fun decompressBrotli(chunks: List): ByteArray { + val dictionary = compressionDictionary + if (dictionary != null) { + try { + return BrotliInputStream(chunksAsStream(chunks)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + } catch (e: IOException) { + log.debug( + "Dictionary decode failed for a brotli row (likely dictionary-free plugin content); retrying without a dictionary: {}", + e.message, + ) + } + } + return BrotliInputStream(chunksAsStream(chunks)).use { it.readBytes() } + } + /** * Stops the server by closing the listening socket. Safe to call from any thread. * Causes [start]'s accept loop to exit. If [start] hasn't bound the socket yet -- @@ -165,10 +326,8 @@ class WebServer( config.experimentsEnablePath, ) - databaseTimestamp = getDatabaseTimestamp(config.databasePath) - try { - database = SQLiteDatabase.openDatabase(config.databasePath, null, SQLiteDatabase.OPEN_READONLY) + switchToDatabase(config.databasePath, getDatabaseTimestamp(config.databasePath)) } catch (e: Exception) { log.error("Cannot open database: {}", e.message) return @@ -284,8 +443,6 @@ class WebServer( val writer = PrintWriter(output, true) if (debugEnabled) log.debug(" writer is {}.", writer) - var brotliSupported = false // assume nothing - // Read the request method line, it is always the first line of the request var requestLine = readLineFromStream(input) if (requestLine == null) { @@ -306,7 +463,7 @@ class WebServer( var path = parts[1].split("?")[0] // Discard any HTTP query parameters. path = path.substring(1) - // Read all headers until blank line (needed for Content-Length on POST and Accept-Encoding on GET) + // Read all headers until blank line (needed for Content-Length on POST) val headers = mutableMapOf() while (true) { requestLine = readLineFromStream(input) ?: break @@ -317,7 +474,6 @@ class WebServer( headers[requestLine.substring(0, colon).trim().lowercase()] = requestLine.substring(colon + 1).trim() } } - brotliSupported = headers["accept-encoding"]?.contains(brotliCompression) == true // Playground endpoint: POST only, handled before GET-only check if (false && path == "playground/execute") { @@ -332,11 +488,18 @@ class WebServer( // check to see if there is a newer version of the documentation.db database on the sdcard // if there is use that for our responses val debugDatabaseTimestamp = getDatabaseTimestamp(config.debugDatabasePath, true) - if (debugDatabaseTimestamp > databaseTimestamp) { - bookshelfTemplateId = -1 - database.close() - database = SQLiteDatabase.openDatabase(config.debugDatabasePath, null, SQLiteDatabase.OPEN_READONLY) - databaseTimestamp = debugDatabaseTimestamp + if (debugDatabaseTimestamp > databaseTimestamp && debugDatabaseTimestamp != failedDebugSwapTimestamp) { + try { + switchToDatabase(config.debugDatabasePath, debugDatabaseTimestamp) + failedDebugSwapTimestamp = -1 + } catch (e: Exception) { + failedDebugSwapTimestamp = debugDatabaseTimestamp + log.error( + "Cannot swap to debug database '{}'; ignoring it until it changes: {}", + config.debugDatabasePath, + e.message, + ) + } } // Handle the special "pr" endpoint with highest priority @@ -352,6 +515,22 @@ class WebServer( } } + // Lazily (re)loaded here -- the one place the dictionary is actually consumed (see + // decompressBrotli) -- rather than eagerly at database-open/swap time, but only once per + // database change: a swap (just above) marks compressionDictionaryStale rather than + // reloading immediately, so this only hits the database again when that flag is set. + // Only clears the flag on a clean load (definitive dictionary or definitive absence) -- + // an unexpected exception leaves it set so the next request retries, rather than caching + // a transient failure as "no dictionary" for the rest of this database's lifetime. + if (compressionDictionaryStale) { + try { + compressionDictionary = loadCompressionDictionary(database) + compressionDictionaryStale = false + } catch (e: Exception) { + log.error("Could not load compression dictionary; will retry on the next request: {}", e.message) + } + } + // Database fetch val query = """ SELECT C.content, CT.value, CT.compression, C.templateId @@ -377,24 +556,31 @@ class WebServer( } cursor.moveToFirst() - var dbContent = cursor.getBlob(0) + val firstChunk = cursor.getBlob(0) val dbMimeType = cursor.getString(1) var compression = cursor.getString(2) val templateId = cursor.getInt(3) - // Fragment handling for large content (> 1MB) - if (dbContent.size == contentChunkSize) { + // Fragment handling for large content (> 1MB). The chunks stay a list rather than + // being eagerly concatenated: the old accumulate-into-a-ByteArrayOutputStream-then-copy + // held both the doubling buffer and its toByteArray() copy of the *compressed* chunks + // live at once, on top of the decompressed output that follows -- for the largest + // bundled PDF (8.8 MB over 9 chunks) that's a real, if partial, reduction: the + // decompressed output still goes through a comparable accumulate-then-copy in + // decompressBrotli's own readBytes() call, so the compressed-side saving here doesn't + // eliminate that separate transient. + val chunks = mutableListOf(firstChunk) + if (firstChunk.size == contentChunkSize) { val query2 = "SELECT content FROM Content WHERE path = ? AND languageId = 1" var fragmentNumber = 1 - val combined = ByteArrayOutputStream().apply { write(dbContent) } - var dbContent2 = dbContent - while (dbContent2.size == contentChunkSize) { + var nextChunk = firstChunk + while (nextChunk.size == contentChunkSize) { val path2 = "$path-$fragmentNumber" val cursor2 = database.rawQuery(query2, arrayOf(path2)) try { if (cursor2.moveToFirst()) { - dbContent2 = cursor2.getBlob(0) - combined.write(dbContent2) + nextChunk = cursor2.getBlob(0) + chunks.add(nextChunk) fragmentNumber++ } else { break @@ -403,19 +589,20 @@ class WebServer( cursor2.close() } } - dbContent = combined.toByteArray() } - // If a document is stored in brotli form and the client doesn't support that encoding - // decompress and send that to the client. - // Pebble templates have to be in string form so the retrieved database content may need to be - // decompressed. - if (compression == "brotli" && (!brotliSupported || templateId > 0)) { - dbContent = BrotliInputStream(ByteArrayInputStream(dbContent)).use { it.readBytes() } - compression = "none" - } else if (compression == "brotli") { - compression = "br" - } + // Content is compressed at rest with brotli -- most rows against the shared dictionary + // loaded into compressionDictionary (see ADFA-5153), but plugin-contributed Tier 3 docs + // (PluginDocumentationManager/BrotliCompressor) are plain brotli with no dictionary. + // This server always decompresses before responding, so it never needs to negotiate + // Content-Encoding with the client. + var dbContent = + if (compression == "brotli") { + compression = "none" + decompressBrotli(chunks) + } else { + joinChunks(chunks) + } // If the file is associated with a template, instantiate that template and send the result to the client if (templateId > 0) { @@ -425,7 +612,6 @@ class WebServer( writer.println("HTTP/1.1 200 OK") writer.println("Content-Type: $dbMimeType") writer.println("Content-Length: ${dbContent.size}") - if (compression != "none") writer.println("Content-Encoding: $compression") writer.println("Connection: close") writer.println() writer.flush() @@ -446,7 +632,7 @@ class WebServer( * @param dbContent JSON bytes that will be parsed and supplied as the template context. * @param path The request/content path associated with this template (used for diagnostic/logging purposes). * @param dbMimeType The MIME type of the stored content (used for diagnostic/logging purposes). - * @param compression The compression label of the stored content (e.g., "br", "none") (used for diagnostic/logging purposes). + * @param compression The compression label of the stored content (always "none" by this point, since decompression already happened) (used for diagnostic/logging purposes). * @return The rendered template encoded as UTF-8 bytes. * @throws Exception If the template ID is not found, is duplicated in the database, or if template lookup/instantiation fails. */ diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt new file mode 100644 index 0000000000..80a1ac152c --- /dev/null +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -0,0 +1,251 @@ +package com.itsaky.androidide.localWebServer + +import com.aayushatharva.brotli4j.Brotli4jLoader +import com.aayushatharva.brotli4j.decoder.BrotliInputStream +import com.aayushatharva.brotli4j.encoder.BrotliOutputStream +import com.aayushatharva.brotli4j.encoder.Encoder +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertSame +import org.junit.Assert.assertThrows +import org.junit.BeforeClass +import org.junit.Test +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets +import java.util.Base64 + +// Deliberately routed through production's toDirectByteBuffer rather than allocating here: +// attachDictionary reads the buffer's capacity, so an over-allocated buffer fails every decode. +// Duplicating the allocation would leave that helper untested and let the two drift apart. +private fun decodeBase64ToDirectBuffer(base64: String): ByteBuffer = toDirectByteBuffer(Base64.getDecoder().decode(base64)) + +// Regression coverage for ADFA-5153: documentation.db's Content rows are Brotli-compressed +// against a shared dictionary trained by OfflineDocumentationTools' zstd/brotli CLI pipeline +// (see populate_db.py's DictionaryCompressor), not by brotli4j itself. These fixtures were +// produced by that exact pipeline, so this test is what protects the cross-tool contract: a +// brotli4j upgrade (or native lib change) that silently broke compatibility with the CLI-produced +// wire format would otherwise only surface as garbled content on-device. +class BrotliDictionaryDecodeTest { + companion object { + // Unlike on-device (where ToolsManager/AssetsInstallationHelper already load it before + // WebServer ever runs), nothing loads brotli4j's native lib in a plain JVM unit test -- + // without this, every test below fails with UnsatisfiedLinkError instead of exercising + // real decode behavior. + @JvmStatic + @BeforeClass + fun loadNativeLibrary() { + Brotli4jLoader.ensureAvailability() + } + } + + // A ~3.3 KB zstd fast-cover dictionary trained on synthetic doc-page-like text, and a small + // payload Brotli-compressed against it via the `brotli` CLI's `-D` flag (OfflineDocumentationTools' + // actual encode path) -- see ADFA-5153. + private val dictionaryBase64 = + "N6Qw7OTyEGgfENCSpAP//////49QsrssRMqWGsnNSkLy/zfL/Ef3/zMAADhYoPCcRptTLgAEQIEAAMAS" + + "pykQlqZI41QGmTEGEAIAAAAAAAAAAAAAAABkXQEAAAAAAAAAAAAAAAAAAAABAAAABAAAAAgAAABhY2Ug" + + "dG9jLWVsZW1lbnQgZG9jcy1zaWRlYmFyIGludGVyZmFjZSB2YWwgZnVuIG9iamxlbWVudCB0b2MtZWxl" + + "bWVudCBrb3RsaW4gb3ZlcnJpZGUgdG9jLWVsZW1lbnQgb3ZlciBrb3RsaW4ga290bGluIHZhciBkb2Nz" + + "LXNpZGViYXIgdmFsIGNvbXBhbmlvbiBjb21wZSBmdW4gcGFnZS5wZWIga290bGluIGZ1biB2YXIgb2Jq" + + "ZWN0IHRlbXBsYXRlIGRvY24gdmFyIHRlbXBsYXRlIGludGVyZmFjZSBjb21wYW5pb24gcGFnZS5wZWIg" + + "dmFyIGlua290bGluIENvbnRlbnQtVHlwZSBkb2NzLXNpZGViYXIgbmF2IGludGVyZmFjZSBjb20gdG9j" + + "LWVsZW1lbnQgY29tcGFuaW9uIG9iamVjdCBpbnRlcmZhY2Uga290bGluIGRvY2RlYmFyIG5hdiB0b2Mt" + + "ZWxlbWVudCBDb250ZW50LVR5cGUgdGVtcGxhdGUgdmFyIGNsbiBzaWRlYmFyIHNpZGViYXIgdG9jLWVs" + + "ZW1lbnQgb2JqZWN0IGNvbXBhbmlvbiBpbnRycmlkZSB0b2MtZWxlbWVudCBmdW4gY2xhc3MgdGVtcGxh" + + "dGUgaW50ZXJmYWNlIGRvYyB0b2MtZWxlbWVudCBmdW4gdG9jLWVsZW1lbnQgdmFsIG9iamVjdCBvYmpl" + + "Y3QgdG9jYmplY3QgbmF2IGZ1biBzaWRlYmFyIG92ZXJyaWRlIG9iamVjdCBmdW4gdmFsIG92ZXJhdGUg" + + "aW50ZXJmYWNlIHZhciB0ZW1wbGF0ZSB0ZW1wbGF0ZSB2YXIgb2JqZWN0IGtvdGUga290bGluIG92ZXJy" + + "aWRlIHBhZ2UucGViIG92ZXJyaWRlIGZ1biBjbGFzcyB2YXIgaW50ZXJmYWNlIGNsYXNzIHRlbXBsYXRl" + + "IHNpZGViYXIgZnVuIHBhZ2UucGViIGRvY3NlIENvbnRlbnQtVHlwZSBpbnRlcmZhY2UgdGVtcGxhdGUg" + + "aW50ZXJmYWNlIHZhciB0ZWVudC1UeXBlIENvbnRlbnQtVHlwZSBvYmplY3QgcGFnZS5wZWIgdGVtcGxh" + + "dGUgb3ZlZW50LVR5cGUgb3ZlcnJpZGUgQ29udGVudC1UeXBlIHBhZ2UucGViIGNsYXNzIHNpZGVyIHRv" + + "Yy1lbGVtZW50IHZhciBzaWRlYmFyIG5hdiBmdW4gY2xhc3Mga290bGluIHBhZyBvdmVycmlkZSBpbnRl" + + "cmZhY2UgbmF2IHZhciBvdmVycmlkZSBjb21wYW5pb24gcGFnY2xhc3MgdmFsIGNsYXNzIENvbnRlbnQt" + + "VHlwZSBkb2NzLXNpZGViYXIgbmF2IGNvbXAgZnVuIHRlbXBsYXRlIHBhZ2UucGViIGNsYXNzIG5hdiBw" + + "YWdlLnBlYiBuYXYgQ29udCBjb21wYW5pb24gb3ZlcnJpZGUgdGVtcGxhdGUga290bGluIHNpZGViYXIg" + + "dmFyIHBhdmFsIG5hdiBjbGFzcyBmdW4gb3ZlcnJpZGUgaW50ZXJmYWNlIGludGVyZmFjZSBrb3RudGVu" + + "dC1UeXBlIENvbnRlbnQtVHlwZSBjbGFzcyBvYmplY3QgcGFnZS5wZWIgQ29udGJhciBzaWRlYmFyIHBh" + + "Z2UucGViIHZhbCBDb250ZW50LVR5cGUgdGVtcGxhdGUgdmFsbCBjb21wYW5pb24gZnVuIGRvY3Mtc2lk" + + "ZWJhciBjbGFzcyB0b2MtZWxlbWVudCBDb25kZWJhciB2YWwgZG9jcy1zaWRlYmFyIHZhciBDb250ZW50" + + "LVR5cGUgY2xhc3MgcGFnZXVuIHNpZGViYXIgQ29udGVudC1UeXBlIHZhbCBvYmplY3QgdGVtcGxhdGUg" + + "bmF2IG92ZmFjZSBDb250ZW50LVR5cGUgcGFnZS5wZWIga290bGluIGZ1biBvdmVycmlkZSB2YXJuaW9u" + + "IENvbnRlbnQtVHlwZSBrb3RsaW4gbmF2IHRvYy1lbGVtZW50IG9iamVjdCBvYmF2IG92ZXJyaWRlIHRv" + + "Yy1lbGVtZW50IHZhbCB2YWwgbmF2IG5hdiBvYmplY3QgcGFnbGluIGZ1biB2YWwgY2xhc3MgaW50ZXJm" + + "YWNlIHRvYy1lbGVtZW50IHNpZGViYXIgY29hdGUgc2lkZWJhciB2YXIgQ29udGVudC1UeXBlIGNvbXBh" + + "bmlvbiB2YXIgZnVuIHNpZCBrb3RsaW4gZnVuIENvbnRlbnQtVHlwZSBpbnRlcmZhY2UgdG9jLWVsZW1l" + + "bnQgZnVuYWdlLnBlYiB0ZW1wbGF0ZSBjb21wYW5pb24gdmFyIG92ZXJyaWRlIGtvdGxpbiBuYXZpbnRl" + + "cmZhY2UgZnVuIGludGVyZmFjZSBvYmplY3QgdGVtcGxhdGUgY2xhc3MgZG9jc2xpbiB0ZW1wbGF0ZSB0" + + "b2MtZWxlbWVudCB0b2MtZWxlbWVudCBuYXYga290bGluIGRvbmlvbiB0ZW1wbGF0ZSBvYmplY3QgY2xh" + + "c3Mgb2JqZWN0IENvbnRlbnQtVHlwZSBmdW5lY3QgY2xhc3MgY2xhc3MgdG9jLWVsZW1lbnQgY2xhc3Mg" + + "bmF2IHRlbXBsYXRlIENvbiBuYXYgdGVtcGxhdGUgZnVuIG5hdiBzaWRlYmFyIG92ZXJyaWRlIHZhbCBm" + + "dW4gdmFsZW50IGNsYXNzIHZhbCB2YXIgb2JqZWN0IGNsYXNzIGZ1biBrb3RsaW4gdmFsIGludGVvbXBh" + + "bmlvbiBjbGFzcyBrb3RsaW4gZnVuIGRvY3Mtc2lkZWJhciBrb3RsaW4gQ29udG4gZG9jcy1zaWRlYmFy" + + "IHRvYy1lbGVtZW50IG9iamVjdCB2YWwgbmF2IG5hdiBzaWRlciBDb250ZW50LVR5cGUgbmF2IHBhZ2Uu" + + "cGViIG5hdiBjbGFzcyBvdmVycmlkZSBzaWRpZGViYXIgb2JqZWN0IHNpZGViYXIgdmFsIG5hdiBpbnRl" + + "cmZhY2Ugb2JqZWN0IGRvYyBpbnRlcmZhY2Ugb3ZlcnJpZGUgcGFnZS5wZWIgb3ZlcnJpZGUgb3ZlcnJp" + + "ZGUgY2xhb2NzLXNpZGViYXIgY2xhc3MgY29tcGFuaW9uIGtvdGxpbiB0b2MtZWxlbWVudCBpbnQucGVi" + + "IHRvYy1lbGVtZW50IGNvbXBhbmlvbiBzaWRlYmFyIGRvY3Mtc2lkZWJhciBuYW1lbnQgcGFnZS5wZWIg" + + "dmFsIGtvdGxpbiBvYmplY3QgdmFyIHZhciBvYmplY3QgdGVtYWwgcGFnZS5wZWIgdmFyIHRvYy1lbGVt" + + "ZW50IHRlbXBsYXRlIHBhZ2UucGViIHNpZGVuYXYgcGFnZS5wZWIgdmFyIGtvdGxpbiBpbnRlcmZhY2Ug" + + "c2lkZWJhciB2YXIgY29tcGUga290bGluIGNsYXNzIHZhbCBzaWRlYmFyIHBhZ2UucGViIGludGVyZmFj" + + "ZSBwYWdlZ2UucGViIGNvbXBhbmlvbiBuYXYgb2JqZWN0IGNsYXNzIENvbnRlbnQtVHlwZSB0b2NiYXIg" + + "b3ZlcnJpZGUgdGVtcGxhdGUgdmFyIHNpZGViYXIga290bGluIGZ1biB2YXIgQ25pb24gdmFsIHBhZ2Uu" + + "cGViIGZ1biB0ZW1wbGF0ZSB0b2MtZWxlbWVudCB2YWwgY29tbnRlcmZhY2UgdmFsIGNsYXNzIGNvbXBh" + + "bmlvbiBzaWRlYmFyIHRlbXBsYXRlIGludGV2YWwgdGVtcGxhdGUgdGVtcGxhdGUgb2JqZWN0IG5hdiBk" + + "b2NzLXNpZGViYXIgc2lkZWUgY29tcGFuaW9uIG9iamVjdCBvdmVycmlkZSBmdW4gZnVuIGNvbXBhbmlv" + + "biB0b2MtVHlwZSBvdmVycmlkZSBuYXYgdmFsIHRvYy1lbGVtZW50IGtvdGxpbiB2YXIgbmF2IHBudC1U" + + "eXBlIHZhciBkb2NzLXNpZGViYXIgQ29udGVudC1UeXBlIHNpZGViYXIgcGFnZWViYXIgdmFsIHBhZ2Uu" + + "cGViIG9iamVjdCBmdW4gcGFnZS5wZWIgcGFnZS5wZWIgZG9jbiBvdmVycmlkZSBkb2NzLXNpZGViYXIg" + + "b2JqZWN0IGludGVyZmFjZSBjbGFzcyBrb3RhciB0ZW1wbGF0ZSB2YXIga290bGluIGNvbXBhbmlvbiBk" + + "b2NzLXNpZGViYXIgZnVuICB0b2MtZWxlbWVudCBkb2NzLXNpZGViYXIgaW50ZXJmYWNlIENvbnRlbnQt" + + "VHlwZSBj" + + private val compressedBase64 = + "H6AEIBypU5+7WdgVm1yEUcQuEA0twSdtb3qRIOfy83EJ6BCu9aGiz72LjySb9TQmV4wATYW9JhfwdjwI" + + "woRvurJjIaNH/hC6U59+QaiVFTX9XajztuGO9hS2C2GJEnZn+6vh0spFMR6RDFwzXTjCHWzxThsHAcW2" + + "9ev+Wau/71qnhgYFy8JNHS3F87DOOc02MhMXA9ZP9Ti9LOWqrKld7hlsgT8bDn888jGY1CPGtwU=" + + private val expectedBase64 = + "dmFsIG92ZXJyaWRlIGZ1biB2YXIgaW50ZXJmYWNlIHNpZGViYXIgaW50ZXJmYWNlIHNpZGViYXIgb2Jq" + + "ZWN0IGNsYXNzIGZ1biBDb250ZW50LVR5cGUgcGFnZS5wZWIgZnVuIHNpZGViYXIgaW50ZXJmYWNlIG92" + + "ZXJyaWRlIHNpZGViYXIgb3ZlcnJpZGUgZG9jcy1zaWRlYmFyIGtvdGxpbiBDb250ZW50LVR5cGUgdG9j" + + "LWVsZW1lbnQgb2JqZWN0IG92ZXJyaWRlIGNvbXBhbmlvbiBrb3RsaW4gZG9jcy1zaWRlYmFyIGtvdGxp" + + "biB2YWwgdG9jLWVsZW1lbnQgbmF2IGNvbXBhbmlvbiB2YXIgQ29udGVudC1UeXBlIG92ZXJyaWRlIGNs" + + "YXNzIGtvdGxpbiBuYXYgcGFnZS5wZWIgc2lkZWJhciBDb250ZW50LVR5cGUgb3ZlcnJpZGUgaW50ZXJm" + + "YWNlIHRvYy1lbGVtZW50IGludGVyZmFjZSBzaWRlYmFyIHNpZGViYXIgaW50ZXJmYWNlIG92ZXJyaWRl" + + "IHNpZGViYXIgc2lkZWJhciBmdW4gZG9jcy1zaWRlYmFyIHZhciB2YWwgY2xhc3MgZnVuIHBhZ2UucGVi" + + "IENvbnRlbnQtVHlwZSB2YWwgc2lkZWJhciB2YXIgaW50ZXJmYWNlIGNsYXNzIHRlbXBsYXRlIGludGVy" + + "ZmFjZSBmdW4gdG9jLWVsZW1lbnQgY2xhc3MgdmFsIHRlbXBsYXRlIHNpZGViYXIgY2xhc3MgbmF2IHNp" + + "ZGViYXIgdmFyIG9iamVjdCB2YXIgZG9jcy1zaWRlYmFyIHZhciBpbnRlcmZhY2UgdmFyIHRvYy1lbGVt" + + "ZW50IHRlbXBsYXRlIG9iamVjdCBjb21wYW5pb24ga290bGluIGNvbXBhbmlvbiBvdmVycmlkZSBpbnRl" + + "cmZhY2UgdmFsIG9iamVjdCB0ZW1wbGF0ZSBkb2NzLXNpZGViYXIgZG9jcy1zaWRlYmFyIGludGVyZmFj" + + "ZSBzaWRlYmFyIGRvY3Mtc2lkZWJhciBrb3RsaW4gdmFsIGZ1biBpbnRlcmZhY2UgdGVtcGxhdGUgaW50" + + "ZXJmYWNlIGludGVyZmFjZSBvdmVycmlkZSBkb2NzLXNpZGViYXIgc2lkZWJhciB2YWwgdmFsIG9iamVj" + + "dCBvYmplY3QgdGVtcGxhdGUgdmFsIGtvdGxpbiBuYXYgdGVtcGxhdGUgdGVtcGxhdGUgZnVuIHRvYy1l" + + "bGVtZW50IG92ZXJyaWRlIHRlbXBsYXRlIGludGVyZmFjZSB2YWwgb3ZlcnJpZGUgdmFyIHBhZ2UucGVi" + + "IHZhciBrb3RsaW4gdGVtcGxhdGUgdmFyIHRlbXBsYXRlIG5hdiBuYXYgdGVtcGxhdGUgQ29udGVudC1U" + + "eXBlIGtvdGxpbiB2YWwgaW50ZXJmYWNlIGRvY3Mtc2lkZWJhciBwYWdlLnBlYiBvYmplY3Qgb2JqZWN0" + + "IGZ1biBrb3RsaW4gc2lkZWJhciB2YXIgdGVtcGxhdGUgZG9jcy1zaWRlYmFy" + + @Test + fun `decodes CLI dictionary-compressed content correctly`() { + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val compressed = Base64.getDecoder().decode(compressedBase64) + val expected = Base64.getDecoder().decode(expectedBase64) + + val result = + BrotliInputStream(ByteArrayInputStream(compressed)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + + assertArrayEquals(expected, result) + } + + @Test + fun `the same dictionary buffer instance is safe to reuse across multiple decodes`() { + // WebServer holds one long-lived dictionary buffer across many requests -- + // this guards against a brotli4j change that mutates buffer position/limit + // state in a way that would break the second decode. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val compressed = Base64.getDecoder().decode(compressedBase64) + val expected = Base64.getDecoder().decode(expectedBase64) + + repeat(3) { + val result = + BrotliInputStream(ByteArrayInputStream(compressed)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + assertArrayEquals(expected, result) + } + } + + @Test + fun `decoding dictionary-compressed content without attaching a dictionary fails`() { + // Unlike a *wrong* dictionary (whose backward distances resolve into real, + // just incorrect, bytes -- silently wrong output, no error), decoding with + // no dictionary at all leaves distances that reach into the dictionary + // region out of bounds for any spec-compliant decoder, which must reject + // the stream as corrupt. Verified empirically: brotli4j throws IOException + // here, not an arbitrary Exception subtype. + val compressed = Base64.getDecoder().decode(compressedBase64) + + assertThrows(IOException::class.java) { + BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() } + } + } + + @Test + fun `dictionary-free plugin content fails with a dictionary attached but decodes plain`() { + // Regression coverage for the WebServer.decompressBrotli fallback: plugin-contributed + // Tier 3 docs (PluginDocumentationManager/BrotliCompressor) are compressed with the same + // encoder params (quality 11, window 24) but no dictionary, coexisting in the same Content + // table as ADFA-5153-migrated, dictionary-compressed rows. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val plaintext = "plugin-contributed Tier 3 content, compressed with no dictionary" + val expected = plaintext.toByteArray(StandardCharsets.UTF_8) + val compressed = + ByteArrayOutputStream() + .apply { + BrotliOutputStream(this, Encoder.Parameters().setQuality(11).setWindow(24)).use { it.write(expected) } + }.toByteArray() + + assertThrows(IOException::class.java) { + BrotliInputStream(ByteArrayInputStream(compressed)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + } + + val plainResult = BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() } + assertArrayEquals(expected, plainResult) + } + + @Test + fun `content split across chunks decodes the same as one contiguous array`() { + // Rows over 1 MB are stored as several Content rows and were previously concatenated + // before decoding; they are now fed to the decoder as a stream over the chunk list, so + // a compressed stream must decode identically no matter where the chunk boundaries fall. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val compressed = Base64.getDecoder().decode(compressedBase64) + val expected = Base64.getDecoder().decode(expectedBase64) + + // Deliberately uneven, and not aligned to anything in the brotli stream. + val chunks = + listOf( + compressed.copyOfRange(0, 7), + compressed.copyOfRange(7, 8), + compressed.copyOfRange(8, compressed.size - 1), + compressed.copyOfRange(compressed.size - 1, compressed.size), + ) + + val result = + BrotliInputStream(chunksAsStream(chunks)).use { stream -> + stream.attachDictionary(dictionary) + stream.readBytes() + } + + assertArrayEquals(expected, result) + } + + @Test + fun `joinChunks concatenates in order and sizes the result exactly`() { + val chunks = listOf(byteArrayOf(1, 2, 3), byteArrayOf(), byteArrayOf(4), byteArrayOf(5, 6)) + + val joined = joinChunks(chunks) + + assertArrayEquals(byteArrayOf(1, 2, 3, 4, 5, 6), joined) + assertEquals(6, joined.size) + } + + @Test + fun `joinChunks hands back a lone chunk without copying it`() { + val only = byteArrayOf(7, 8, 9) + + assertSame(only, joinChunks(listOf(only))) + } +} diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt index ef1e18de8f..835d3c0d5d 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt @@ -1,16 +1,19 @@ package com.itsaky.androidide.localWebServer +import android.database.Cursor import android.database.sqlite.SQLiteDatabase import android.net.TrafficStats import io.mockk.every import io.mockk.mockk import io.mockk.mockkStatic import io.mockk.unmockkAll +import io.mockk.verify import org.junit.After import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test +import java.io.File import java.net.InetSocketAddress import java.net.ServerSocket import java.net.Socket @@ -102,6 +105,194 @@ class WebServerTest { assertPortIsFree(port) } + // ADFA-5153: the compression dictionary is loaded lazily -- not merely from starting the + // server -- but only once per database, cached across every subsequent request against that + // same database rather than re-fetched per-request. + @Test + fun `compression dictionary loads lazily on first use, once per database, not once per request`() { + val port = freePort() + + val dictionaryExistsCursor = + mockk(relaxed = true) { + every { moveToFirst() } returns true + } + val dictionaryDataCursor = + mockk(relaxed = true) { + every { moveToFirst() } returns true + every { getBlob(0) } returns "test-dictionary-bytes".toByteArray() + } + val contentCursor = + mockk(relaxed = true) { + every { count } returns 1 + every { moveToFirst() } returns true + every { getBlob(0) } returns "hello".toByteArray() + every { getString(1) } returns "text/plain" + every { getString(2) } returns "none" + every { getInt(3) } returns 0 + } + + val db = mockk(relaxed = true) + every { SQLiteDatabase.openDatabase(any(), isNull(), any()) } returns db + every { + db.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } returns dictionaryExistsCursor + every { + db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } returns dictionaryDataCursor + every { + db.rawQuery(match { it.contains("FROM Content") }, any()) + } returns contentCursor + + val server = WebServer(testConfig(port)) + val serverThread = Thread { server.start() }.apply { isDaemon = true } + serverThread.start() + try { + awaitPortBound(port) + + // Nothing fetches the dictionary merely from starting the server -- only a content + // fetch does, so before any request there should be no dictionary query at all yet -- + // neither the sqlite_master existence check nor the data fetch. + verify(exactly = 0) { + db.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } + verify(exactly = 0) { + db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + + repeat(3) { sendRawGetRequestAndAwaitClose(port, "/some/path") } + + // Exactly one dictionary load across all 3 requests against the same, unchanged + // database -- the first request's lazy load, cached for the other two. Both queries + // loadCompressionDictionary issues (the sqlite_master existence check, then the data + // fetch) must be checked, or a regression re-running just the existence check on + // every request would pass unnoticed. + verify(exactly = 1) { + db.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } + verify(exactly = 1) { + db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + } finally { + server.stop() + serverThread.join(2_000) + } + } + + // ADFA-5153: a database swap (the debug-DB override) must invalidate the cached dictionary -- + // the new database can have a different one, or none -- causing exactly one fresh reload on + // the first content fetch against the new database, not a reload on every later request too. + @Test + fun `database swap invalidates the cached dictionary, reloading it once for the new database`() { + val port = freePort() + val debugDbFile = File.createTempFile("webserver-test-debug", ".db") + debugDbFile.delete() // must not exist yet -- the first request should stay on the primary db + + fun contentCursorFor(marker: String) = + mockk(relaxed = true) { + every { count } returns 1 + every { moveToFirst() } returns true + every { getBlob(0) } returns marker.toByteArray() + every { getString(1) } returns "text/plain" + every { getString(2) } returns "none" + every { getInt(3) } returns 0 + } + + fun stubDatabase( + db: SQLiteDatabase, + dictionaryBytes: String, + ) { + every { + db.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } returns mockk(relaxed = true) { every { moveToFirst() } returns true } + every { + db.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } returns + mockk(relaxed = true) { + every { moveToFirst() } returns true + every { getBlob(0) } returns dictionaryBytes.toByteArray() + } + every { + db.rawQuery(match { it.contains("FROM Content") }, any()) + } returns contentCursorFor(dictionaryBytes) + } + + val primaryDb = mockk(relaxed = true) + val debugDb = mockk(relaxed = true) + stubDatabase(primaryDb, "dict-primary") + stubDatabase(debugDb, "dict-debug") + + val config = testConfig(port).copy(debugDatabasePath = debugDbFile.absolutePath) + every { SQLiteDatabase.openDatabase(config.databasePath, isNull(), any()) } returns primaryDb + every { SQLiteDatabase.openDatabase(config.debugDatabasePath, isNull(), any()) } returns debugDb + + val server = WebServer(config) + val serverThread = Thread { server.start() }.apply { isDaemon = true } + serverThread.start() + try { + awaitPortBound(port) + + sendRawGetRequestAndAwaitClose(port, "/some/path") + verify(exactly = 1) { + primaryDb.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } + verify(exactly = 1) { + primaryDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + verify(exactly = 0) { + debugDb.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } + verify(exactly = 0) { + debugDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + + // Now make the debug override newer than the primary database -- the swap check in + // handleClient() picks this up on the very next request. + debugDbFile.createNewFile() + debugDbFile.setLastModified(System.currentTimeMillis() + 60_000) + + repeat(2) { sendRawGetRequestAndAwaitClose(port, "/some/path") } + + // Exactly one reload for the new (debug) database, across both post-swap requests -- + // not zero (it must invalidate), not two (it must still cache after the first reload). + // Both queries loadCompressionDictionary issues must be checked (see the sibling test). + verify(exactly = 1) { + debugDb.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } + verify(exactly = 1) { + debugDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + // The primary database's dictionary is never touched again after the swap. + verify(exactly = 1) { + primaryDb.rawQuery(match { it.contains("FROM sqlite_master") && it.contains("CompressionDictionary") }, null) + } + verify(exactly = 1) { + primaryDb.rawQuery(match { it.contains("SELECT data FROM CompressionDictionary") }, null) + } + } finally { + server.stop() + serverThread.join(2_000) + debugDbFile.delete() + } + } + + // Blocks until the server closes the connection (every response sends "Connection: close"), + // so by the time this returns the server has fully finished processing this one request -- + // making repeated calls a reliable way to serialize several full request/response cycles. + private fun sendRawGetRequestAndAwaitClose( + port: Int, + path: String, + ) { + Socket().use { socket -> + socket.connect(InetSocketAddress("localhost", port), 2_000) + socket.soTimeout = 2_000 + socket.getOutputStream().apply { + write("GET $path HTTP/1.1\r\n\r\n".toByteArray(Charsets.ISO_8859_1)) + flush() + } + socket.getInputStream().readBytes() + } + } + // Polls by attempting an actual TCP connect rather than sleeping a fixed // duration: as soon as WebServer's accept() loop is listening, the connect // succeeds, which is the readiness signal. (A bind-then-unbind probe was diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 566703ad1b..4c60cf2af1 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -34,7 +34,7 @@ CREATE TABLE Content ( One row per file the web server can serve (HTML, CSS, JS, image, video, PDF, ...) — 30,000+ rows. Key points: - **`path`** is the lookup key (indexed via the `UNIQUE` constraint) and is what `WebServer` matches the HTTP request path against. Paths carry a short source prefix to avoid collisions between doc sets, e.g. `k/index.html` (Kotlin) vs `j/index.html` (Java). -- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. +- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every migrated `Content` row with `ContentTypes.compression = 'brotli'` is Brotli-compressed against the single shared dictionary in `CompressionDictionary` (see below), converted in one pass by ADFA-5153 — but plugin-contributed Tier 3 rows (`PluginDocumentationManager`/`BrotliCompressor`, see below) are plain, dictionary-free Brotli, and there is no per-row flag distinguishing the two, because a dictionary-compressed stream and a plain one are not distinguishable at decode time by inspection. They *are* distinguishable by attempting the decode: attaching the *wrong* dictionary decodes without error to different bytes than were compressed (its backward distances resolve into real, just incorrect, bytes) — but attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are then out of bounds for any spec-compliant decoder. `WebServer` relies on exactly this: it tries the dictionary first and falls back to a plain decode on `IOException`, which correctly handles both dictionary-compressed and plain rows — but never rely on decode success/failure to detect a *wrong* dictionary, since that case is silent. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. - **`templateId`**: `0` (or unset) means `content` is legacy HTML with presentation baked in (the pre-CMS Release 0/1 format). A positive value means `content` is JSON *facts only*, rendered through the matching row in `Templates` (a Pebble template) — the ongoing move to a proper CMS that de-duplicates presentation across near-identical pages (e.g. `sin`/`cos` docs). - The `UNIQUE(path)` constraint rejects any duplicate `path`, regardless of `languageID` — a second language for an existing path isn't supported yet (only `EN-us` currently exists). Getting there needs an upstream schema change to composite uniqueness on `(path, languageID)` (see *Known rough edges* below). @@ -62,6 +62,7 @@ CREATE TABLE Tooltips ( ### Supporting tables +- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `WebServer` loads it lazily -- not merely from starting the server or swapping databases, but on the first content fetch that needs it after `database` changes -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Per row, it tries decoding with the dictionary attached first via brotli4j's `attachDictionary`, falling back to a plain decode on failure — needed both for a database predating this migration (no `CompressionDictionary` table at all) and for plugin-contributed rows within an otherwise-migrated database (see `PluginDocumentationManager` below). - **`Templates(id, name, content)`** — Pebble template source, keyed by id (and by `name` for well-known templates like `bookshelf`). Referenced by `Content.templateId`. - **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`). - **`LastChange(documentationSet, changeTime, who)`** — audit trail for edits made through `docdb-studio`; not shown to end users. `DatabaseVersionResolver` reads the `documentationSet = 'wholedb'` row to report the DB's build/edit stamp in debug logging, falling back to the most recent row of any set if `'wholedb'` is missing. @@ -80,7 +81,7 @@ All three sites below open the file with `SQLiteDatabase.openDatabase(..., OPEN_ AND C.path = ? ``` - then reassembles chunked blobs, decompresses Brotli when the client can't accept it (or when a Pebble template needs a string to render), and instantiates the template if `templateId > 0`. Also serves a Dynamic Bookshelf JSON payload (joining `Content`/`Bookshelf`/`BookCategories`, rendered through the `bookshelf` template) and debug-only HTML dumps at `/pr/db` (`LastChange`, last 20 rows) and `/pr/pr` (recent projects, from a *different* database). + then reassembles chunked blobs, always decompresses Brotli content (attaching `CompressionDictionary`'s bytes first, if loaded — see above) since this server never negotiates `Content-Encoding` with the client, and instantiates the template if `templateId > 0`. Also serves a Dynamic Bookshelf JSON payload (joining `Content`/`Bookshelf`/`BookCategories`, rendered through the `bookshelf` template) and debug-only HTML dumps at `/pr/db` (`LastChange`, last 20 rows) and `/pr/pr` (recent projects, from a *different* database). - **`idetooltips/.../ToolTipManager.kt`** — serves Tier 1/2. Looks up `Tooltips` joined to `TooltipCategories` by `(category, tag)`, then `TooltipButtons` for the Tier 3 links shown at the bottom. - **`plugin-manager/.../documentation/PluginDocumentationManager.kt`** (with `Tier3AssetWalker.kt`, and the `DocumentationExtension` contract in `plugin-api`) — lets plugins contribute their own help content into the same lookup paths.