From 6acaa7c7f63963ee4de83ec5a485869dcfa27ef5 Mon Sep 17 00:00:00 2001 From: Matthew Lorimor Date: Wed, 19 Aug 2026 22:53:47 -0400 Subject: [PATCH 1/2] Refuse an absurd payload for every structure, not just the newest TestPersistenceHostilePayloads exercised the field-level guards of 18 of the 33 structures. The drift ran backwards from the sweeps fixed earlier: the structures added most recently had the most guards tested, and fifteen of the oldest -- HyperLogLog, DDSketch, ThetaSketch, CountMinSketch, QuotientFilter, BinaryFuseFilter, BloomierFilter and the rest -- had none at all. Their readers already had the guards. Nothing exercised them, so a guard that had stopped firing would have been found by a user with an edited file rather than here. Eighteen tests now reach them, each asserting the message its own guard produces, which is what keeps a payload refused for the right reason. TestEveryStructureRefusesSomeAbsurdPayload is the completeness check. It cannot be one loop the way the persistence sweeps are, because what counts as absurd differs per structure, so it is a map from each structure to the test carrying it -- with both halves checked. The roster is StructureId, and each named test has to exist and be a test, so an entry pointing at something renamed or un-attributed fails rather than silently vouching for nothing. Also removes an orphaned doc comment that had drifted off PokeUInt64 and stacked a second onto PokeByte. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JH2NRDbhF5bwAsTAP7znb9 --- .../TestPersistenceHostilePayloads.cs | 443 +++++++++++++++++- 1 file changed, 437 insertions(+), 6 deletions(-) diff --git a/TestProbabilisticDataStructures/TestPersistenceHostilePayloads.cs b/TestProbabilisticDataStructures/TestPersistenceHostilePayloads.cs index 2facfea..9b76d57 100644 --- a/TestProbabilisticDataStructures/TestPersistenceHostilePayloads.cs +++ b/TestProbabilisticDataStructures/TestPersistenceHostilePayloads.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Buffers.Binary; using System.IO; @@ -45,11 +45,6 @@ private static byte[] PokeUInt32(byte[] original, int payloadOffset, uint value) return bytes; } - /// - /// Overwrites a u64 at an offset within the payload and repairs the checksum -- - /// the eight-byte sibling of , for doubles poked by - /// their bit pattern. - /// /// /// Overwrites a single byte at an offset within the payload and repairs the /// checksum. @@ -1066,5 +1061,441 @@ private static (byte[] Bytes, DpswLayout Layout) SmallDpswPayload() return (bytes, layout); } + + /// + /// Overwrites the hash id in the envelope header and repairs the checksum. The + /// hash id lives before the payload, so the poking helpers above cannot reach + /// it. + /// + private static byte[] PokeHashId(byte[] original, ushort hashId) + { + var bytes = (byte[])original.Clone(); + BinaryPrimitives.WriteUInt16LittleEndian(bytes.AsSpan(8), hashId); + RepairChecksum(bytes); + return bytes; + } + + /// + /// A filter's declared bit count and the buckets it carries have to describe the + /// same filter. They are written separately, so a payload can claim one and + /// carry the other. + /// + [TestMethod] + public void TestCountingFilterWhoseSizeDisagreesWithItsBucketsIsRefused() + { + var bytes = Filled(new CountingBloomFilter(200, 4, 0.01)).ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray(PokeUInt32(bytes, 0, 12_345)), + "do not describe the same filter"); + } + + /// + /// The deletable filter's data region and its buckets, likewise. + /// + [TestMethod] + public void TestDeletableFilterWhoseSizeDisagreesWithItsBucketsIsRefused() + { + var bytes = Filled(new DeletableBloomFilter(200, 10, 0.01)).ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray(PokeUInt32(bytes, 0, 12_345)), + "do not describe the same filter"); + } + + /// + /// A stable filter with no hash functions sets no cells and tests none, so it + /// answers no to everything rather than failing. + /// + [TestMethod] + public void TestStableFilterWithNoHashFunctionsIsRefused() + { + // u32 m, then k. + const int KOffset = 4; + + var bytes = Filled(new StableBloomFilter(200, 2, 0.01, seed: 5)).ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray(PokeUInt32(bytes, KOffset, 0)), + "no hash functions"); + } + + /// + /// The inverse filter divides by its capacity to pick a slot, so a capacity of + /// zero is a division by zero on the first add rather than a smaller filter. + /// + [TestMethod] + public void TestInverseFilterWithNoCapacityIsRefused() + { + var bytes = Filled(new InverseBloomFilter(64)).ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray(PokeUInt32(bytes, 0, 0)), + "capacity of zero"); + } + + /// + /// An entry whose slot index is past the end of the filter would be written + /// outside the array it belongs to. + /// + [TestMethod] + public void TestInverseFilterHoldingAnEntryPastItsCapacityIsRefused() + { + // u32 capacity, u32 occupied, then the first entry's slot index. + const int FirstSlotOffset = 8; + + var bytes = Filled(new InverseBloomFilter(64)).ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray( + PokeUInt32(bytes, FirstSlotOffset, 100_000)), + "beyond its capacity"); + } + + /// + /// A sketch with no rows reports every element as seen ulong.MaxValue times, + /// which is the emptiest possible sketch answering as confidently as a full one. + /// + [TestMethod] + public void TestCountMinSketchWithNoRowsIsRefused() + { + // f64 epsilon, f64 delta, u32 width, then depth. + const int DepthOffset = 8 + 8 + 4; + + var bytes = FilledSketch().ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray(PokeUInt32(bytes, DepthOffset, 0)), + "no rows"); + } + + /// + /// A register index is taken from the top bits of a hash, so a register count + /// that is not a power of two cannot be indexed at all. + /// + [TestMethod] + public void TestHyperLogLogWithARegisterCountThatIsNotAPowerOfTwoIsRefused() + { + var estimator = new HyperLogLog(64); + for (var i = 0; i < 40; i++) estimator.Add(Key($"item-{i}")); + + AssertRefused( + () => Persistence.FromByteArray( + PokeUInt32(estimator.ToByteArray(), 0, 100)), + "not a power of two"); + } + + /// + /// The fingerprint width decides how many bytes each entry occupies, so a width + /// this library does not build would be read at the wrong stride. + /// + [TestMethod] + public void TestBinaryFuseFilterWithAnImpossibleFingerprintWidthIsRefused() + { + // u32 keys, u32 segment length, u32 segment count, u64 seed, then the width. + const int WidthOffset = 4 + 4 + 4 + 8; + + var bytes = BinaryFuseFilter.Build( + Enumerable.Range(0, 40).Select(i => Key($"item-{i}"))).ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray(PokeByte(bytes, WidthOffset, 4)), + "8 or 16 bits wide"); + } + + /// + /// The relative accuracy is the whole of a DDSketch's promise: it fixes the + /// bucket boundaries, so a value outside nought to one describes no sketch. + /// + [TestMethod] + public void TestDDSketchWithAnImpossibleAccuracyIsRefused() + { + var sketch = new DDSketch(0.1); + for (var i = 1; i <= 40; i++) sketch.Add(i); + + AssertRefused( + () => Persistence.FromByteArray( + PokeUInt64(sketch.ToByteArray(), 0, BitConverter.DoubleToUInt64Bits(2.0))), + "does not describe a sketch"); + } + + /// + /// A precision outside the buildable range would size the register array to + /// something the estimator's own constants do not describe. + /// + [TestMethod] + public void TestHyperLogLogPlusWithAnImpossiblePrecisionIsRefused() + { + var bytes = FilledHllPlus().ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray(PokeUInt32(bytes, 0, 3)), + "and this library builds"); + } + + /// + /// There are two representations and no third. A payload naming one this library + /// does not have would otherwise be read as whichever the reader defaulted to. + /// + [TestMethod] + public void TestHyperLogLogPlusWithAnUnknownRepresentationIsRefused() + { + // u32 precision, then the representation byte. + const int RepresentationOffset = 4; + + var bytes = FilledHllPlus().ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray( + PokeByte(bytes, RepresentationOffset, 7)), + "only the sparse one and the dense one"); + } + + /// + /// The table is indexed by the quotient bits, so zero of them indexes nothing. + /// + [TestMethod] + public void TestQuotientFilterWithNoQuotientBitsIsRefused() + { + var filter = new QuotientFilter(100, 0.01); + for (var i = 0; i < 40; i++) filter.Add(Key($"item-{i}")); + + AssertRefused( + () => Persistence.FromByteArray( + PokeUInt32(filter.ToByteArray(), 0, 0)), + "between 1 and 32"); + } + + /// + /// A value at or above theta is one the sampling that produced the rest would + /// have thrown away, so its presence says the payload was not written by this. + /// + [TestMethod] + public void TestThetaSketchHoldingAValueAboveItsThresholdIsRefused() + { + // u32 k, u64 theta, u32 held, then the first value. + const int FirstValueOffset = 4 + 8 + 4; + + var sketch = new ThetaSketch(16); + for (var i = 0; i < 200; i++) sketch.Add(Key($"item-{i}")); + + AssertRefused( + () => Persistence.FromByteArray( + PokeUInt64(sketch.ToByteArray(), FirstValueOffset, ulong.MaxValue)), + "at or above its theta"); + } + + /// + /// A signature is a fingerprint and nothing else, so the only thing that can be + /// wrong about it is which hash built it -- and comparing fingerprints from + /// different hashes gives a number that means nothing. + /// + [TestMethod] + public void TestSimHashSignatureBuiltWithAnotherHashIsRefused() + { + var bytes = SimHash.Signature(new[] { "a", "b", "c" }).ToByteArray(); + + // 2 is the id for a structure that hashes nothing, which a signature is not. + AssertRefused( + () => Persistence.FromByteArray(PokeHashId(bytes, 2)), + "this version builds them with XxHash3"); + } + + /// + /// A sketch with no rows has no cells to count in, and would answer every query + /// from an empty median. + /// + [TestMethod] + public void TestCountSketchWithNoRowsIsRefused() + { + // u32 width, then depth. + const int DepthOffset = 4; + + var sketch = new CountSketch(0.5, 0.5); + for (var i = 0; i < 40; i++) sketch.Add(Key($"item-{i}"), i % 5); + + AssertRefused( + () => Persistence.FromByteArray( + PokeUInt32(sketch.ToByteArray(), DepthOffset, 0)), + "no cells to count in"); + } + + /// + /// A key occupies several cells at once, so a table with fewer cells than that + /// cannot hold one key. + /// + [TestMethod] + public void TestIbltWithFewerCellsThanAKeyOccupiesIsRefused() + { + var table = new InvertibleBloomLookupTable(8, 8); + for (var i = 0; i < 6; i++) + { + var key = new byte[8]; + key[0] = (byte)i; + table.Add(key); + } + + AssertRefused( + () => Persistence.FromByteArray( + PokeUInt32(table.ToByteArray(), 0, 2)), + "a key occupies"); + } + + /// + /// A key size of zero leaves the stored keys no width, so every key in the table + /// would be the same empty one. + /// + [TestMethod] + public void TestIbltWithNoKeyWidthIsRefused() + { + // u32 cells, then the key size. + const int KeySizeOffset = 4; + + var table = new InvertibleBloomLookupTable(8, 8); + table.Add(new byte[8]); + + AssertRefused( + () => Persistence.FromByteArray( + PokeUInt32(table.ToByteArray(), KeySizeOffset, 0)), + "a key is at least one"); + } + + /// + /// The value width decides how wide each cell is, so one this library does not + /// build would be read at the wrong stride. + /// + [TestMethod] + public void TestBloomierFilterWithAnImpossibleValueWidthIsRefused() + { + // u32 keys, u32 segment length, u32 segment count, u64 seed, then value bits. + const int ValueBitsOffset = 4 + 4 + 4 + 8; + + var bytes = BloomierFilter.Build( + Enumerable.Range(0, 40).Select(i => + new System.Collections.Generic.KeyValuePair( + Key($"item-{i}"), (ulong)i)), + 8).ToByteArray(); + + AssertRefused( + () => Persistence.FromByteArray( + PokeUInt32(bytes, ValueBitsOffset, 50)), + "between 1 and 40 bits"); + } + + private static T Filled(T filter) where T : IFilter + { + for (var i = 0; i < 40; i++) filter.Add(Key($"item-{i}")); + return filter; + } + + private static CountMinSketch FilledSketch() + { + var sketch = new CountMinSketch(0.1, 0.5); + for (var i = 0; i < 40; i++) sketch.Add(Key($"item-{i % 5}")); + return sketch; + } + + private static HyperLogLogPlus FilledHllPlus() + { + var estimator = new HyperLogLogPlus(6); + for (var i = 0; i < 40; i++) estimator.Add(Key($"item-{i}")); + return estimator; + } + + /// + /// Every structure is refused at least one payload that is internally absurd. + /// + /// + /// Unlike the sweeps in , this cannot + /// be one loop: what is absurd differs per structure, because the fields differ. + /// So the coverage is a map from each structure to the test that carries it, and + /// both halves are checked -- the roster comes from , + /// which a new structure cannot leave itself off of, and each named test has to + /// exist, so an entry pointing at a test someone renamed or deleted fails rather + /// than silently vouching for nothing. + /// + /// The drift here ran backwards from the sweeps: the structures added most + /// recently had the most field-level guards tested, and fifteen of the oldest -- + /// including HyperLogLog, DDSketch and ThetaSketch -- had none at all. Their + /// readers had the guards; nothing exercised them. + /// + /// + [TestMethod] + public void TestEveryStructureRefusesSomeAbsurdPayload() + { + var carriedBy = new (StructureId Id, string Test)[] + { + (StructureId.BloomFilter, nameof(TestFormatVersionZeroIsRefused)), + (StructureId.BloomFilter64, nameof(TestAbsurdBucketArrayCountIsRefused)), + (StructureId.CountingBloomFilter, + nameof(TestCountingFilterWhoseSizeDisagreesWithItsBucketsIsRefused)), + (StructureId.DeletableBloomFilter, + nameof(TestDeletableFilterWhoseSizeDisagreesWithItsBucketsIsRefused)), + (StructureId.PartitionedBloomFilter, + nameof(TestAbsurdPartitionCountIsRefused)), + (StructureId.ScalableBloomFilter, + nameof(TestAbsurdContainedFilterCountIsRefused)), + (StructureId.StableBloomFilter, + nameof(TestStableFilterWithNoHashFunctionsIsRefused)), + (StructureId.InverseBloomFilter, + nameof(TestInverseFilterWithNoCapacityIsRefused)), + (StructureId.CuckooBloomFilter, + nameof(TestAbsurdCuckooBucketCountIsRefused)), + (StructureId.CountMinSketch, nameof(TestCountMinSketchWithNoRowsIsRefused)), + (StructureId.HyperLogLog, + nameof(TestHyperLogLogWithARegisterCountThatIsNotAPowerOfTwoIsRefused)), + (StructureId.TopK, nameof(TestAbsurdTopKSizeIsRefused)), + (StructureId.MinHashSignature, nameof(TestAbsurdSignatureLengthIsRefused)), + (StructureId.BinaryFuseFilter, + nameof(TestBinaryFuseFilterWithAnImpossibleFingerprintWidthIsRefused)), + (StructureId.DDSketch, + nameof(TestDDSketchWithAnImpossibleAccuracyIsRefused)), + (StructureId.HyperLogLogPlus, + nameof(TestHyperLogLogPlusWithAnImpossiblePrecisionIsRefused)), + (StructureId.QuotientFilter, + nameof(TestQuotientFilterWithNoQuotientBitsIsRefused)), + (StructureId.ThetaSketch, + nameof(TestThetaSketchHoldingAValueAboveItsThresholdIsRefused)), + (StructureId.SimHashSignature, + nameof(TestSimHashSignatureBuiltWithAnotherHashIsRefused)), + (StructureId.CountSketch, nameof(TestCountSketchWithNoRowsIsRefused)), + (StructureId.InvertibleBloomLookupTable, + nameof(TestIbltWithFewerCellsThanAKeyOccupiesIsRefused)), + (StructureId.BloomierFilter, + nameof(TestBloomierFilterWithAnImpossibleValueWidthIsRefused)), + (StructureId.HeavyKeeper, nameof(TestHeavyKeeperTrackingNothingIsRefused)), + (StructureId.VarOpt, nameof(TestVarOptKeepingNothingIsRefused)), + (StructureId.UltraLogLog, + nameof(TestUltraLogLogWithUnsupportedPrecisionIsRefused)), + (StructureId.Grafite, + nameof(TestGrafiteWithADegenerateMultiplierIsRefused)), + (StructureId.InfiniFilter, nameof(TestInfiniFilterWithNoTablesIsRefused)), + (StructureId.MementoFilter, + nameof(TestMementoFilterWithAnAbsurdMementoWidthIsRefused)), + (StructureId.SublimeCountMinSketch, + nameof(TestSublimeCountMinSketchWithAWidthThatIsNotAPowerOfTwoIsRefused)), + (StructureId.SetSketch, + nameof(TestSetSketchWithARegisterAboveItsCeilingIsRefused)), + (StructureId.TupleSketch, + nameof(TestTupleSketchWithAnUnknownPolicyIsRefused)), + (StructureId.PrivateCountMinSketch, + nameof(TestAPrivateSketchWithNoNoiseIsRefused)), + (StructureId.DpswSketch, nameof(TestADpswWindowWithNoNoiseIsRefused)), + }; + + foreach (var (id, test) in carriedBy) + { + var method = typeof(TestPersistenceHostilePayloads).GetMethod(test); + Assert.IsNotNull(method, + $"{id} is said to be covered by {test}, which is not a test here"); + Assert.IsTrue( + method.GetCustomAttributes(typeof(TestMethodAttribute), false).Length > 0, + $"{id} is said to be covered by {test}, which is not a test method"); + } + + StructureRoster.AssertCoversEveryStructure( + "absurd payloads", carriedBy.Select(c => c.Id)); + } + } } From 62e07e3bd67ae63dc42751af6d5a9b4bcc300902 Mon Sep 17 00:00:00 2001 From: Matthew Lorimor Date: Wed, 19 Aug 2026 22:56:05 -0400 Subject: [PATCH 2/2] Record the map-shaped roster in TESTING.md Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JH2NRDbhF5bwAsTAP7znb9 --- TESTING.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/TESTING.md b/TESTING.md index a70a294..e83a57c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -277,6 +277,15 @@ that two filters write version 2 and "the other eleven" version 1 — true at th structures, wrong from the next release onward. A count in a comment is a roster maintained by hand, and the fix is to remove the number rather than correct it. +Not every sweep can be one loop. `TestPersistenceHostilePayloads` refuses payloads whose +fields are internally absurd, and what counts as absurd differs per structure because the +fields differ — there is nothing to iterate. Its roster is a map from each structure to +the test that carries it, and both halves are checked: the structures come from +`StructureId`, and each named test must exist and be a test, so an entry pointing at +something renamed or un-attributed fails rather than silently vouching for nothing. That +sweep had drifted the *other* way from the rest — the newest structures were the +best-covered, and fifteen of the oldest had no field-level test at all, guards and all. + The rosters also disagree with your own audit, which is the point of deriving them. The span-equivalence sweep was short by six, and adding those six turned up two more the audit had passed over — `BinaryFuseFilter` and `BloomierFilter`, which are built once