diff --git a/TESTING.md b/TESTING.md
index a70a294..e83a57c 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -277,6 +277,15 @@ that two filters write version 2 and "the other eleven" version 1 — true at th
structures, wrong from the next release onward. A count in a comment is a roster
maintained by hand, and the fix is to remove the number rather than correct it.
+Not every sweep can be one loop. `TestPersistenceHostilePayloads` refuses payloads whose
+fields are internally absurd, and what counts as absurd differs per structure because the
+fields differ — there is nothing to iterate. Its roster is a map from each structure to
+the test that carries it, and both halves are checked: the structures come from
+`StructureId`, and each named test must exist and be a test, so an entry pointing at
+something renamed or un-attributed fails rather than silently vouching for nothing. That
+sweep had drifted the *other* way from the rest — the newest structures were the
+best-covered, and fifteen of the oldest had no field-level test at all, guards and all.
+
The rosters also disagree with your own audit, which is the point of deriving them. The
span-equivalence sweep was short by six, and adding those six turned up two more the
audit had passed over — `BinaryFuseFilter` and `BloomierFilter`, which are built once
diff --git a/TestProbabilisticDataStructures/TestPersistenceHostilePayloads.cs b/TestProbabilisticDataStructures/TestPersistenceHostilePayloads.cs
index 2facfea..9b76d57 100644
--- a/TestProbabilisticDataStructures/TestPersistenceHostilePayloads.cs
+++ b/TestProbabilisticDataStructures/TestPersistenceHostilePayloads.cs
@@ -1,4 +1,4 @@
-using System;
+using System;
using System.Linq;
using System.Buffers.Binary;
using System.IO;
@@ -45,11 +45,6 @@ private static byte[] PokeUInt32(byte[] original, int payloadOffset, uint value)
return bytes;
}
- ///
- /// Overwrites a u64 at an offset within the payload and repairs the checksum --
- /// the eight-byte sibling of , for doubles poked by
- /// their bit pattern.
- ///
///
/// Overwrites a single byte at an offset within the payload and repairs the
/// checksum.
@@ -1066,5 +1061,441 @@ private static (byte[] Bytes, DpswLayout Layout) SmallDpswPayload()
return (bytes, layout);
}
+
+ ///
+ /// Overwrites the hash id in the envelope header and repairs the checksum. The
+ /// hash id lives before the payload, so the poking helpers above cannot reach
+ /// it.
+ ///
+ private static byte[] PokeHashId(byte[] original, ushort hashId)
+ {
+ var bytes = (byte[])original.Clone();
+ BinaryPrimitives.WriteUInt16LittleEndian(bytes.AsSpan(8), hashId);
+ RepairChecksum(bytes);
+ return bytes;
+ }
+
+ ///
+ /// A filter's declared bit count and the buckets it carries have to describe the
+ /// same filter. They are written separately, so a payload can claim one and
+ /// carry the other.
+ ///
+ [TestMethod]
+ public void TestCountingFilterWhoseSizeDisagreesWithItsBucketsIsRefused()
+ {
+ var bytes = Filled(new CountingBloomFilter(200, 4, 0.01)).ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(PokeUInt32(bytes, 0, 12_345)),
+ "do not describe the same filter");
+ }
+
+ ///
+ /// The deletable filter's data region and its buckets, likewise.
+ ///
+ [TestMethod]
+ public void TestDeletableFilterWhoseSizeDisagreesWithItsBucketsIsRefused()
+ {
+ var bytes = Filled(new DeletableBloomFilter(200, 10, 0.01)).ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(PokeUInt32(bytes, 0, 12_345)),
+ "do not describe the same filter");
+ }
+
+ ///
+ /// A stable filter with no hash functions sets no cells and tests none, so it
+ /// answers no to everything rather than failing.
+ ///
+ [TestMethod]
+ public void TestStableFilterWithNoHashFunctionsIsRefused()
+ {
+ // u32 m, then k.
+ const int KOffset = 4;
+
+ var bytes = Filled(new StableBloomFilter(200, 2, 0.01, seed: 5)).ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(PokeUInt32(bytes, KOffset, 0)),
+ "no hash functions");
+ }
+
+ ///
+ /// The inverse filter divides by its capacity to pick a slot, so a capacity of
+ /// zero is a division by zero on the first add rather than a smaller filter.
+ ///
+ [TestMethod]
+ public void TestInverseFilterWithNoCapacityIsRefused()
+ {
+ var bytes = Filled(new InverseBloomFilter(64)).ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(PokeUInt32(bytes, 0, 0)),
+ "capacity of zero");
+ }
+
+ ///
+ /// An entry whose slot index is past the end of the filter would be written
+ /// outside the array it belongs to.
+ ///
+ [TestMethod]
+ public void TestInverseFilterHoldingAnEntryPastItsCapacityIsRefused()
+ {
+ // u32 capacity, u32 occupied, then the first entry's slot index.
+ const int FirstSlotOffset = 8;
+
+ var bytes = Filled(new InverseBloomFilter(64)).ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeUInt32(bytes, FirstSlotOffset, 100_000)),
+ "beyond its capacity");
+ }
+
+ ///
+ /// A sketch with no rows reports every element as seen ulong.MaxValue times,
+ /// which is the emptiest possible sketch answering as confidently as a full one.
+ ///
+ [TestMethod]
+ public void TestCountMinSketchWithNoRowsIsRefused()
+ {
+ // f64 epsilon, f64 delta, u32 width, then depth.
+ const int DepthOffset = 8 + 8 + 4;
+
+ var bytes = FilledSketch().ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(PokeUInt32(bytes, DepthOffset, 0)),
+ "no rows");
+ }
+
+ ///
+ /// A register index is taken from the top bits of a hash, so a register count
+ /// that is not a power of two cannot be indexed at all.
+ ///
+ [TestMethod]
+ public void TestHyperLogLogWithARegisterCountThatIsNotAPowerOfTwoIsRefused()
+ {
+ var estimator = new HyperLogLog(64);
+ for (var i = 0; i < 40; i++) estimator.Add(Key($"item-{i}"));
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeUInt32(estimator.ToByteArray(), 0, 100)),
+ "not a power of two");
+ }
+
+ ///
+ /// The fingerprint width decides how many bytes each entry occupies, so a width
+ /// this library does not build would be read at the wrong stride.
+ ///
+ [TestMethod]
+ public void TestBinaryFuseFilterWithAnImpossibleFingerprintWidthIsRefused()
+ {
+ // u32 keys, u32 segment length, u32 segment count, u64 seed, then the width.
+ const int WidthOffset = 4 + 4 + 4 + 8;
+
+ var bytes = BinaryFuseFilter.Build(
+ Enumerable.Range(0, 40).Select(i => Key($"item-{i}"))).ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(PokeByte(bytes, WidthOffset, 4)),
+ "8 or 16 bits wide");
+ }
+
+ ///
+ /// The relative accuracy is the whole of a DDSketch's promise: it fixes the
+ /// bucket boundaries, so a value outside nought to one describes no sketch.
+ ///
+ [TestMethod]
+ public void TestDDSketchWithAnImpossibleAccuracyIsRefused()
+ {
+ var sketch = new DDSketch(0.1);
+ for (var i = 1; i <= 40; i++) sketch.Add(i);
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeUInt64(sketch.ToByteArray(), 0, BitConverter.DoubleToUInt64Bits(2.0))),
+ "does not describe a sketch");
+ }
+
+ ///
+ /// A precision outside the buildable range would size the register array to
+ /// something the estimator's own constants do not describe.
+ ///
+ [TestMethod]
+ public void TestHyperLogLogPlusWithAnImpossiblePrecisionIsRefused()
+ {
+ var bytes = FilledHllPlus().ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(PokeUInt32(bytes, 0, 3)),
+ "and this library builds");
+ }
+
+ ///
+ /// There are two representations and no third. A payload naming one this library
+ /// does not have would otherwise be read as whichever the reader defaulted to.
+ ///
+ [TestMethod]
+ public void TestHyperLogLogPlusWithAnUnknownRepresentationIsRefused()
+ {
+ // u32 precision, then the representation byte.
+ const int RepresentationOffset = 4;
+
+ var bytes = FilledHllPlus().ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeByte(bytes, RepresentationOffset, 7)),
+ "only the sparse one and the dense one");
+ }
+
+ ///
+ /// The table is indexed by the quotient bits, so zero of them indexes nothing.
+ ///
+ [TestMethod]
+ public void TestQuotientFilterWithNoQuotientBitsIsRefused()
+ {
+ var filter = new QuotientFilter(100, 0.01);
+ for (var i = 0; i < 40; i++) filter.Add(Key($"item-{i}"));
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeUInt32(filter.ToByteArray(), 0, 0)),
+ "between 1 and 32");
+ }
+
+ ///
+ /// A value at or above theta is one the sampling that produced the rest would
+ /// have thrown away, so its presence says the payload was not written by this.
+ ///
+ [TestMethod]
+ public void TestThetaSketchHoldingAValueAboveItsThresholdIsRefused()
+ {
+ // u32 k, u64 theta, u32 held, then the first value.
+ const int FirstValueOffset = 4 + 8 + 4;
+
+ var sketch = new ThetaSketch(16);
+ for (var i = 0; i < 200; i++) sketch.Add(Key($"item-{i}"));
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeUInt64(sketch.ToByteArray(), FirstValueOffset, ulong.MaxValue)),
+ "at or above its theta");
+ }
+
+ ///
+ /// A signature is a fingerprint and nothing else, so the only thing that can be
+ /// wrong about it is which hash built it -- and comparing fingerprints from
+ /// different hashes gives a number that means nothing.
+ ///
+ [TestMethod]
+ public void TestSimHashSignatureBuiltWithAnotherHashIsRefused()
+ {
+ var bytes = SimHash.Signature(new[] { "a", "b", "c" }).ToByteArray();
+
+ // 2 is the id for a structure that hashes nothing, which a signature is not.
+ AssertRefused(
+ () => Persistence.FromByteArray(PokeHashId(bytes, 2)),
+ "this version builds them with XxHash3");
+ }
+
+ ///
+ /// A sketch with no rows has no cells to count in, and would answer every query
+ /// from an empty median.
+ ///
+ [TestMethod]
+ public void TestCountSketchWithNoRowsIsRefused()
+ {
+ // u32 width, then depth.
+ const int DepthOffset = 4;
+
+ var sketch = new CountSketch(0.5, 0.5);
+ for (var i = 0; i < 40; i++) sketch.Add(Key($"item-{i}"), i % 5);
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeUInt32(sketch.ToByteArray(), DepthOffset, 0)),
+ "no cells to count in");
+ }
+
+ ///
+ /// A key occupies several cells at once, so a table with fewer cells than that
+ /// cannot hold one key.
+ ///
+ [TestMethod]
+ public void TestIbltWithFewerCellsThanAKeyOccupiesIsRefused()
+ {
+ var table = new InvertibleBloomLookupTable(8, 8);
+ for (var i = 0; i < 6; i++)
+ {
+ var key = new byte[8];
+ key[0] = (byte)i;
+ table.Add(key);
+ }
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeUInt32(table.ToByteArray(), 0, 2)),
+ "a key occupies");
+ }
+
+ ///
+ /// A key size of zero leaves the stored keys no width, so every key in the table
+ /// would be the same empty one.
+ ///
+ [TestMethod]
+ public void TestIbltWithNoKeyWidthIsRefused()
+ {
+ // u32 cells, then the key size.
+ const int KeySizeOffset = 4;
+
+ var table = new InvertibleBloomLookupTable(8, 8);
+ table.Add(new byte[8]);
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeUInt32(table.ToByteArray(), KeySizeOffset, 0)),
+ "a key is at least one");
+ }
+
+ ///
+ /// The value width decides how wide each cell is, so one this library does not
+ /// build would be read at the wrong stride.
+ ///
+ [TestMethod]
+ public void TestBloomierFilterWithAnImpossibleValueWidthIsRefused()
+ {
+ // u32 keys, u32 segment length, u32 segment count, u64 seed, then value bits.
+ const int ValueBitsOffset = 4 + 4 + 4 + 8;
+
+ var bytes = BloomierFilter.Build(
+ Enumerable.Range(0, 40).Select(i =>
+ new System.Collections.Generic.KeyValuePair(
+ Key($"item-{i}"), (ulong)i)),
+ 8).ToByteArray();
+
+ AssertRefused(
+ () => Persistence.FromByteArray(
+ PokeUInt32(bytes, ValueBitsOffset, 50)),
+ "between 1 and 40 bits");
+ }
+
+ private static T Filled(T filter) where T : IFilter
+ {
+ for (var i = 0; i < 40; i++) filter.Add(Key($"item-{i}"));
+ return filter;
+ }
+
+ private static CountMinSketch FilledSketch()
+ {
+ var sketch = new CountMinSketch(0.1, 0.5);
+ for (var i = 0; i < 40; i++) sketch.Add(Key($"item-{i % 5}"));
+ return sketch;
+ }
+
+ private static HyperLogLogPlus FilledHllPlus()
+ {
+ var estimator = new HyperLogLogPlus(6);
+ for (var i = 0; i < 40; i++) estimator.Add(Key($"item-{i}"));
+ return estimator;
+ }
+
+ ///
+ /// Every structure is refused at least one payload that is internally absurd.
+ ///
+ ///
+ /// Unlike the sweeps in , this cannot
+ /// be one loop: what is absurd differs per structure, because the fields differ.
+ /// So the coverage is a map from each structure to the test that carries it, and
+ /// both halves are checked -- the roster comes from ,
+ /// which a new structure cannot leave itself off of, and each named test has to
+ /// exist, so an entry pointing at a test someone renamed or deleted fails rather
+ /// than silently vouching for nothing.
+ ///
+ /// The drift here ran backwards from the sweeps: the structures added most
+ /// recently had the most field-level guards tested, and fifteen of the oldest --
+ /// including HyperLogLog, DDSketch and ThetaSketch -- had none at all. Their
+ /// readers had the guards; nothing exercised them.
+ ///
+ ///
+ [TestMethod]
+ public void TestEveryStructureRefusesSomeAbsurdPayload()
+ {
+ var carriedBy = new (StructureId Id, string Test)[]
+ {
+ (StructureId.BloomFilter, nameof(TestFormatVersionZeroIsRefused)),
+ (StructureId.BloomFilter64, nameof(TestAbsurdBucketArrayCountIsRefused)),
+ (StructureId.CountingBloomFilter,
+ nameof(TestCountingFilterWhoseSizeDisagreesWithItsBucketsIsRefused)),
+ (StructureId.DeletableBloomFilter,
+ nameof(TestDeletableFilterWhoseSizeDisagreesWithItsBucketsIsRefused)),
+ (StructureId.PartitionedBloomFilter,
+ nameof(TestAbsurdPartitionCountIsRefused)),
+ (StructureId.ScalableBloomFilter,
+ nameof(TestAbsurdContainedFilterCountIsRefused)),
+ (StructureId.StableBloomFilter,
+ nameof(TestStableFilterWithNoHashFunctionsIsRefused)),
+ (StructureId.InverseBloomFilter,
+ nameof(TestInverseFilterWithNoCapacityIsRefused)),
+ (StructureId.CuckooBloomFilter,
+ nameof(TestAbsurdCuckooBucketCountIsRefused)),
+ (StructureId.CountMinSketch, nameof(TestCountMinSketchWithNoRowsIsRefused)),
+ (StructureId.HyperLogLog,
+ nameof(TestHyperLogLogWithARegisterCountThatIsNotAPowerOfTwoIsRefused)),
+ (StructureId.TopK, nameof(TestAbsurdTopKSizeIsRefused)),
+ (StructureId.MinHashSignature, nameof(TestAbsurdSignatureLengthIsRefused)),
+ (StructureId.BinaryFuseFilter,
+ nameof(TestBinaryFuseFilterWithAnImpossibleFingerprintWidthIsRefused)),
+ (StructureId.DDSketch,
+ nameof(TestDDSketchWithAnImpossibleAccuracyIsRefused)),
+ (StructureId.HyperLogLogPlus,
+ nameof(TestHyperLogLogPlusWithAnImpossiblePrecisionIsRefused)),
+ (StructureId.QuotientFilter,
+ nameof(TestQuotientFilterWithNoQuotientBitsIsRefused)),
+ (StructureId.ThetaSketch,
+ nameof(TestThetaSketchHoldingAValueAboveItsThresholdIsRefused)),
+ (StructureId.SimHashSignature,
+ nameof(TestSimHashSignatureBuiltWithAnotherHashIsRefused)),
+ (StructureId.CountSketch, nameof(TestCountSketchWithNoRowsIsRefused)),
+ (StructureId.InvertibleBloomLookupTable,
+ nameof(TestIbltWithFewerCellsThanAKeyOccupiesIsRefused)),
+ (StructureId.BloomierFilter,
+ nameof(TestBloomierFilterWithAnImpossibleValueWidthIsRefused)),
+ (StructureId.HeavyKeeper, nameof(TestHeavyKeeperTrackingNothingIsRefused)),
+ (StructureId.VarOpt, nameof(TestVarOptKeepingNothingIsRefused)),
+ (StructureId.UltraLogLog,
+ nameof(TestUltraLogLogWithUnsupportedPrecisionIsRefused)),
+ (StructureId.Grafite,
+ nameof(TestGrafiteWithADegenerateMultiplierIsRefused)),
+ (StructureId.InfiniFilter, nameof(TestInfiniFilterWithNoTablesIsRefused)),
+ (StructureId.MementoFilter,
+ nameof(TestMementoFilterWithAnAbsurdMementoWidthIsRefused)),
+ (StructureId.SublimeCountMinSketch,
+ nameof(TestSublimeCountMinSketchWithAWidthThatIsNotAPowerOfTwoIsRefused)),
+ (StructureId.SetSketch,
+ nameof(TestSetSketchWithARegisterAboveItsCeilingIsRefused)),
+ (StructureId.TupleSketch,
+ nameof(TestTupleSketchWithAnUnknownPolicyIsRefused)),
+ (StructureId.PrivateCountMinSketch,
+ nameof(TestAPrivateSketchWithNoNoiseIsRefused)),
+ (StructureId.DpswSketch, nameof(TestADpswWindowWithNoNoiseIsRefused)),
+ };
+
+ foreach (var (id, test) in carriedBy)
+ {
+ var method = typeof(TestPersistenceHostilePayloads).GetMethod(test);
+ Assert.IsNotNull(method,
+ $"{id} is said to be covered by {test}, which is not a test here");
+ Assert.IsTrue(
+ method.GetCustomAttributes(typeof(TestMethodAttribute), false).Length > 0,
+ $"{id} is said to be covered by {test}, which is not a test method");
+ }
+
+ StructureRoster.AssertCoversEveryStructure(
+ "absurd payloads", carriedBy.Select(c => c.Id));
+ }
+
}
}