diff --git a/TestProbabilisticDataStructures/StructureRoster.cs b/TestProbabilisticDataStructures/StructureRoster.cs
index 4298c71..f7a8a38 100644
--- a/TestProbabilisticDataStructures/StructureRoster.cs
+++ b/TestProbabilisticDataStructures/StructureRoster.cs
@@ -64,6 +64,39 @@ internal static class StructureRoster
.OrderBy(t => t.Name, StringComparer.Ordinal)
.ToArray();
+ ///
+ /// Every public structure offering a span query that answers without changing
+ /// anything: Test, Count, TryGetValue.
+ ///
+ internal static IReadOnlyList WithPureSpanQueries { get; } =
+ SpanMethodsWhere(name => name != "Add" && !Mutates(name));
+
+ ///
+ /// Every public structure offering a span query that answers *and* changes the
+ /// structure: TestAndAdd, TestAndRemove, Remove.
+ ///
+ ///
+ /// These need a different check from the pure ones. Two overloads returning the
+ /// same answer is only half of it -- an array path and a span path can agree on
+ /// every answer and still leave the structure in different states, and a filter
+ /// that answers correctly while holding the wrong thing fails later, somewhere
+ /// else, for no visible reason.
+ ///
+ internal static IReadOnlyList WithMutatingSpanQueries { get; } =
+ SpanMethodsWhere(Mutates);
+
+ private static bool Mutates(string method) =>
+ method is "TestAndAdd" or "TestAndRemove" or "Remove";
+
+ private static IReadOnlyList SpanMethodsWhere(Func wanted) =>
+ Library.GetTypes()
+ .Where(t => t.IsPublic && !t.IsAbstract && IsAStructure(t))
+ .Where(t => t.GetMethods(BindingFlags.Public | BindingFlags.Instance)
+ .Any(m => wanted(m.Name) && m.GetParameters()
+ .Any(param => param.ParameterType == typeof(ReadOnlySpan))))
+ .OrderBy(t => t.Name, StringComparer.Ordinal)
+ .ToArray();
+
///
/// Every public structure that can be handed a hash as it is built, whether
/// through a constructor or a static factory. This is a wider set than
diff --git a/TestProbabilisticDataStructures/TestSpanOverloads.cs b/TestProbabilisticDataStructures/TestSpanOverloads.cs
index 58196bf..ea9b1cc 100644
--- a/TestProbabilisticDataStructures/TestSpanOverloads.cs
+++ b/TestProbabilisticDataStructures/TestSpanOverloads.cs
@@ -1,4 +1,5 @@
using System;
+using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.VisualStudio.TestTools.UnitTesting;
@@ -231,55 +232,285 @@ public void TestSpanAndArrayPathsLeaveIdenticalState()
}
///
- /// Queries must answer identically through either overload, sliced.
+ /// A span query, as a delegate. A span cannot travel through , so the sweeps below need their own delegate types to hold one.
+ ///
+ private delegate object SpanQuery(T structure, ReadOnlySpan data);
+
+ private delegate void SpanMutation(T structure, ReadOnlySpan data);
+
+ ///
+ /// Queries that answer without changing anything must answer identically
+ /// through either overload, sliced.
///
[TestMethod]
public void TestSpanAndArrayQueriesAgree()
{
- var (buffer, slices) = PackedKeys();
+ var bloom = Filled(new BloomFilter(1000, 0.01), (f, k) => f.Add(k));
+ var bloom64 = Filled(new BloomFilter64(1000, 0.01), (f, k) => f.Add(k));
+ var counting = Filled(new CountingBloomFilter(1000, 4, 0.01), (f, k) => f.Add(k));
+ var deletable = Filled(new DeletableBloomFilter(1000, 100, 0.01), (f, k) => f.Add(k));
+ var partitioned = Filled(new PartitionedBloomFilter(1000, 0.01), (f, k) => f.Add(k));
+ var scalable = Filled(new ScalableBloomFilter(100, 0.01, 0.8), (f, k) => f.Add(k));
+ var stable = Filled(new StableBloomFilter(1000, 4, 0.01, seed: 5), (f, k) => f.Add(k));
+ var inverse = Filled(new InverseBloomFilter(500), (f, k) => f.Add(k));
+ var cuckoo = Filled(new CuckooBloomFilter(1000, 0.01, seed: 9), (f, k) => f.Add(k));
+ var quotient = Filled(new QuotientFilter(1000, 0.01), (f, k) => f.Add(k));
+ var infini = Filled(new InfiniFilter(64, 8), (f, k) => f.Add(k));
+ var cms = Filled(new CountMinSketch(0.001, 0.01), (s, k) => s.Add(k));
+ var countSketch = Filled(new CountSketch(0.01, 0.01), (s, k) => s.Add(k, 3));
+ var keeper = Filled(new HeavyKeeper(20, 512, seed: 3), (s, k) => s.Add(k));
+ var sublime = Filled(new SublimeCountMinSketch(0.01, 0.5, 1.0), (s, k) => s.Add(k));
+ var priv = Filled(new PrivateCountMinSketch(64, 4, 1.0, seed: 3), (s, k) => s.Add(k));
+ var dpsw = Filled(
+ new DpswSketch(window: 128, rho: 4.0, alpha: 0.6, width: 8, depth: 2, seed: 3),
+ (s, k) => s.Add(k));
+
+ var half = Enumerable.Range(0, Items).Where(i => i % 2 == 0).ToArray();
+ var fuse = BinaryFuseFilter.Build(half.Select(KeyAt));
+ var bloomier = BloomierFilter.Build(
+ // Eight value bits, so the values have to stay under 256.
+ half.Select(i => new KeyValuePair(KeyAt(i), (ulong)(i % 200))), 8);
- var bloom = new BloomFilter(1000, 0.01);
- var cms = new CountMinSketch(0.001, 0.01);
- var cs = new CountSketch(0.01, 0.01);
- var qf = new QuotientFilter(1000, 0.01);
- var inverse = new InverseBloomFilter(500);
+ var covered = new[]
+ {
+ AssertQueryAgrees("BloomFilter.Test", bloom,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("BloomFilter64.Test", bloom64,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("CountingBloomFilter.Test", counting,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("DeletableBloomFilter.Test", deletable,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("PartitionedBloomFilter.Test", partitioned,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("ScalableBloomFilter.Test", scalable,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("StableBloomFilter.Test", stable,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("InverseBloomFilter.Test", inverse,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("CuckooBloomFilter.Test", cuckoo,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("QuotientFilter.Test", quotient,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("InfiniFilter.Test", infini,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+ AssertQueryAgrees("BinaryFuseFilter.Test", fuse,
+ (f, k) => f.Test(k), (f, s) => f.Test(s)),
+
+ AssertQueryAgrees("CountMinSketch.Count", cms,
+ (s, k) => s.Count(k), (s, p) => s.Count(p)),
+ AssertQueryAgrees("CountSketch.Count", countSketch,
+ (s, k) => s.Count(k), (s, p) => s.Count(p)),
+ AssertQueryAgrees("HeavyKeeper.Count", keeper,
+ (s, k) => s.Count(k), (s, p) => s.Count(p)),
+ AssertQueryAgrees("SublimeCountMinSketch.Count", sublime,
+ (s, k) => s.Count(k), (s, p) => s.Count(p)),
+
+ // The noisy pair need no special treatment, which is worth saying
+ // because it looks as though they should. Their noise is drawn once at
+ // construction and lives in the counters -- that is what stops repeated
+ // queries from averaging it away -- so a query is an ordinary read and
+ // two calls must return the identical double, not merely a close one.
+ AssertQueryAgrees("PrivateCountMinSketch.Count", priv,
+ (s, k) => s.Count(k), (s, p) => s.Count(p)),
+ AssertQueryAgrees("DpswSketch.Count", dpsw,
+ (s, k) => s.Count(k), (s, p) => s.Count(p)),
+
+ AssertQueryAgrees("BloomierFilter.TryGetValue", bloomier,
+ (f, k) => { var ok = f.TryGetValue(k, out var v); return (ok, v); },
+ (f, s) => { var ok = f.TryGetValue(s, out var v); return (ok, v); }),
+ };
+
+ StructureRoster.AssertCoversEveryType(
+ "pure span queries", StructureRoster.WithPureSpanQueries, covered);
+ }
+
+ ///
+ /// Queries that answer *and* change the structure must do both identically.
+ ///
+ ///
+ /// Agreeing on every answer is only half of this. Two paths can return the same
+ /// value at every step and leave the structure holding different things, and a
+ /// filter that answers correctly while holding the wrong thing does not fail
+ /// here -- it fails later, somewhere else, for no visible reason. So each pair
+ /// is driven step for step and then compared through its payload, the same
+ /// oracle the equivalence sweep uses.
+ ///
+ [TestMethod]
+ public void TestSpanAndArrayMutatingQueriesAgree()
+ {
+ var covered = new[]
+ {
+ AssertMutatingQueryAgrees("BloomFilter.TestAndAdd",
+ () => new BloomFilter(1000, 0.01),
+ (f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
+ AssertMutatingQueryAgrees("BloomFilter64.TestAndAdd",
+ () => new BloomFilter64(1000, 0.01),
+ (f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
+ AssertMutatingQueryAgrees("PartitionedBloomFilter.TestAndAdd",
+ () => new PartitionedBloomFilter(1000, 0.01),
+ (f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
+ AssertMutatingQueryAgrees("ScalableBloomFilter.TestAndAdd",
+ () => new ScalableBloomFilter(100, 0.01, 0.8),
+ (f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
+ AssertMutatingQueryAgrees("InverseBloomFilter.TestAndAdd",
+ () => new InverseBloomFilter(500),
+ (f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
+ AssertMutatingQueryAgrees("StableBloomFilter.TestAndAdd",
+ () => new StableBloomFilter(1000, 4, 0.01, seed: 5),
+ (f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
+
+ AssertMutatingQueryAgrees("CountingBloomFilter.TestAndAdd",
+ () => new CountingBloomFilter(1000, 4, 0.01),
+ (f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
+ AssertMutatingQueryAgrees("CountingBloomFilter.TestAndRemove",
+ () => Filled(new CountingBloomFilter(1000, 4, 0.01), (f, k) => f.Add(k)),
+ (f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),
+
+ AssertMutatingQueryAgrees("DeletableBloomFilter.TestAndAdd",
+ () => new DeletableBloomFilter(1000, 100, 0.01),
+ (f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
+ AssertMutatingQueryAgrees("DeletableBloomFilter.TestAndRemove",
+ () => Filled(new DeletableBloomFilter(1000, 100, 0.01), (f, k) => f.Add(k)),
+ (f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),
+
+ // Its TestAndAdd answers with a pair -- whether the key was there, and
+ // whether room was found for it -- so both halves are compared.
+ AssertMutatingQueryAgrees("CuckooBloomFilter.TestAndAdd",
+ () => new CuckooBloomFilter(1000, 0.01, seed: 9),
+ (f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
+ AssertMutatingQueryAgrees("CuckooBloomFilter.TestAndRemove",
+ () => Filled(new CuckooBloomFilter(1000, 0.01, seed: 9), (f, k) => f.Add(k)),
+ (f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),
+
+ AssertMutatingQueryAgrees("QuotientFilter.TestAndRemove",
+ () => Filled(new QuotientFilter(1000, 0.01), (f, k) => f.Add(k)),
+ (f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),
+ AssertMutatingQueryAgrees("InfiniFilter.TestAndRemove",
+ () => Filled(new InfiniFilter(64, 8), (f, k) => f.Add(k)),
+ (f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),
+
+ // These two answer with themselves, so there is no value to compare and
+ // the state is the whole of it.
+ AssertMutationAgrees("SublimeCountMinSketch.Remove",
+ () => Filled(new SublimeCountMinSketch(0.01, 0.5, 1.0), (s, k) => s.Add(k)),
+ (s, k) => s.Remove(k), (s, p) => s.Remove(p)),
+ AssertFixedWidthMutationAgrees("InvertibleBloomLookupTable.Remove", 8,
+ () => new InvertibleBloomLookupTable(64, 8),
+ (t, k) => t.Remove(k), (t, p) => t.Remove(p)),
+ };
+
+ StructureRoster.AssertCoversEveryType(
+ "mutating span queries", StructureRoster.WithMutatingSpanQueries, covered);
+ }
+
+ /// Adds every key to a structure and hands it back.
+ private static T Filled(T structure, Action add)
+ {
for (int i = 0; i < Items; i += 2)
{
- var k = KeyAt(i);
- bloom.Add(k); cms.Add(k); cs.Add(k); qf.Add(k); inverse.Add(k);
+ add(structure, KeyAt(i));
}
+ return structure;
+ }
+
+ ///
+ /// Asks one structure the same question through both overloads and requires the
+ /// same answer. Safe on one instance because the query changes nothing.
+ ///
+ private static Type AssertQueryAgrees(
+ string name, T structure, Func viaArray, SpanQuery viaSpan)
+ {
+ var (buffer, slices) = PackedKeys();
for (int i = 0; i < Items; i++)
{
- var k = KeyAt(i);
- var span = buffer.AsSpan(slices[i].Offset, slices[i].Length);
+ Assert.AreEqual(
+ viaArray(structure, KeyAt(i)),
+ viaSpan(structure, buffer.AsSpan(slices[i].Offset, slices[i].Length)),
+ $"{name} answered differently for key {i} as a span than as an array");
+ }
- Assert.AreEqual(bloom.Test(k), bloom.Test(span), $"BloomFilter.Test at {i}");
- Assert.AreEqual(cms.Count(k), cms.Count(span), $"CountMinSketch.Count at {i}");
- Assert.AreEqual(cs.Count(k), cs.Count(span), $"CountSketch.Count at {i}");
- Assert.AreEqual(qf.Test(k), qf.Test(span), $"QuotientFilter.Test at {i}");
- Assert.AreEqual(inverse.Test(k), inverse.Test(span), $"InverseBloomFilter.Test at {i}");
+ return typeof(T);
+ }
+
+ ///
+ /// Drives two structures the same way, one through each overload, and requires
+ /// both the answers and the states they end in to match.
+ ///
+ private static Type AssertMutatingQueryAgrees(
+ string name, Func create, Func viaArray, SpanQuery viaSpan)
+ where T : IBinaryPersistable
+ {
+ var (buffer, slices) = PackedKeys();
+ var arrayDriven = create();
+ var spanDriven = create();
+
+ for (int i = 0; i < Items; i++)
+ {
+ Assert.AreEqual(
+ viaArray(arrayDriven, KeyAt(i)),
+ viaSpan(spanDriven, buffer.AsSpan(slices[i].Offset, slices[i].Length)),
+ $"{name} answered differently for key {i} as a span than as an array");
}
- var fuse = BinaryFuseFilter.Build(
- Enumerable.Range(0, Items).Where(i => i % 2 == 0).Select(KeyAt).ToArray());
+ CollectionAssert.AreEqual(arrayDriven.ToByteArray(), spanDriven.ToByteArray(),
+ $"{name}: the two paths agreed on every answer and still left the " +
+ "structure holding different things.");
+
+ return typeof(T);
+ }
+
+ ///
+ /// The same, for a mutator that answers with the structure itself. There is no
+ /// value to compare, so the state is the whole of the check.
+ ///
+ private static Type AssertMutationAgrees(
+ string name, Func create, Action viaArray, SpanMutation viaSpan)
+ where T : IBinaryPersistable
+ {
+ var (buffer, slices) = PackedKeys();
+ var arrayDriven = create();
+ var spanDriven = create();
+
for (int i = 0; i < Items; i++)
{
- Assert.AreEqual(fuse.Test(KeyAt(i)),
- fuse.Test(buffer.AsSpan(slices[i].Offset, slices[i].Length)),
- $"BinaryFuseFilter.Test at {i}");
+ viaArray(arrayDriven, KeyAt(i));
+ viaSpan(spanDriven, buffer.AsSpan(slices[i].Offset, slices[i].Length));
}
- var bloomier = BloomierFilter.Build(
- Enumerable.Range(0, Items).ToDictionary(KeyAt, i => (ulong)(i % 200)), 8);
+ CollectionAssert.AreEqual(arrayDriven.ToByteArray(), spanDriven.ToByteArray(),
+ $"{name}: driving the two overloads the same way left different states.");
+
+ return typeof(T);
+ }
+
+ /// The same again, for a structure that will only take a fixed width.
+ private static Type AssertFixedWidthMutationAgrees(
+ string name, int keySize, Func create,
+ Action viaArray, SpanMutation viaSpan)
+ where T : IBinaryPersistable
+ {
+ var buffer = new byte[Items * keySize];
for (int i = 0; i < Items; i++)
{
- var found = bloomier.TryGetValue(KeyAt(i), out var fromArray);
- var foundSpan = bloomier.TryGetValue(
- buffer.AsSpan(slices[i].Offset, slices[i].Length), out var fromSpan);
- Assert.AreEqual(found, foundSpan, $"BloomierFilter.TryGetValue at {i}");
- Assert.AreEqual(fromArray, fromSpan, $"BloomierFilter value at {i}");
+ BitConverter.TryWriteBytes(buffer.AsSpan(i * keySize, keySize), (long)i * 2654435761L);
}
+
+ var arrayDriven = create();
+ var spanDriven = create();
+ for (int i = 0; i < Items; i++)
+ {
+ viaArray(arrayDriven, buffer.AsSpan(i * keySize, keySize).ToArray());
+ viaSpan(spanDriven, buffer.AsSpan(i * keySize, keySize));
+ }
+
+ CollectionAssert.AreEqual(arrayDriven.ToByteArray(), spanDriven.ToByteArray(),
+ $"{name}: driving the two overloads the same way left different states.");
+
+ return typeof(T);
}
///