Skip to content

Sweep every span query, mutating ones included #134

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mattlorimor merged 1 commit into from
Aug 20, 2026
Merged

Sweep every span query, mutating ones included #134

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions TestProbabilisticDataStructures/StructureRoster.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,39 @@ internal static class StructureRoster
.OrderBy(t => t.Name, StringComparer.Ordinal)
.ToArray();

/// <summary>
/// Every public structure offering a span query that answers without changing
/// anything: Test, Count, TryGetValue.
/// </summary>
internal static IReadOnlyList<Type> WithPureSpanQueries { get; } =
SpanMethodsWhere(name => name != "Add" && !Mutates(name));

/// <summary>
/// Every public structure offering a span query that answers *and* changes the
/// structure: TestAndAdd, TestAndRemove, Remove.
/// </summary>
/// <remarks>
/// These need a different check from the pure ones. Two overloads returning the
/// same answer is only half of it -- an array path and a span path can agree on
/// every answer and still leave the structure in different states, and a filter
/// that answers correctly while holding the wrong thing fails later, somewhere
/// else, for no visible reason.
/// </remarks>
internal static IReadOnlyList<Type> WithMutatingSpanQueries { get; } =
SpanMethodsWhere(Mutates);

private static bool Mutates(string method) =>
method is "TestAndAdd" or "TestAndRemove" or "Remove";

private static IReadOnlyList<Type> SpanMethodsWhere(Func<string, bool> wanted) =>
Library.GetTypes()
.Where(t => t.IsPublic && !t.IsAbstract && IsAStructure(t))
.Where(t => t.GetMethods(BindingFlags.Public | BindingFlags.Instance)
.Any(m => wanted(m.Name) && m.GetParameters()
.Any(param => param.ParameterType == typeof(ReadOnlySpan<byte>))))
.OrderBy(t => t.Name, StringComparer.Ordinal)
.ToArray();

/// <summary>
/// Every public structure that can be handed a hash as it is built, whether
/// through a constructor or a static factory. This is a wider set than
Expand Down
287 changes: 259 additions & 28 deletions TestProbabilisticDataStructures/TestSpanOverloads.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.VisualStudio.TestTools.UnitTesting;
Expand Down Expand Up @@ -231,55 +232,285 @@ public void TestSpanAndArrayPathsLeaveIdenticalState()
}

/// <summary>
/// Queries must answer identically through either overload, sliced.
/// A span query, as a delegate. A span cannot travel through <see cref="Func{T,
/// TResult}"/>, so the sweeps below need their own delegate types to hold one.
/// </summary>
private delegate object SpanQuery<T>(T structure, ReadOnlySpan<byte> data);

private delegate void SpanMutation<T>(T structure, ReadOnlySpan<byte> data);

/// <summary>
/// Queries that answer without changing anything must answer identically
/// through either overload, sliced.
/// </summary>
[TestMethod]
public void TestSpanAndArrayQueriesAgree()
{
var (buffer, slices) = PackedKeys();
var bloom = Filled(new BloomFilter(1000, 0.01), (f, k) => f.Add(k));
var bloom64 = Filled(new BloomFilter64(1000, 0.01), (f, k) => f.Add(k));
var counting = Filled(new CountingBloomFilter(1000, 4, 0.01), (f, k) => f.Add(k));
var deletable = Filled(new DeletableBloomFilter(1000, 100, 0.01), (f, k) => f.Add(k));
var partitioned = Filled(new PartitionedBloomFilter(1000, 0.01), (f, k) => f.Add(k));
var scalable = Filled(new ScalableBloomFilter(100, 0.01, 0.8), (f, k) => f.Add(k));
var stable = Filled(new StableBloomFilter(1000, 4, 0.01, seed: 5), (f, k) => f.Add(k));
var inverse = Filled(new InverseBloomFilter(500), (f, k) => f.Add(k));
var cuckoo = Filled(new CuckooBloomFilter(1000, 0.01, seed: 9), (f, k) => f.Add(k));
var quotient = Filled(new QuotientFilter(1000, 0.01), (f, k) => f.Add(k));
var infini = Filled(new InfiniFilter(64, 8), (f, k) => f.Add(k));
var cms = Filled(new CountMinSketch(0.001, 0.01), (s, k) => s.Add(k));
var countSketch = Filled(new CountSketch(0.01, 0.01), (s, k) => s.Add(k, 3));
var keeper = Filled(new HeavyKeeper(20, 512, seed: 3), (s, k) => s.Add(k));
var sublime = Filled(new SublimeCountMinSketch(0.01, 0.5, 1.0), (s, k) => s.Add(k));
var priv = Filled(new PrivateCountMinSketch(64, 4, 1.0, seed: 3), (s, k) => s.Add(k));
var dpsw = Filled(
new DpswSketch(window: 128, rho: 4.0, alpha: 0.6, width: 8, depth: 2, seed: 3),
(s, k) => s.Add(k));

var half = Enumerable.Range(0, Items).Where(i => i % 2 == 0).ToArray();
var fuse = BinaryFuseFilter.Build(half.Select(KeyAt));
var bloomier = BloomierFilter.Build(
// Eight value bits, so the values have to stay under 256.
half.Select(i => new KeyValuePair<byte[], ulong>(KeyAt(i), (ulong)(i % 200))), 8);

var bloom = new BloomFilter(1000, 0.01);
var cms = new CountMinSketch(0.001, 0.01);
var cs = new CountSketch(0.01, 0.01);
var qf = new QuotientFilter(1000, 0.01);
var inverse = new InverseBloomFilter(500);
var covered = new[]
{
AssertQueryAgrees("BloomFilter.Test", bloom,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("BloomFilter64.Test", bloom64,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("CountingBloomFilter.Test", counting,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("DeletableBloomFilter.Test", deletable,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("PartitionedBloomFilter.Test", partitioned,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("ScalableBloomFilter.Test", scalable,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("StableBloomFilter.Test", stable,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("InverseBloomFilter.Test", inverse,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("CuckooBloomFilter.Test", cuckoo,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("QuotientFilter.Test", quotient,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("InfiniFilter.Test", infini,
(f, k) => f.Test(k), (f, s) => f.Test(s)),
AssertQueryAgrees("BinaryFuseFilter.Test", fuse,
(f, k) => f.Test(k), (f, s) => f.Test(s)),

AssertQueryAgrees("CountMinSketch.Count", cms,
(s, k) => s.Count(k), (s, p) => s.Count(p)),
AssertQueryAgrees("CountSketch.Count", countSketch,
(s, k) => s.Count(k), (s, p) => s.Count(p)),
AssertQueryAgrees("HeavyKeeper.Count", keeper,
(s, k) => s.Count(k), (s, p) => s.Count(p)),
AssertQueryAgrees("SublimeCountMinSketch.Count", sublime,
(s, k) => s.Count(k), (s, p) => s.Count(p)),

// The noisy pair need no special treatment, which is worth saying
// because it looks as though they should. Their noise is drawn once at
// construction and lives in the counters -- that is what stops repeated
// queries from averaging it away -- so a query is an ordinary read and
// two calls must return the identical double, not merely a close one.
AssertQueryAgrees("PrivateCountMinSketch.Count", priv,
(s, k) => s.Count(k), (s, p) => s.Count(p)),
AssertQueryAgrees("DpswSketch.Count", dpsw,
(s, k) => s.Count(k), (s, p) => s.Count(p)),

AssertQueryAgrees("BloomierFilter.TryGetValue", bloomier,
(f, k) => { var ok = f.TryGetValue(k, out var v); return (ok, v); },
(f, s) => { var ok = f.TryGetValue(s, out var v); return (ok, v); }),
};

StructureRoster.AssertCoversEveryType(
"pure span queries", StructureRoster.WithPureSpanQueries, covered);
}

/// <summary>
/// Queries that answer *and* change the structure must do both identically.
/// </summary>
/// <remarks>
/// Agreeing on every answer is only half of this. Two paths can return the same
/// value at every step and leave the structure holding different things, and a
/// filter that answers correctly while holding the wrong thing does not fail
/// here -- it fails later, somewhere else, for no visible reason. So each pair
/// is driven step for step and then compared through its payload, the same
/// oracle the equivalence sweep uses.
/// </remarks>
[TestMethod]
public void TestSpanAndArrayMutatingQueriesAgree()
{
var covered = new[]
{
AssertMutatingQueryAgrees("BloomFilter.TestAndAdd",
() => new BloomFilter(1000, 0.01),
(f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
AssertMutatingQueryAgrees("BloomFilter64.TestAndAdd",
() => new BloomFilter64(1000, 0.01),
(f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
AssertMutatingQueryAgrees("PartitionedBloomFilter.TestAndAdd",
() => new PartitionedBloomFilter(1000, 0.01),
(f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
AssertMutatingQueryAgrees("ScalableBloomFilter.TestAndAdd",
() => new ScalableBloomFilter(100, 0.01, 0.8),
(f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
AssertMutatingQueryAgrees("InverseBloomFilter.TestAndAdd",
() => new InverseBloomFilter(500),
(f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
AssertMutatingQueryAgrees("StableBloomFilter.TestAndAdd",
() => new StableBloomFilter(1000, 4, 0.01, seed: 5),
(f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),

AssertMutatingQueryAgrees("CountingBloomFilter.TestAndAdd",
() => new CountingBloomFilter(1000, 4, 0.01),
(f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
AssertMutatingQueryAgrees("CountingBloomFilter.TestAndRemove",
() => Filled(new CountingBloomFilter(1000, 4, 0.01), (f, k) => f.Add(k)),
(f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),

AssertMutatingQueryAgrees("DeletableBloomFilter.TestAndAdd",
() => new DeletableBloomFilter(1000, 100, 0.01),
(f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
AssertMutatingQueryAgrees("DeletableBloomFilter.TestAndRemove",
() => Filled(new DeletableBloomFilter(1000, 100, 0.01), (f, k) => f.Add(k)),
(f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),

// Its TestAndAdd answers with a pair -- whether the key was there, and
// whether room was found for it -- so both halves are compared.
AssertMutatingQueryAgrees("CuckooBloomFilter.TestAndAdd",
() => new CuckooBloomFilter(1000, 0.01, seed: 9),
(f, k) => f.TestAndAdd(k), (f, s) => f.TestAndAdd(s)),
AssertMutatingQueryAgrees("CuckooBloomFilter.TestAndRemove",
() => Filled(new CuckooBloomFilter(1000, 0.01, seed: 9), (f, k) => f.Add(k)),
(f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),

AssertMutatingQueryAgrees("QuotientFilter.TestAndRemove",
() => Filled(new QuotientFilter(1000, 0.01), (f, k) => f.Add(k)),
(f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),
AssertMutatingQueryAgrees("InfiniFilter.TestAndRemove",
() => Filled(new InfiniFilter(64, 8), (f, k) => f.Add(k)),
(f, k) => f.TestAndRemove(k), (f, s) => f.TestAndRemove(s)),

// These two answer with themselves, so there is no value to compare and
// the state is the whole of it.
AssertMutationAgrees("SublimeCountMinSketch.Remove",
() => Filled(new SublimeCountMinSketch(0.01, 0.5, 1.0), (s, k) => s.Add(k)),
(s, k) => s.Remove(k), (s, p) => s.Remove(p)),
AssertFixedWidthMutationAgrees("InvertibleBloomLookupTable.Remove", 8,
() => new InvertibleBloomLookupTable(64, 8),
(t, k) => t.Remove(k), (t, p) => t.Remove(p)),
};

StructureRoster.AssertCoversEveryType(
"mutating span queries", StructureRoster.WithMutatingSpanQueries, covered);
}

/// <summary>Adds every key to a structure and hands it back.</summary>
private static T Filled<T>(T structure, Action<T, byte[]> add)
{
for (int i = 0; i < Items; i += 2)
{
var k = KeyAt(i);
bloom.Add(k); cms.Add(k); cs.Add(k); qf.Add(k); inverse.Add(k);
add(structure, KeyAt(i));
}
return structure;
}

/// <summary>
/// Asks one structure the same question through both overloads and requires the
/// same answer. Safe on one instance because the query changes nothing.
/// </summary>
private static Type AssertQueryAgrees<T>(
string name, T structure, Func<T, byte[], object> viaArray, SpanQuery<T> viaSpan)
{
var (buffer, slices) = PackedKeys();

for (int i = 0; i < Items; i++)
{
var k = KeyAt(i);
var span = buffer.AsSpan(slices[i].Offset, slices[i].Length);
Assert.AreEqual(
viaArray(structure, KeyAt(i)),
viaSpan(structure, buffer.AsSpan(slices[i].Offset, slices[i].Length)),
$"{name} answered differently for key {i} as a span than as an array");
}

Assert.AreEqual(bloom.Test(k), bloom.Test(span), $"BloomFilter.Test at {i}");
Assert.AreEqual(cms.Count(k), cms.Count(span), $"CountMinSketch.Count at {i}");
Assert.AreEqual(cs.Count(k), cs.Count(span), $"CountSketch.Count at {i}");
Assert.AreEqual(qf.Test(k), qf.Test(span), $"QuotientFilter.Test at {i}");
Assert.AreEqual(inverse.Test(k), inverse.Test(span), $"InverseBloomFilter.Test at {i}");
return typeof(T);
}

/// <summary>
/// Drives two structures the same way, one through each overload, and requires
/// both the answers and the states they end in to match.
/// </summary>
private static Type AssertMutatingQueryAgrees<T>(
string name, Func<T> create, Func<T, byte[], object> viaArray, SpanQuery<T> viaSpan)
where T : IBinaryPersistable<T>
{
var (buffer, slices) = PackedKeys();
var arrayDriven = create();
var spanDriven = create();

for (int i = 0; i < Items; i++)
{
Assert.AreEqual(
viaArray(arrayDriven, KeyAt(i)),
viaSpan(spanDriven, buffer.AsSpan(slices[i].Offset, slices[i].Length)),
$"{name} answered differently for key {i} as a span than as an array");
}

var fuse = BinaryFuseFilter.Build(
Enumerable.Range(0, Items).Where(i => i % 2 == 0).Select(KeyAt).ToArray());
CollectionAssert.AreEqual(arrayDriven.ToByteArray(), spanDriven.ToByteArray(),
$"{name}: the two paths agreed on every answer and still left the " +
"structure holding different things.");

return typeof(T);
}

/// <summary>
/// The same, for a mutator that answers with the structure itself. There is no
/// value to compare, so the state is the whole of the check.
/// </summary>
private static Type AssertMutationAgrees<T>(
string name, Func<T> create, Action<T, byte[]> viaArray, SpanMutation<T> viaSpan)
where T : IBinaryPersistable<T>
{
var (buffer, slices) = PackedKeys();
var arrayDriven = create();
var spanDriven = create();

for (int i = 0; i < Items; i++)
{
Assert.AreEqual(fuse.Test(KeyAt(i)),
fuse.Test(buffer.AsSpan(slices[i].Offset, slices[i].Length)),
$"BinaryFuseFilter.Test at {i}");
viaArray(arrayDriven, KeyAt(i));
viaSpan(spanDriven, buffer.AsSpan(slices[i].Offset, slices[i].Length));
}

var bloomier = BloomierFilter.Build(
Enumerable.Range(0, Items).ToDictionary(KeyAt, i => (ulong)(i % 200)), 8);
CollectionAssert.AreEqual(arrayDriven.ToByteArray(), spanDriven.ToByteArray(),
$"{name}: driving the two overloads the same way left different states.");

return typeof(T);
}

/// <summary>The same again, for a structure that will only take a fixed width.</summary>
private static Type AssertFixedWidthMutationAgrees<T>(
string name, int keySize, Func<T> create,
Action<T, byte[]> viaArray, SpanMutation<T> viaSpan)
where T : IBinaryPersistable<T>
{
var buffer = new byte[Items * keySize];
for (int i = 0; i < Items; i++)
{
var found = bloomier.TryGetValue(KeyAt(i), out var fromArray);
var foundSpan = bloomier.TryGetValue(
buffer.AsSpan(slices[i].Offset, slices[i].Length), out var fromSpan);
Assert.AreEqual(found, foundSpan, $"BloomierFilter.TryGetValue at {i}");
Assert.AreEqual(fromArray, fromSpan, $"BloomierFilter value at {i}");
BitConverter.TryWriteBytes(buffer.AsSpan(i * keySize, keySize), (long)i * 2654435761L);
}

var arrayDriven = create();
var spanDriven = create();
for (int i = 0; i < Items; i++)
{
viaArray(arrayDriven, buffer.AsSpan(i * keySize, keySize).ToArray());
viaSpan(spanDriven, buffer.AsSpan(i * keySize, keySize));
}

CollectionAssert.AreEqual(arrayDriven.ToByteArray(), spanDriven.ToByteArray(),
$"{name}: driving the two overloads the same way left different states.");

return typeof(T);
}

/// <summary>
Expand Down
Loading