Add PFS-MS v1.0 reference implementation with compression support

.github/workflows/ci-pfs.yml

@@ -0,0 +1,106 @@
+name: CI (PFS-MS)
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+defaults:
+  run:
+    working-directory: reference/PFS-MS-v1.0
+
+jobs:
+  fmt:
+    name: rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt
+      - run: cargo fmt --all -- --check
+
+  clippy:
+    name: clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: reference/PFS-MS-v1.0
+      - run: cargo clippy --all-targets --all-features -- -D warnings
+
+  test:
+    name: test (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: reference/PFS-MS-v1.0
+      - run: cargo build --verbose
+      - run: cargo test --all-targets --verbose
+      - run: cargo test --doc --verbose
+
+  test-vector:
+    name: regenerate spec test vector
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: reference/PFS-MS-v1.0
+      - name: Build and run the test-vector example
+        run: cargo run --example gen_testvector
+      - name: Inspect generated test vector
+        run: |
+          ls -l pfs_ms_testvector.bin
+          test "$(wc -c < pfs_ms_testvector.bin)" = "2986"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pfs-ms-testvector
+          path: reference/PFS-MS-v1.0/pfs_ms_testvector.bin
+
+  coverage:
+    name: code coverage
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: llvm-tools-preview
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: reference/PFS-MS-v1.0
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Generate coverage report (lcov)
+        run: cargo llvm-cov --all-features --lcov --output-path lcov.info
+      - name: Print coverage summary
+        run: cargo llvm-cov report
+      # The `pfs` CLI binary is exercised manually, not by `cargo test`; the
+      # library floor is enforced over everything except the binary and examples.
+      - name: Enforce minimum library coverage
+        run: |
+          cargo llvm-cov report --summary-only \
+            --ignore-filename-regex 'bin/|examples/' \
+            --fail-under-lines 90 \
+            --fail-under-functions 90
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pfs-ms-coverage-lcov
+          path: reference/PFS-MS-v1.0/lcov.info

Cargo.toml

@@ -1,3 +1,3 @@
 [workspace]
 resolver = "2"
-members = ["reference/PCF-v1.0", "tools/pcf-debug"]
+members = ["reference/PCF-v1.0", "reference/PFS-MS-v1.0", "tools/pcf-debug"]

implementations/dotnet/src/Pcf/BlockView.cs

@@ -0,0 +1,35 @@
+using System.Collections.Generic;
+
+namespace Pcf;
+
+/// <summary>
+/// One table block read from disk: its absolute <see cref="Offset"/>, its parsed
+/// <see cref="TableBlockHeader"/> (including <c>table_hash</c> and
+/// <c>next_table_offset</c>), and its <see cref="PartitionEntry"/> list.
+/// </summary>
+/// <remarks>
+/// This is a read-only view returned by <see cref="Container.ReadBlockAt"/>. It
+/// exists so that profiles layered on PCF (which must group blocks, inspect each
+/// block's <c>table_hash</c>, and follow non-default <c>next_table_offset</c>
+/// chains) can reuse PCF's block parsing rather than re-decoding raw bytes. It
+/// plays no part in the writer's in-memory bookkeeping.
+/// </remarks>
+public sealed class BlockView
+{
+    /// <summary>Absolute file offset of the table block.</summary>
+    public ulong Offset { get; }
+
+    /// <summary>Parsed 74-byte block header.</summary>
+    public TableBlockHeader Header { get; }
+
+    /// <summary>The block's entries, in stored order.</summary>
+    public IReadOnlyList<PartitionEntry> Entries { get; }
+
+    /// <summary>Create a block view.</summary>
+    public BlockView(ulong offset, TableBlockHeader header, IReadOnlyList<PartitionEntry> entries)
+    {
+        Offset = offset;
+        Header = header;
+        Entries = entries;
+    }
+}

implementations/dotnet/src/Pcf/Container.cs

@@ -227,6 +227,20 @@ public List<PartitionEntry> Entries()
         return outp;
     }
 
+    /// <summary>
+    /// Read a single table block at an absolute <paramref name="offset"/>,
+    /// returning its parsed header (including <c>table_hash</c>) and entries.
+    /// Unlike <see cref="Entries"/>, which flattens the whole chain, this exposes
+    /// one block at a time so a caller can follow an arbitrary
+    /// <c>next_table_offset</c> chain and inspect each block's <c>table_hash</c>.
+    /// It is a read-only operation and does not alter the container.
+    /// </summary>
+    public BlockView ReadBlockAt(ulong offset)
+    {
+        (TableBlockHeader h, List<PartitionEntry> entries) = ReadBlock(offset);
+        return new BlockView(offset, h, entries);
+    }
+
     /// <summary>Read a partition's used data.</summary>
     public byte[] ReadPartitionData(PartitionEntry entry)
     {

implementations/dotnet/tests/Pcf.Tests/RoundtripTests.cs

@@ -132,4 +132,35 @@ public void Overflow_chain_roundtrips()
         reopened.Verify();
         Assert.Equal(5, reopened.Entries().Count);
     }
+
+    [Fact]
+    public void ReadBlockAt_exposes_block_view()
+    {
+        // A first-block capacity of 2 forces a second (overflow) block for 3
+        // partitions, so we can walk the chain block-by-block via ReadBlockAt.
+        var c = Container.CreateWith(new MemoryStream(), 2, HashAlgo.Sha256);
+        for (byte i = 1; i <= 3; i++)
+        {
+            c.AddPartition(i, TestSupport.Uid(i), $"p{i}",
+                new byte[] { i, i, i, i }, 0, HashAlgo.Sha256);
+        }
+
+        ulong off = c.Header.PartitionTableOffset;
+        int total = 0, blocks = 0;
+        while (off != 0)
+        {
+            BlockView view = c.ReadBlockAt(off);
+            Assert.Equal(off, view.Offset);
+            Assert.Equal((int)view.Header.PartitionCount, view.Entries.Count);
+            // The exposed table_hash must match a recomputation over the block.
+            byte[] recomputed = TableBlockHeader.ComputeTableHash(
+                view.Header.TableHashAlgo, view.Header.NextTableOffset, view.Entries);
+            Assert.Equal(recomputed, view.Header.TableHash);
+            total += view.Entries.Count;
+            blocks++;
+            off = view.Header.NextTableOffset;
+        }
+        Assert.Equal(3, total);
+        Assert.Equal(2, blocks);
+    }
 }

implementations/php/src/BlockView.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Kduma\PCF;
+
+/**
+ * One table block read from disk: its absolute offset, its parsed
+ * {@see TableBlockHeader} (including table_hash and next_table_offset), and its
+ * {@see PartitionEntry} list.
+ *
+ * This is a read-only view returned by {@see Container::readBlockAt()}. It lets
+ * code layered on PCF group blocks, inspect each block's table_hash, and follow
+ * non-default next_table_offset chains, instead of {@see Container::entries()}
+ * which flattens the whole chain.
+ */
+final class BlockView
+{
+    /**
+     * @param PartitionEntry[] $entries the block's entries, in stored order
+     */
+    public function __construct(
+        public int $offset,
+        public TableBlockHeader $header,
+        public array $entries,
+    ) {
+    }
+}

implementations/php/src/Container.php

@@ -187,6 +187,21 @@ public function entries(): array
         return $out;
     }
 
+    /**
+     * Read a single table block at an absolute offset, returning its parsed
+     * header (including table_hash) and entries.
+     *
+     * Unlike {@see entries()}, which flattens the whole chain, this exposes one
+     * block at a time so a caller can follow an arbitrary next_table_offset
+     * chain and inspect each block's table_hash. Read-only.
+     */
+    public function readBlockAt(int $offset): BlockView
+    {
+        [$h, $entries] = $this->readBlock($offset);
+
+        return new BlockView($offset, $h, $entries);
+    }
+
     /** Read a partition's used data. */
     public function readPartitionData(PartitionEntry $entry): string
     {

implementations/php/tests/RoundtripTest.php

@@ -10,6 +10,7 @@
 use Kduma\PCF\HashAlgo;
 use Kduma\PCF\Storage\MemoryStorage;
 use Kduma\PCF\Storage\StreamStorage;
+use Kduma\PCF\TableBlockHeader;
 
 /**
  * End-to-end container tests, porting `roundtrip.rs` and `coverage.rs`.
@@ -265,4 +266,35 @@ public function testCompactIntoWritesImage(): void
         $c->compactInto($out);
         self::assertSame($c->compactedImage(), $out->getContents());
     }
+
+    public function testReadBlockAtExposesBlockView(): void
+    {
+        // First block capacity of 2 forces a second (overflow) block for 3
+        // partitions, so we can walk the chain block-by-block via readBlockAt.
+        $c = Container::createWith(new MemoryStorage(), 2, HashAlgo::Sha256);
+        for ($i = 1; $i <= 3; ++$i) {
+            $c->addPartition($i, self::uid($i), "p{$i}", str_repeat(\chr($i), 4), 0, HashAlgo::Sha256);
+        }
+
+        $off = $c->header()->partitionTableOffset;
+        $total = 0;
+        $blocks = 0;
+        while ($off !== 0) {
+            $view = $c->readBlockAt($off);
+            self::assertSame($off, $view->offset);
+            self::assertCount($view->header->partitionCount, $view->entries);
+            // The exposed table_hash must match a recomputation over the block.
+            $recomputed = TableBlockHeader::computeTableHash(
+                $view->header->tableHashAlgo,
+                $view->header->nextTableOffset,
+                $view->entries,
+            );
+            self::assertSame($recomputed, $view->header->tableHash);
+            $total += \count($view->entries);
+            ++$blocks;
+            $off = $view->header->nextTableOffset;
+        }
+        self::assertSame(3, $total);
+        self::assertSame(2, $blocks);
+    }
 }

implementations/ts/src/container.ts

@@ -68,6 +68,25 @@ interface BlockInfo {
   next: number;
 }
 
+/**
+ * One table block read from disk: its absolute `offset`, its parsed
+ * {@link TableBlockHeader} (including `tableHash` and `nextTableOffset`), and
+ * its {@link PartitionEntry} list.
+ *
+ * Returned by {@link Container.readBlockAt}. It lets code layered on PCF group
+ * blocks, inspect each block's `tableHash`, and follow non-default
+ * `nextTableOffset` chains, instead of {@link Container.entries} which flattens
+ * the whole chain.
+ */
+export interface BlockView {
+  /** Absolute file offset of the table block. */
+  offset: number;
+  /** Parsed 74-byte block header. */
+  header: TableBlockHeader;
+  /** The block's entries, in stored order. */
+  entries: PartitionEntry[];
+}
+
 function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
   if (a.length !== b.length) {
     return false;
@@ -242,6 +261,18 @@ export class Container {
     return out;
   }
 
+  /**
+   * Read a single table block at an absolute `offset`, returning its parsed
+   * header (including `tableHash`) and entries. Unlike {@link entries}, which
+   * flattens the whole chain, this exposes one block at a time so a caller can
+   * follow an arbitrary `nextTableOffset` chain and inspect each block's
+   * `tableHash`. It is a read-only operation and does not alter the container.
+   */
+  readBlockAt(offset: number): BlockView {
+    const [header, entries] = this.readBlock(offset);
+    return { offset, header, entries };
+  }
+
   /** Read a partition's used data. */
   readPartitionData(entry: PartitionEntry): Uint8Array {
     const used = Number(entry.usedBytes);

implementations/ts/src/index.ts

@@ -65,4 +65,4 @@ export {
 } from "./table.js";
 export { type Storage, MemoryStorage } from "./storage.js";
 export { NodeFileStorage } from "./node-storage.js";
-export { Container } from "./container.js";
+export { Container, type BlockView } from "./container.js";

implementations/ts/test/roundtrip.test.ts

@@ -3,6 +3,7 @@
 import { describe, expect, it } from "vitest";
 
 import {
+  computeTableHash,
   Container,
   entryLabelString,
   freeBytes,
@@ -150,4 +151,34 @@ describe("roundtrip", () => {
     c2.verify();
     expect(c2.entries().length).toBe(1);
   });
+
+  it("readBlockAt exposes a block view", () => {
+    // First block capacity of 2 forces a second (overflow) block for 3
+    // partitions, so we can walk the chain block-by-block via readBlockAt.
+    const c = Container.createWith(new MemoryStorage(), 2, HashAlgo.Sha256);
+    for (let i = 1; i <= 3; i++) {
+      c.addPartition(i, uid(i), `p${i}`, new Uint8Array([i, i, i, i]), 0, HashAlgo.Sha256);
+    }
+
+    let off = Number(c.header().partitionTableOffset);
+    let total = 0;
+    let blocks = 0;
+    while (off !== 0) {
+      const view = c.readBlockAt(off);
+      expect(view.offset).toBe(off);
+      expect(view.entries.length).toBe(view.header.partitionCount);
+      // The exposed tableHash must match a recomputation over the block.
+      const recomputed = computeTableHash(
+        view.header.tableHashAlgo,
+        view.header.nextTableOffset,
+        view.entries,
+      );
+      expect(recomputed).toEqual(view.header.tableHash);
+      total += view.entries.length;
+      blocks++;
+      off = Number(view.header.nextTableOffset);
+    }
+    expect(total).toBe(3);
+    expect(blocks).toBe(2);
+  });
 });