Enhancing ctable with a new utf8() string type

CMakeLists.txt

@@ -59,6 +59,13 @@ add_custom_command(
   DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/groupby_ext.pyx"
   VERBATIM)
 
+add_custom_command(
+  OUTPUT utf8_ext.c
+  COMMAND Python::Interpreter -m cython
+          "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/utf8_ext.pyx" --output-file utf8_ext.c
+  DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/utf8_ext.pyx"
+  VERBATIM)
+
 # ...and add it to the target
 Python_add_library(blosc2_ext MODULE blosc2_ext.c WITH_SOABI)
 target_sources(blosc2_ext PRIVATE src/blosc2/matmul_kernels.c)
@@ -68,11 +75,17 @@ if(UNIX)
 endif()
 Python_add_library(indexing_ext MODULE indexing_ext.c WITH_SOABI)
 Python_add_library(groupby_ext MODULE groupby_ext.c WITH_SOABI)
+Python_add_library(utf8_ext MODULE utf8_ext.c WITH_SOABI)
+# NpyString_pack() and friends are part of NumPy's 2.0 C API; opt in
+# explicitly since numpy/*.h otherwise targets an older API version by
+# default for source compatibility.
+target_compile_definitions(utf8_ext PRIVATE NPY_TARGET_VERSION=NPY_2_0_API_VERSION)
 
 # We need to link against NumPy
 target_link_libraries(blosc2_ext PRIVATE Python::NumPy)
 target_link_libraries(indexing_ext PRIVATE Python::NumPy)
 target_link_libraries(groupby_ext PRIVATE Python::NumPy)
+target_link_libraries(utf8_ext PRIVATE Python::NumPy)
 
 # Fetch and build miniexpr library
 include(FetchContent)
@@ -110,6 +123,7 @@ endif()
 target_compile_features(blosc2_ext PRIVATE c_std_11)
 target_compile_features(indexing_ext PRIVATE c_std_11)
 target_compile_features(groupby_ext PRIVATE c_std_11)
+target_compile_features(utf8_ext PRIVATE c_std_11)
 if(WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Clang")
     execute_process(
         COMMAND "${CMAKE_C_COMPILER}" -print-resource-dir
@@ -184,7 +198,7 @@ endif()
 
 # Python extension -> site-packages/blosc2
 install(
-  TARGETS blosc2_ext indexing_ext groupby_ext
+  TARGETS blosc2_ext indexing_ext groupby_ext utf8_ext
   LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}/blosc2
 )
 

bench/ctable/bench_groupby_keys.py

@@ -32,13 +32,20 @@ class Row:
     ikey: int = blosc2.field(blosc2.int64())
     skey: str = blosc2.field(blosc2.string(max_length=8))
     dkey: str = blosc2.field(blosc2.dictionary())
+    ukey: str = blosc2.field(blosc2.utf8())
     val: float = blosc2.field(blosc2.float64())
 
 
 print(f"building table ({N:.0e} rows)...", flush=True)
 t = CTable(Row)
 t.extend(
-    {"ikey": int_keys, "skey": str_keys, "dkey": [str(s) for s in str_keys], "val": float_vals},
+    {
+        "ikey": int_keys,
+        "skey": str_keys,
+        "dkey": [str(s) for s in str_keys],
+        "ukey": [str(s) for s in str_keys],
+        "val": float_vals,
+    },
     validate=False,
 )
 
@@ -56,7 +63,9 @@ def bench(label, fn, reps=3):
 bench("int key, mean", lambda: t.group_by("ikey").agg({"val": "mean"}))
 bench("string key, sum", lambda: t.group_by("skey").sum("val"))
 bench("dict key, sum", lambda: t.group_by("dkey").sum("val"))
+bench("utf8 key, sum", lambda: t.group_by("ukey").sum("val"))
 bench("two keys (int+dict), sum", lambda: t.group_by(["ikey", "dkey"]).sum("val"))
+bench("two keys (int+utf8), sum", lambda: t.group_by(["ikey", "ukey"]).sum("val"))
 
 try:
     import pandas as pd

bench/ctable/bench_string_kinds.py

@@ -0,0 +1,151 @@
+#######################################################################
+# Copyright (c) 2019-present, Blosc Development Team <blosc@blosc.org>
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#######################################################################
+
+"""Compare the three plain-string column representations head to head:
+``utf8()`` (offsets + bytes, StringDType reads), ``string(max_length=L)``
+(fixed-width UTF-32), and ``vlstring()`` (msgpack cells).
+
+Two workloads:
+- "taxi company": the real ``company`` column from the Chicago taxi dataset
+  (medium-length, low-cardinality strings), when the parquet file is present.
+- "synthetic free text": high-cardinality random words of 0-60 chars with
+  some multi-byte values — the workload utf8() is designed for.
+
+For each representation: ingest, storage footprint, full read, equality
+filter, groupby-key aggregation, sort, and Arrow export.  Operations a
+representation does not support are reported as such rather than skipped
+silently.
+"""
+
+import argparse
+import pathlib
+import time
+from dataclasses import make_dataclass
+
+import numpy as np
+
+import blosc2
+from blosc2 import CTable
+
+N_TAXI = 10_000_000
+N_SYNTH = 2_000_000  # fixed-width U~130 at 1e7 rows would need a ~5 GB ingest buffer
+REPS = 3
+TAXI_PARQUET = pathlib.Path(__file__).parent.parent / "chicago-taxi" / "chicago-taxi-flat.parquet"
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument(
+    "--ingest-only",
+    action="store_true",
+    help="Only measure ingest and storage footprint; skip full read, filter, "
+    "groupby, sort, and to_arrow (the last of which alone can take over a "
+    "minute per column kind on the full taxi workload). Meant for fast "
+    "iteration when only ingest performance is under investigation.",
+)
+args = parser.parse_args()
+
+rng = np.random.default_rng(42)
+
+
+def bench(label, fn, reps=REPS):
+    times = []
+    result = None
+    for _ in range(reps):
+        t0 = time.perf_counter()
+        result = fn()
+        times.append(time.perf_counter() - t0)
+    print(f"  {label:34s} {min(times) * 1e3:9.1f} ms")
+    return result
+
+
+def load_taxi_company(n):
+    import pyarrow.parquet as pq
+
+    tbl = pq.read_table(TAXI_PARQUET, columns=["company"])
+    col = tbl.column("company").combine_chunks()
+    values = col.to_pylist()[:n]
+    return [v if v is not None else "" for v in values]
+
+
+def synth_free_text(n):
+    words = np.array(
+        ["taxi", "río", "航空", "boulevard", "x" * 40, "café", "", "downtown", "zürich", "o'hare"]
+    )
+    # 2-6 words per row, high cardinality via a row counter suffix on ~half.
+    parts = words[rng.integers(0, len(words), (n, 3))]
+    joined = [" ".join(p) for p in parts]
+    salt = rng.integers(0, 100_000, n)  # ~100k distinct values: high cardinality, sane group count
+    return [f"{s} #{salt[i]}" if i % 2 else s for i, s in enumerate(joined)]
+
+
+def run_workload(title, values, filter_value):
+    n = len(values)
+    max_len = max(len(v) for v in values)
+    float_vals = rng.random(n)
+    print(f"\n=== {title} ({n:.0e} rows, max length {max_len} chars) ===")
+
+    specs = {
+        "utf8": blosc2.utf8(),
+        "string": blosc2.string(max_length=max_len),
+        "vlstring": blosc2.vlstring(),
+    }
+    for kind, spec in specs.items():
+        row_cls = make_dataclass(
+            "Row", [("s", str, blosc2.field(spec)), ("val", float, blosc2.field(blosc2.float64()))]
+        )
+        print(f"[{kind}]")
+        t0 = time.perf_counter()
+        t = CTable(row_cls)
+        t.extend({"s": values, "val": float_vals}, validate=False)
+        t._flush_varlen_columns()
+        print(f"  {'ingest':34s} {(time.perf_counter() - t0) * 1e3:9.1f} ms")
+
+        col = t._cols["s"]
+        nbytes = getattr(col, "nbytes", None)
+        cbytes = getattr(col, "cbytes", None)
+        if nbytes is None:  # NDArray-backed fixed-width column
+            nbytes, cbytes = col.schunk.nbytes, col.schunk.cbytes
+        print(
+            f"  {'storage nbytes -> cbytes':34s} {nbytes / 2**20:7.1f} MB -> {cbytes / 2**20:7.1f} MB (cratio {nbytes / cbytes:.1f}x)"
+        )
+
+        if args.ingest_only:
+            del t
+            continue
+
+        bench("full column read", lambda t=t: t["s"][:])
+        try:
+            bench("filter: count(s == value)", lambda t=t: int((t.s == filter_value)[:].sum()))
+        except (NotImplementedError, TypeError) as exc:
+            print(f"  {'filter: count(s == value)':34s}   unsupported: {str(exc)[:60]}")
+        try:
+            bench("groupby key: sum(val)", lambda t=t: t.group_by("s").sum("val"), reps=1)
+        except (NotImplementedError, TypeError) as exc:
+            print(f"  {'groupby key: sum(val)':34s}   unsupported: {str(exc)[:60]}")
+        try:
+            bench("sort_by(s) (copy)", lambda t=t: t.sort_by("s"), reps=1)
+        except (NotImplementedError, TypeError) as exc:
+            print(f"  {'sort_by(s) (copy)':34s}   unsupported: {str(exc)[:60]}")
+        try:
+            bench("to_arrow()", lambda t=t: t.to_arrow(), reps=1)
+        except Exception as exc:
+            print(f"  {'to_arrow()':34s}   failed: {str(exc)[:60]}")
+        del t
+
+
+if TAXI_PARQUET.exists():
+    print("loading taxi company column...", flush=True)
+    taxi = load_taxi_company(N_TAXI)
+    # the most frequent company value as the filter probe
+    vals, counts = np.unique(np.array(taxi, dtype=np.dtypes.StringDType()), return_counts=True)
+    run_workload("chicago-taxi company", taxi, str(vals[np.argmax(counts)]))
+    del taxi
+else:
+    print(f"({TAXI_PARQUET} not found; skipping the real-data workload)")
+
+print("building synthetic free text...", flush=True)
+synth = synth_free_text(N_SYNTH)
+run_workload("synthetic free text", synth, synth[123])

doc/getting_started/overview.rst

@@ -270,7 +270,7 @@ types including integers, floats, booleans, and strings:
         tips: float = blosc2.field(blosc2.float32())
         km: float = blosc2.field(blosc2.float32())
         lon: float = blosc2.field(blosc2.float32())
-        company: str = blosc2.field(blosc2.string(max_length=50))
+        company: str = blosc2.field(blosc2.utf8())  # variable-length text
 
 
     t = blosc2.CTable(Row, expected_size=10_000_000)

doc/guides/parquet_to_blosc2.rst

@@ -57,10 +57,11 @@ Use the ``.b2d`` extension to produce a directory-backed (sparse) store:
 Step 4 — Fixed-width string import
 ------------------------------------
 
-By default, string columns are stored as variable-length strings
-(``vlstring``).  Pass ``--fixed-str-maxlen`` to pre-scan strings and store
-columns whose maximum character length fits within the given limit as
-fixed-width, indexable strings:
+By default, string columns are stored as variable-length ``utf8`` columns
+(Arrow-style offsets + bytes; falls back to ``vlstring`` on NumPy < 2.0).
+Pass ``--fixed-str-maxlen`` to pre-scan strings and store columns whose
+maximum character length fits within the given limit as fixed-width,
+indexable strings:
 
 .. code-block:: console
 

doc/reference/ctable.rst

@@ -938,15 +938,109 @@ Text & binary
 .. autosummary::
 
     string
+    utf8
     bytes
     vlstring
     vlbytes
 
 .. autoclass:: string
+.. autofunction:: utf8
 .. autoclass:: bytes
 .. autofunction:: vlstring
 .. autofunction:: vlbytes
 
+.. _ChoosingStringType:
+
+Choosing a string column type
+-----------------------------
+
+CTable offers four ways to store strings.  As a quick decision path:
+
+* Low-cardinality strings (categories, enumerations, repeated labels):
+  use :func:`dictionary` — repeated values are stored once as integer codes.
+* Everything else (names, free text, high-cardinality values):
+  use :func:`utf8` — the recommended default for variable-length text.
+* Short codes of near-uniform length, or columns that need
+  :meth:`CTable.create_index`: use :class:`string` (fixed-width).
+* NumPy < 2.0, or nullable columns where any string value can legally occur
+  (native ``None`` nulls, no sentinel): use :func:`vlstring`.
+
+.. list-table::
+   :header-rows: 1
+   :stub-columns: 1
+
+   * -
+     - :class:`string`
+     - :func:`utf8`
+     - :func:`dictionary`
+     - :func:`vlstring`
+   * - Storage layout
+     - fixed-width UTF-32 NDArray
+     - int64 offsets + UTF-8 bytes (Arrow-style)
+     - integer codes + unique values
+     - msgpack batches
+   * - Per-row cost (pre-compression)
+     - 4 × ``max_length`` bytes
+     - exact UTF-8 length + 8-byte offset
+     - one integer code
+     - value + msgpack framing
+   * - Length limit
+     - ``max_length`` characters
+     - none
+     - none
+     - none
+   * - Bulk read returns
+     - NumPy ``U`` array
+     - NumPy ``StringDType`` array
+     - decoded strings
+     - Python list of ``str``
+   * - Nulls
+     - sentinel
+     - sentinel
+     - native
+     - native ``None``
+   * - Filters (``==``, ``<``, …)
+     - ✓ (incl. string expressions)
+     - ✓ operators only [#utf8expr]_
+     - ``==`` / ``isin()`` only
+     - ✗
+   * - :meth:`CTable.group_by` key / :meth:`CTable.sort_by`
+     - ✓
+     - ✓
+     - ✓
+     - ✗
+   * - :meth:`CTable.create_index`
+     - ✓
+     - not yet
+     - ✓ (rank-based)
+     - ✗
+   * - Arrow / Parquet
+     - ✓
+     - ✓ (``large_string``)
+     - ✓
+     - ✓
+   * - NumPy requirement
+     - any
+     - >= 2.0
+     - any
+     - any
+   * - Best for
+     - short, near-uniform codes
+     - **general text (recommended)**
+     - low-cardinality categories
+     - NumPy < 2.0; native-``None`` nulls
+
+.. [#utf8expr] utf8 columns support the operator form ``t[t.name == "x"]``
+   (also ``!=``, ``<``, ``<=``, ``>``, ``>=``), but not the string-expression
+   form ``t.where("name == 'x'")`` yet.  :meth:`CTable.create_index` on utf8
+   columns is not supported yet either; both raise ``NotImplementedError``
+   with a clear message.
+
+Note that a plain ``str`` annotation without an explicit :func:`field` spec
+still maps to fixed-width ``string(max_length=32)`` for backward
+compatibility; opt in to variable-length storage with
+``blosc2.field(blosc2.utf8())``.
+
 Array, encoded, and compound specs
 ----------------------------------
 

doc/reference/misc.rst

@@ -74,6 +74,7 @@ public objects into the appropriate reference section.
         uint16,
         uint32,
         uint64,
+        utf8,
         vlbytes,
         vlstring,
         Array,

examples/ctable/arrow_interop.py

@@ -59,7 +59,9 @@ class Stock:
 t2 = blosc2.CTable.from_arrow(at2.schema, at2.to_batches())
 print("CTable from Arrow (inferred schema):")
 print(t2)
-print(f"  label dtype: {t2['label'].dtype}  (max_length inferred from data)")
+# Arrow string columns import as variable-length utf8() columns (StringDType
+# reads); pass string_max_length= to from_arrow() for fixed-width instead.
+print(f"  label dtype: {t2['label'].dtype}")
 
 # -- pandas round-trip ------------------------------------------------------
 try: