diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml new file mode 100644 index 000000000..07d04b5a8 --- /dev/null +++ b/.github/workflows/docs-deploy.yml @@ -0,0 +1,77 @@ +name: Deploy documentation + +# Publishes versioned docs to the `gh-pages` branch using the Zensical-compatible +# fork of `mike` (https://github.com/squidfunk/mike): each version is committed +# as a subdirectory of `site_url` (see zensical.toml), with a `latest` alias and +# a root redirect pointing at the newest release. +# +# Runs automatically on release tags (semantic-release tags the bare version, +# e.g. 0.95.1) and can be triggered manually. Release tags publish the minor +# series (e.g. 0.95) so that patch releases update the same series page. + +on: + push: + tags: ["[0-9]+.[0-9]+.[0-9]+"] + workflow_dispatch: + inputs: + version: + description: "Version label to publish (e.g. 0.95 or dev)" + required: true + default: "dev" + alias: + description: "Alias to update" + required: false + default: "latest" + set_default: + description: "Point the site root at this alias" + type: boolean + default: true + +permissions: + contents: write # mike pushes the built docs to the gh-pages branch + +concurrency: + group: docs-deploy + cancel-in-progress: false + +jobs: + deploy: + name: Deploy docs with mike + runs-on: ubuntu-latest + # Auto-deploy only on the canonical repo; manual runs are allowed anywhere + # (e.g. to bootstrap gh-pages on a fork while testing). + if: ${{ github.event_name == 'workflow_dispatch' || github.repository == 'substrait-io/substrait-java' }} + steps: + - name: Checkout code + uses: actions/checkout@v7 + with: + fetch-depth: 0 # mike reads/writes the full gh-pages history + - name: Set up pixi + uses: prefix-dev/setup-pixi@v0.10.0 + with: + environments: docs + - name: Configure git identity + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + - name: Resolve version and alias + id: v + run: | + if [ "${{ github.event_name }}" = "push" ]; then + full="${GITHUB_REF_NAME}" # 0.95.1 + echo "version=${full%.*}" >> "$GITHUB_OUTPUT" # 0.95 (minor series) + echo "alias=latest" >> "$GITHUB_OUTPUT" + echo "set_default=true" >> "$GITHUB_OUTPUT" + else + echo "version=${{ inputs.version }}" >> "$GITHUB_OUTPUT" + echo "alias=${{ inputs.alias }}" >> "$GITHUB_OUTPUT" + echo "set_default=${{ inputs.set_default }}" >> "$GITHUB_OUTPUT" + fi + - name: Deploy version + run: | + pixi run -e docs mike deploy --push --update-aliases \ + "${{ steps.v.outputs.version }}" "${{ steps.v.outputs.alias }}" + - name: Set default version + if: ${{ steps.v.outputs.set_default == 'true' }} + run: | + pixi run -e docs mike set-default --push "${{ steps.v.outputs.alias }}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..39bff9c23 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,28 @@ +name: Build documentation + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +# Build-check only: this verifies the docs compile on every PR so broken pages +# or a broken link are caught early. It does NOT publish. Versioned publishing +# to GitHub Pages is handled separately by .github/workflows/docs-deploy.yml +# (mike -> gh-pages branch). + +jobs: + build: + name: Build docs + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v7 + - name: Set up pixi + uses: prefix-dev/setup-pixi@v0.10.0 + with: + environments: docs + - name: Build the documentation site + run: pixi run docs-build diff --git a/.gitignore b/.gitignore index ad203c00a..31978ca04 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,7 @@ bin .classpath .settings bin/ + +# Documentation site (Zensical + pixi) +/site +.pixi diff --git a/AGENTS.md b/AGENTS.md index 264124de2..e7995df55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -192,6 +192,28 @@ compile — they have their own visitor implementors: fidelity. See `core/src/test/java/io/substrait/type/proto/DynamicParameterRoundtripTest.java` for the canonical pattern. +## Documentation + +User-facing documentation lives in `docs/` (Markdown, one file per page) and is built with +**Zensical** (config in `zensical.toml`). Python and the doc tooling are managed with **pixi**, +independent of the Gradle build — the only prerequisite is a `pixi` install. + +- Preview locally with `pixi run docs-serve` (live reload); build the static site with + `pixi run docs-build` (output to `site/`, gitignored). `docs-build` validates internal links, + so run it before pushing doc changes. +- Pages are grouped under `docs/{core,isthmus,isthmus-cli,spark}/` plus a landing page and + getting-started. Navigation is **explicit** in `zensical.toml` (`nav`) — when you add a page, + add it to `nav`. +- **Keep the guide in sync with the code as substrait-java evolves.** When you add or change + user-facing behavior — a new expression/relation type, a `SubstraitBuilder`/`ExpressionCreator` + factory, a newly supported function, an isthmus/spark capability, or a CLI flag — update the + matching `docs/` page in the same PR. Ground snippets in real APIs/tests, and (as in source) + keep GitHub issue/PR numbers out of the docs. +- CI: `.github/workflows/docs.yml` build-checks docs on every PR and push to `main`; + `.github/workflows/docs-deploy.yml` publishes versioned docs to the `gh-pages` branch via the + Zensical-compatible `mike` fork on release tags (the bare `X.Y.Z` tag publishes the minor + series `X.Y` and updates the `latest` alias). Site: . + ## Conventions & workflow - **Conventional commits** are required (CI lints them, and PR title + body must form a @@ -204,7 +226,8 @@ compile — they have their own visitor implementors: - Many features track upstream Substrait spec releases (see epic-style issues); a new proto message usually needs: POJO + visitor wiring + both proto converters + a round-trip test, and often `ExpressionCreator` factories and `dsl/SubstraitBuilder` - helpers for ergonomics. + helpers for ergonomics — plus a matching update to the user guide under `docs/` + (see [Documentation](#documentation)). - When monitoring PR checks, budget for a long tail: the **macOS `Build Isthmus Native Image`** job is the long pole — it `needs:` the `java` + `integration` jobs (so it starts late), then AOT-compiles for ~15–20 min on a slower macOS runner. A PR staying diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d4f61950..e2d37daff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,7 @@ This page provides some orientation and recommendations on how to get the best r 1. [Commit conventions](#commit-conventions) 2. [Style Guide](#style-guide) +3. [Documentation](#documentation) ## Commit Conventions @@ -48,3 +49,29 @@ org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAME --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED ``` + +## Documentation + +The user-facing documentation site lives under [`docs/`](docs) and is built with +[Zensical](https://zensical.org). Python and the documentation dependencies are managed with +[pixi](https://pixi.sh), so the only prerequisite is a pixi installation. + +Preview your changes locally with a live-reloading dev server, or produce the static site: + +```bash +pixi run docs-serve # live-reloading preview at http://localhost:8000 +pixi run docs-build # build the static site into ./site +``` + +Guidelines: + +* Every page is a Markdown file under `docs/`; the navigation is defined explicitly in + [`zensical.toml`](zensical.toml). When you add a page, add it to the `nav`. +* Keep code samples accurate — base them on the real APIs and tests rather than inventing method + names. `pixi run docs-build` validates internal links. +* As elsewhere in the codebase, do not reference GitHub issue or PR numbers in the docs. + +Documentation is built on every pull request by the `Build documentation` workflow. On each +release, the `Deploy documentation` workflow publishes a versioned copy to + (via the `mike` version manager, on the +`gh-pages` branch). diff --git a/docs/core/building-plans.md b/docs/core/building-plans.md new file mode 100644 index 000000000..08a154bdc --- /dev/null +++ b/docs/core/building-plans.md @@ -0,0 +1,124 @@ +# Building plans + +`SubstraitBuilder` (in `io.substrait.dsl`) is the recommended high-level way to +construct a Substrait `Plan`. It wraps the immutable POJO builders with concise, +type-aware helpers for relations (scan, filter, project, join, aggregate, ...), +expressions (literals, field references, casts, functions), and the plan root. + +## Creating a builder + +```java +import io.substrait.dsl.SubstraitBuilder; + +SubstraitBuilder b = new SubstraitBuilder(); +``` + +The no-arg constructor uses `DefaultExtensionCatalog.DEFAULT_COLLECTION`, which +loads the standard Substrait function extensions (arithmetic, comparison, +boolean, aggregate, string, datetime, and more). To resolve custom functions, +pass your own `SimpleExtension.ExtensionCollection` — see +[Function & type extensions](extensions.md). + +```java +SubstraitBuilder b = new SubstraitBuilder(myExtensionCollection); +``` + +## Type shortcuts + +Types are created with `TypeCreator`. Throughout these examples we use two +aliases, matching the convention used across the codebase and tests: + +```java +import io.substrait.type.TypeCreator; + +TypeCreator R = TypeCreator.REQUIRED; // non-nullable types +TypeCreator N = TypeCreator.NULLABLE; // nullable types +``` + +So `R.I32` is a non-nullable 32-bit integer and `N.STRING` is a nullable string. +See [Types](types.md) for the full catalog. + +## Relation helpers take lambdas + +Most relation helpers accept a function from the input relation to the +expression(s) they need. This lets the builder resolve field references against +the input's schema for you. For example, `filter` takes a +`Function` that produces the condition: + +```java +NamedScan scan = + b.namedScan(List.of("t"), List.of("a", "b"), List.of(R.I32, R.STRING)); + +Filter filter = + b.filter(rel -> b.equal(b.fieldReference(rel, 0), b.i32(10)), scan); +``` + +`project`, `aggregate`, `sort`, and `join` follow the same pattern: the lambda +receives the input relation (or, for joins, a `JoinInput` exposing `left()` and +`right()`). + +## End-to-end example + +The following assembles a plan that scans a table, filters it, projects two +columns, and wraps the result in a plan root with output names: + +```java +import io.substrait.dsl.SubstraitBuilder; +import io.substrait.plan.Plan; +import io.substrait.relation.Filter; +import io.substrait.relation.NamedScan; +import io.substrait.relation.Project; +import io.substrait.type.TypeCreator; +import java.util.List; + +TypeCreator R = TypeCreator.REQUIRED; +SubstraitBuilder b = new SubstraitBuilder(); + +// 1. scan: table "t" with columns a (i32) and b (string) +NamedScan scan = + b.namedScan(List.of("t"), List.of("a", "b"), List.of(R.I32, R.STRING)); + +// 2. filter: keep rows where a == 10 +Filter filter = + b.filter(rel -> b.equal(b.fieldReference(rel, 0), b.i32(10)), scan); + +// 3. project: output columns a and b +Project project = + b.project( + rel -> List.of(b.fieldReference(rel, 0), b.fieldReference(rel, 1)), + filter); + +// 4. root: name the plan's output columns +Plan.Root root = b.root(project, List.of("a", "b")); + +// 5. plan: wrap the root +Plan plan = b.plan(root); +``` + +!!! note + `root(input, names)` validates that the number of output names matches the + field count of the input relation, so keep the names list in sync with the + final projection. + +`b.plan(root)` uses a default execution behavior of +`VariableEvaluationMode.PER_PLAN`. Overloads accept a custom +`Plan.ExecutionBehavior` and/or multiple roots. + +## What the builder covers + +- **Relations** — `namedScan`, `filter`, `project`, `innerJoin` / `join`, + `aggregate`, `sort`, `fetch` / `limit` / `offset`, `cross`, `set`, + `emptyVirtualTableScan`, `namedWrite`, `namedUpdate`, `expand`, and the + physical joins `hashJoin` / `mergeJoin` / `nestedLoopJoin`. See + [Relations](relations.md). +- **Expressions** — literal helpers (`bool`, `i8`..`i64`, `fp32`, `fp64`, `str`), + `fieldReference`(s), `cast`, arithmetic (`add`, `subtract`, `multiply`, + `divide`, `negate`), comparison/boolean (`equal`, `and`, `or`, `not`, + `isNull`), `ifThen`, `switchExpression`, and the generic `scalarFn` / + `aggregateFn` / `windowFn`. See [Expressions & literals](expressions.md). +- **Aggregates** — `grouping`, `measure`, and the shortcuts `count`, + `countStar`, `sum`, `sum0`, `min`, `max`, `avg`. +- **Plan assembly** — `root`, `plan`, and `remap` for output field remapping. + +Once you have a `Plan`, convert it to protobuf for storage or exchange as +described in [Serialization](serialization.md). diff --git a/docs/core/expressions.md b/docs/core/expressions.md new file mode 100644 index 000000000..a2c588a11 --- /dev/null +++ b/docs/core/expressions.md @@ -0,0 +1,153 @@ +# Expressions & literals + +Expressions are POJOs under `io.substrait.expression.Expression`. There are two +convenient ways to create them: + +- `io.substrait.expression.ExpressionCreator` — static factory methods for + literals and function invocations, where you control nullability explicitly. +- The [`SubstraitBuilder`](building-plans.md) instance helpers — terse, + extension-aware shortcuts (non-nullable literals, field references, casts, + arithmetic and boolean functions) that resolve function declarations for you. + +## Literals with ExpressionCreator + +`ExpressionCreator` factories take a leading `nullable` flag followed by the +value. This is the most direct way to create a literal of a specific +nullability: + +```java +import io.substrait.expression.ExpressionCreator; + +Expression.I32Literal a = ExpressionCreator.i32(false, 76); // non-nullable 76 +Expression.StrLiteral s = ExpressionCreator.string(true, "hello"); // nullable "hello" +Expression.BoolLiteral b = ExpressionCreator.bool(false, true); +Expression.FP64Literal f = ExpressionCreator.fp64(false, 2.5); +Expression.DateLiteral d = ExpressionCreator.date(false, 19_000); // days since epoch +``` + +There are factories for the full range of literal kinds, including +`i8`/`i16`/`i64`, `fp32`, `binary`, `fixedChar`/`varChar`, `decimal` +(from a `BigDecimal` or two's-complement `ByteString`), the temporal +`precisionTime`/`precisionTimestamp`/`precisionTimestampTZ`, interval literals, +`uuid`, and the nested `list`/`emptyList`, `map`/`emptyMap`, and +`struct`/`nestedStruct` builders. + +A typed null literal carries the type it stands in for: + +```java +Expression.NullLiteral n = ExpressionCreator.typedNull(TypeCreator.NULLABLE.I32); +``` + +## Casts, function invocations, and control flow + +`ExpressionCreator` also builds non-literal expressions: + +```java +// cast with an explicit failure behavior +Expression cast = + ExpressionCreator.cast( + TypeCreator.REQUIRED.I64, someExpression, Expression.FailureBehavior.THROW_EXCEPTION); + +// scalar function invocation from a resolved declaration +Expression.ScalarFunctionInvocation call = + ExpressionCreator.scalarFunction(declaration, TypeCreator.REQUIRED.I32, argExpr1, argExpr2); +``` + +It further provides `aggregateFunction`, `windowFunction`, `switchStatement` / +`switchClause`, `ifThenStatement` / `ifThenClause`, and the execution-context +variables `currentDate`, `currentTimezone`, `currentTimestamp`. The `declaration` +argument comes from an extension collection — see +[Function & type extensions](extensions.md). + +## Builder expression helpers + +When you already have a `SubstraitBuilder`, its instance methods are usually the +easiest path. Literal helpers produce **non-nullable** literals: + +```java +SubstraitBuilder b = new SubstraitBuilder(); + +b.bool(true); // BoolLiteral +b.i8(10); // I8Literal +b.i16(100); // I16Literal +b.i32(1000); // I32Literal +b.i64(10_000L); // I64Literal +b.fp32(1.5f); // FP32Literal +b.fp64(2.5); // FP64Literal +b.str("foo"); // StrLiteral +``` + +### Field references + +Reference an input column by zero-based index; the builder resolves the field +type from the relation's schema: + +```java +NamedScan scan = + b.namedScan(List.of("t"), List.of("a", "b"), + List.of(TypeCreator.REQUIRED.I32, TypeCreator.REQUIRED.STRING)); + +FieldReference col0 = b.fieldReference(scan, 0); +List cols = b.fieldReferences(scan, 0, 1); +``` + +For joins, `fieldReference(JoinInput, index)` indexes across the combined +left-then-right schema. + +### Casts + +```java +Expression cast = b.cast(b.i32(1), TypeCreator.REQUIRED.I64); +``` + +### Arithmetic, comparison, and boolean helpers + +These resolve the right function variant from the default extension collection +based on the argument types, and compute a sensible output type (including +nullability): + +```java +Expression left = b.i32(10); +Expression right = b.i32(20); + +b.add(left, right); // add:i32_i32 -> functions_arithmetic +b.subtract(left, right); +b.multiply(left, right); +b.divide(left, right); +b.negate(left); + +b.equal(left, right); // equal:any_any -> functions_comparison, returns boolean +b.and(cond1, cond2); // and:bool -> functions_boolean +b.or(cond1, cond2); +b.not(cond); +b.isNull(expr); // is_null:any -> functions_comparison +``` + +!!! note + Nullability is propagated automatically: for example `add` yields a nullable + result if either operand is nullable, and `equal` always returns a + non-nullable boolean. + +### Generic function invocations + +For any function not covered by a named helper, call `scalarFn` (or `aggregateFn` +/ `windowFn`) with the extension URN, the function key, the output type, and the +arguments: + +```java +Expression.ScalarFunctionInvocation substr = + b.scalarFn( + DefaultExtensionCatalog.FUNCTIONS_STRING, + "substring:str_i32_i32", + TypeCreator.REQUIRED.STRING, + strArg, startArg, lengthArg); +``` + +Aggregate measures (`count`, `sum`, `min`, `max`, `avg`, and the generic +`aggregateFn`) are covered on the [Relations](relations.md) page, where they are +used inside an `Aggregate`. + +## Next steps + +- Plug these expressions into relations in [Relations](relations.md). +- Round-trip them through protobuf in [Serialization](serialization.md). diff --git a/docs/core/extended-expressions.md b/docs/core/extended-expressions.md new file mode 100644 index 000000000..56df92fc3 --- /dev/null +++ b/docs/core/extended-expressions.md @@ -0,0 +1,126 @@ +# Extended expressions + +Not every use case needs a whole plan. Sometimes you want to describe one or more +expressions — a computed column, a filter predicate, an aggregate — to be +evaluated against a known input schema, without a surrounding relational tree. +Substrait models this with an **extended expression**, and the API lives in +`io.substrait.extendedexpression`. + +An `ExtendedExpression` bundles: + +- a list of **referred expressions**, each named for its output; and +- a **base schema** (a `NamedStruct`) that the expressions reference. + +## The POJO + +`ExtendedExpression` is built through its `builder()`. Each entry is an +`ExpressionReferenceBase`, of which there are two concrete kinds: + +- `ExpressionReference` — wraps a regular `Expression`. +- `AggregateFunctionReference` — wraps an `Aggregate.Measure`. + +```java +import io.substrait.expression.ExpressionCreator; +import io.substrait.extendedexpression.ImmutableExpressionReference; + +// an expression referring to the base schema, named "new-column" +ImmutableExpressionReference literalRef = + ImmutableExpressionReference.builder() + .expression(ExpressionCreator.i32(false, 76)) + .addOutputNames("new-column") + .build(); +``` + +A field reference works the same way — it just points into the base schema: + +```java +import io.substrait.expression.FieldReference; +import io.substrait.expression.ImmutableFieldReference; +import io.substrait.type.TypeCreator; + +ImmutableExpressionReference fieldRef = + ImmutableExpressionReference.builder() + .expression( + ImmutableFieldReference.builder() + .addSegments(FieldReference.StructField.of(0)) + .type(TypeCreator.REQUIRED.decimal(10, 2)) + .build()) + .addOutputNames("new-column") + .build(); +``` + +An aggregate is expressed as an `AggregateFunctionReference` wrapping a measure: + +```java +import io.substrait.extendedexpression.ImmutableAggregateFunctionReference; +import io.substrait.relation.Aggregate; + +ImmutableAggregateFunctionReference aggRef = + ImmutableAggregateFunctionReference.builder() + .measure(measure) // an Aggregate.Measure + .addOutputNames("new-column") + .build(); +``` + +## Assembling with a base schema + +Provide the base schema as a `NamedStruct` and add the references: + +```java +import io.substrait.extendedexpression.ImmutableExtendedExpression; +import io.substrait.type.NamedStruct; +import io.substrait.type.Type; +import io.substrait.type.TypeCreator; + +NamedStruct baseSchema = + NamedStruct.builder() + .addNames("N_NATIONKEY", "N_NAME", "N_REGIONKEY", "N_COMMENT") + .struct( + Type.Struct.builder() + .nullable(false) + .addFields( + TypeCreator.REQUIRED.decimal(10, 2), + TypeCreator.REQUIRED.STRING, + TypeCreator.REQUIRED.decimal(10, 2), + TypeCreator.REQUIRED.STRING) + .build()) + .build(); + +ImmutableExtendedExpression extendedExpression = + ImmutableExtendedExpression.builder() + .referredExpressions(List.of(literalRef)) + .baseSchema(baseSchema) + .build(); +``` + +## Serialization + +Extended expressions have their own converter pair, mirroring the plan +converters described in [Serialization](serialization.md): + +- `ExtendedExpressionProtoConverter.toProto(...)` — POJO to + `io.substrait.proto.ExtendedExpression`. +- `ProtoExtendedExpressionConverter.from(...)` — proto back to POJO. + +```java +import io.substrait.extendedexpression.ExtendedExpressionProtoConverter; +import io.substrait.extendedexpression.ProtoExtendedExpressionConverter; + +io.substrait.proto.ExtendedExpression proto = + new ExtendedExpressionProtoConverter().toProto(extendedExpression); + +ExtendedExpression roundTripped = + new ProtoExtendedExpressionConverter().from(proto); +``` + +Both converters default to `DefaultExtensionCatalog.DEFAULT_COLLECTION`; pass a +custom `SimpleExtension.ExtensionCollection` to +`ProtoExtendedExpressionConverter` when the expressions reference custom +functions. The proto message can then be encoded to bytes or JSON exactly like a +`Plan`. + +## Related + +- Build the expressions and measures used here in + [Expressions & literals](expressions.md). +- Resolve function declarations in [Function & type extensions](extensions.md). diff --git a/docs/core/extensions.md b/docs/core/extensions.md new file mode 100644 index 000000000..3e32423aa --- /dev/null +++ b/docs/core/extensions.md @@ -0,0 +1,163 @@ +# Function & type extensions + +Substrait keeps its function and type definitions out of the core specification +and in **extensions** — YAML documents that declare each function's name, +argument types, and return type. substrait-java loads these into a +`SimpleExtension.ExtensionCollection`, which the builder and converters use to +resolve a reference into a full declaration. + +## The default catalog + +`io.substrait.extension.DefaultExtensionCatalog` bundles the standard Substrait +extensions. `DEFAULT_COLLECTION` is a ready-to-use `ExtensionCollection` +containing all of them, and this is what a no-arg +[`SubstraitBuilder`](building-plans.md), `PlanProtoConverter`, and +`ProtoPlanConverter` use by default. + +```java +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.extension.SimpleExtension; + +SimpleExtension.ExtensionCollection defaults = + DefaultExtensionCatalog.DEFAULT_COLLECTION; +``` + +The class also exposes the extension **URN** constants you pass when invoking a +function. URNs follow the format `extension::` — for example +`FUNCTIONS_ARITHMETIC` is `extension:io.substrait:functions_arithmetic`. Among +them: + +| Constant | Covers | +| --- | --- | +| `FUNCTIONS_ARITHMETIC` | add, subtract, multiply, divide, sum, min, max, avg, ... | +| `FUNCTIONS_COMPARISON` | equal, not_equal, is_null, lt, gt, ... | +| `FUNCTIONS_BOOLEAN` | and, or, not, ... | +| `FUNCTIONS_STRING` | substring, concat, ... | +| `FUNCTIONS_DATETIME` | date/time functions | +| `FUNCTIONS_AGGREGATE_GENERIC` | count, ... | +| `FUNCTIONS_AGGREGATE_APPROX` | approximate aggregates | +| `FUNCTIONS_ARITHMETIC_DECIMAL` | decimal arithmetic | + +(`FUNCTIONS_ROUNDING`, `FUNCTIONS_LOGARITHMIC`, `FUNCTIONS_SET`, +`FUNCTIONS_LIST`, `FUNCTIONS_GEOMETRY`, and `EXTENSION_TYPES` are also declared.) + +## Referencing a built-in function + +Functions are addressed by a `FunctionAnchor` — a URN plus a **key** of the form +`name:arg_types` (for example `add:i32_i32` or `count:any`). The +[`SubstraitBuilder`](building-plans.md) resolves these for you. Named helpers +cover the common cases: + +```java +// arithmetic and comparison helpers resolve FunctionAnchors internally +b.add(b.i32(1), b.i32(2)); // add:i32_i32 in FUNCTIONS_ARITHMETIC +b.equal(colA, colB); // equal:any_any in FUNCTIONS_COMPARISON +``` + +For anything else, call the generic `scalarFn` / `aggregateFn` with the URN, the +function key, the output type, and the arguments: + +```java +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.type.TypeCreator; + +// scalar: substring(str, start, length) +Expression.ScalarFunctionInvocation substr = + b.scalarFn( + DefaultExtensionCatalog.FUNCTIONS_STRING, + "substring:str_i32_i32", + TypeCreator.REQUIRED.STRING, + strArg, startArg, lengthArg); + +// aggregate: count(col) +AggregateFunctionInvocation count = + b.aggregateFn( + DefaultExtensionCatalog.FUNCTIONS_AGGREGATE_GENERIC, + "count:any", + TypeCreator.REQUIRED.I64, + b.fieldReference(scan, 0)); +``` + +Under the hood these call `extensions.getScalarFunction(anchor)` / +`getAggregateFunction(anchor)` on the collection. Looking up a function whose URN +is not loaded throws `IllegalArgumentException`, so make sure the builder's +collection contains the extension you reference. + +## Loading custom extensions + +`SimpleExtension.load(...)` reads extension YAML into an `ExtensionCollection`. +There are three overloads: + +```java +// from classpath resource paths +SimpleExtension.ExtensionCollection fromResources = + SimpleExtension.load(List.of("/my/extensions/functions_custom.yaml")); + +// from a YAML string +SimpleExtension.ExtensionCollection fromString = SimpleExtension.load(yamlContent); + +// from an InputStream +SimpleExtension.ExtensionCollection fromStream = SimpleExtension.load(inputStream); +``` + +Each YAML document must declare a `urn` of the form `extension::`; +loading validates it. A minimal custom extension declaring a type looks like: + +```yaml +--- +urn: extension:my.org:my_types +types: + - name: point + structure: + x: i32 + y: i32 +``` + +Combine your extensions with the defaults using `merge`, then hand the result to +a builder or converter: + +```java +SimpleExtension.ExtensionCollection custom = SimpleExtension.load(yamlContent); + +SimpleExtension.ExtensionCollection combined = + DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(custom); + +SubstraitBuilder b = new SubstraitBuilder(combined); +``` + +!!! warning + The collection used when converting a plan must be able to resolve every + function and type the plan references. Pass the same (merged) collection to + both the builder and the `PlanProtoConverter` / `ProtoPlanConverter` so that + round-tripping succeeds. See [Serialization](serialization.md). + +## Advanced extensions + +`io.substrait.extension.AdvancedExtension` carries producer-specific data +attached to a plan or relation. It has two parts: + +- **optimizations** — optional hints that do not change semantics and may be + ignored by a consumer; and +- an **enhancement** — a semantic change that a consumer must honor. + +```java +import io.substrait.extension.AdvancedExtension; + +AdvancedExtension ext = + AdvancedExtension.builder() + .enhancement(myEnhancement) // implements AdvancedExtension.Enhancement + .addOptimizations(myOptimization) // implements AdvancedExtension.Optimization + .build(); +``` + +Because the payloads are opaque to core, serializing or deserializing a plan that +carries them requires custom extension converters passed to `PlanProtoConverter` +/ `ProtoPlanConverter`; without them, conversion throws +`UnsupportedOperationException`. + +## Related + +- Invoke functions inside relations in [Relations](relations.md) and expressions + in [Expressions & literals](expressions.md). +- Browse the full extension API in the + [API reference](https://javadoc.io/doc/io.substrait/core). diff --git a/docs/core/index.md b/docs/core/index.md new file mode 100644 index 000000000..d188d8c3f --- /dev/null +++ b/docs/core/index.md @@ -0,0 +1,93 @@ +# Core + +The `:core` module is the heart of substrait-java. It provides an immutable POJO +model for Substrait plans, relations, expressions, and types, together with +bidirectional conversion to and from the Substrait protobuf wire format and the +machinery for handling function and type extensions. + +Everything the other modules build on lives here: [Isthmus](../isthmus/index.md) +(Calcite SQL conversion) and the Spark integration both produce and consume the +POJO model described on these pages. + +## Add the dependency + +```xml + + io.substrait + core + 0.95.1 + +``` + +!!! tip + `0.95.1` is the version documented here. Check + [the javadoc index](https://javadoc.io/doc/io.substrait/core) for the latest + released version and pin accordingly. + +## The POJO model + +The model is a tree of immutable value objects generated with +[Immutables](https://immutables.github.io/). For every abstract type such as +`Expression`, `Type`, or `io.substrait.plan.Plan`, the build generates an +`Immutable` implementation, and the POJO exposes a static `builder()` that +delegates to it: + +```java +import io.substrait.plan.Plan; +import io.substrait.relation.NamedScan; + +NamedScan scan = NamedScan.builder() + .addNames("my_table") + .initialSchema(schema) + .build(); + +Plan.Root root = Plan.Root.builder().input(scan).build(); +Plan plan = Plan.builder().addRoots(root).build(); +``` + +Because the objects are immutable, they are safe to share and compare by value. +Builders support `addX`/`addAllX` for collection fields and `withX` copy methods +on the built instances. + +!!! note + Generated `Immutable*` classes only exist after the module is compiled, so an + IDE may report `ImmutableExpression.*` and new `builder()` methods as + unresolved until `:core:compileJava` has run once. + +For most use cases you do not build these objects by hand. The +[`SubstraitBuilder` DSL](building-plans.md) wraps the builders with concise, +type-aware helpers and is the recommended entry point. + +## The visitor pattern + +Traversal of the model uses double-dispatch visitors. Each layer has a visitor +interface plus, in most cases, an abstract base with sensible defaults: + +- **Expressions** — `ExpressionVisitor` (implement every case) and + `AbstractExpressionVisitor` (override only what you need). +- **Relations** — `RelVisitor` / `AbstractRelVisitor`, plus copy-on-write + transformers `RelCopyOnWriteVisitor` and `ExpressionCopyOnWriteVisitor`. +- **Types** — `TypeVisitor`, extended by `ParameterizedTypeVisitor` and + `TypeExpressionVisitor` for function-signature and derived-type expressions. + +The proto converters are themselves visitors: `ExpressionProtoConverter`, +`RelProtoConverter`, and `TypeProtoConverter` walk the POJO model to produce +proto, while the `ProtoConverter` classes switch over the proto `oneof` +cases to rebuild POJOs. See [Serialization](serialization.md) for details. + +## Map of the core documentation + +| Page | What it covers | +| --- | --- | +| [Building plans](building-plans.md) | The `SubstraitBuilder` DSL — the recommended high-level way to assemble a `Plan`. | +| [Types](types.md) | `TypeCreator`: nullability, scalar constants, and parameterized types (decimal, struct, list, map). | +| [Expressions & literals](expressions.md) | `ExpressionCreator` factories and the builder's expression helpers (field references, casts, arithmetic and boolean functions). | +| [Relations](relations.md) | The relation operators (scan, filter, project, join, aggregate, sort, fetch, set, write) and how to build them. | +| [Serialization](serialization.md) | POJO to protobuf and back, plus binary and JSON encodings. | +| [Extended expressions](extended-expressions.md) | Standalone expressions evaluated against a schema, outside a full plan. | +| [Function & type extensions](extensions.md) | The default extension catalog, loading custom YAML, and advanced extensions. | + +## API reference + +Full Javadoc for every public class is published at +[javadoc.io/doc/io.substrait/core](https://javadoc.io/doc/io.substrait/core). diff --git a/docs/core/relations.md b/docs/core/relations.md new file mode 100644 index 000000000..030291487 --- /dev/null +++ b/docs/core/relations.md @@ -0,0 +1,215 @@ +# Relations + +Relations are the operators that make up a plan's query tree. Each is an +immutable POJO under `io.substrait.relation` with a static `builder()`, and each +carries an input (or inputs), the operator-specific configuration, and an +optional output `Rel.Remap`. + +You can build relations two ways: + +- with the [`SubstraitBuilder`](building-plans.md) DSL, whose helpers resolve + field references and function declarations for you; or +- directly with the POJO `builder()` when you need full control. + +The examples below use the `R` / `N` type aliases from [Types](types.md) and a +`SubstraitBuilder b`. + +## NamedScan + +Reads a named table with a fixed schema. + +```java +// DSL: names, column names, column types +NamedScan scan = + b.namedScan(List.of("t"), List.of("a", "b"), List.of(R.I32, R.STRING)); +``` + +```java +// Direct builder +NamedScan scan = + NamedScan.builder() + .addNames("test_table") + .initialSchema( + NamedStruct.builder() + .addNames("only_column") + .struct(R.struct(R.I32)) + .build()) + .build(); +``` + +## Filter + +Keeps rows for which the condition evaluates to true. + +```java +Filter filter = + b.filter(rel -> b.equal(b.fieldReference(rel, 0), b.i32(10)), scan); +``` + +## Project + +Computes a list of output expressions from the input. + +```java +Project project = + b.project( + rel -> List.of(b.i32(1), b.fieldReference(rel, 0)), + scan); +``` + +## Join + +`Join` is the logical join. The DSL exposes `innerJoin` and the generic `join` +(with an explicit `Join.JoinType`); the condition lambda receives a `JoinInput` +exposing `left()` and `right()`. + +```java +Join join = + b.innerJoin( + inputs -> + b.equal( + b.fieldReference(inputs.left(), 0), + b.fieldReference(inputs.right(), 0)), + left, + right); +``` + +```java +// Direct builder, choosing the join type explicitly +Join join = + Join.builder() + .left(leftTable) + .right(rightTable) + .condition(ExpressionCreator.bool(false, true)) + .joinType(Join.JoinType.LEFT) + .build(); +``` + +!!! tip + The builder also offers the physical joins `hashJoin`, `mergeJoin`, and + `nestedLoopJoin`. `hashJoin`/`mergeJoin` take parallel lists of left/right + key indexes and build equality join keys for you. + +## Aggregate + +An `Aggregate` combines groupings with measures. `aggregate` takes a lambda for +the grouping(s) and one for the measures: + +```java +Aggregate aggregate = + b.aggregate( + rel -> b.grouping(rel, 1), // GROUP BY column 1 + rel -> List.of(b.count(rel, 0), b.sum(b.fieldReference(rel, 0))), + scan); +``` + +Measure shortcuts include `count`, `countStar`, `sum`, `sum0`, `min`, `max`, and +`avg`. For any other aggregate, build the invocation with `aggregateFn` and wrap +it in a measure: + +```java +AggregateFunctionInvocation afi = + b.aggregateFn( + DefaultExtensionCatalog.FUNCTIONS_AGGREGATE_GENERIC, + "count:any", + R.I64, + b.fieldReference(scan, 0)); + +Aggregate.Measure measure = b.measure(afi); +``` + +## Sort + +Orders rows by one or more sort fields. `sortFields(rel, indexes...)` produces +ascending, nulls-last sort fields; `sortField(expr, direction)` gives full +control. + +```java +Sort sort = b.sort(rel -> b.sortFields(rel, 0), scan); +``` + +## Fetch (limit / offset) + +`Fetch` skips and/or limits rows. The DSL exposes `limit`, `offset`, and the +combined `fetch`: + +```java +Fetch limited = b.limit(10, scan); // first 10 rows +Fetch skipped = b.offset(5, scan); // skip 5 rows +Fetch window = b.fetch(0, 10, scan); // offset 0, count 10 +``` + +## Cross + +Cartesian product of two inputs. + +```java +Cross cross = b.cross(left, right); +``` + +## Set + +Combines multiple inputs with a set operation. `Set.SetOp` includes +`UNION_ALL`, `UNION_DISTINCT`, `MINUS_PRIMARY`, `INTERSECTION_PRIMARY`, and more. + +```java +Set union = b.set(Set.SetOp.UNION_ALL, input1, input2); +``` + +## VirtualTableScan + +An inline table of literal rows. Build it directly with a schema and one or more +row expressions: + +```java +VirtualTableScan table = + VirtualTableScan.builder() + .initialSchema( + NamedStruct.of(List.of("col1"), R.struct(R.I32))) + .addRows(ExpressionCreator.nestedStruct(false, ExpressionCreator.i32(false, 3))) + .build(); +``` + +The DSL also offers `emptyVirtualTableScan()` for a schema-less, row-less table. + +## NamedWrite + +Writes an input relation to a named table. Specify the write operation, the +create mode, and the output mode: + +```java +NamedWrite write = + b.namedWrite( + List.of("target_table"), + List.of("c1", "c2"), + AbstractWriteRel.WriteOp.INSERT, + AbstractWriteRel.CreateMode.APPEND_IF_EXISTS, + AbstractWriteRel.OutputMode.MODIFIED_RECORDS, + scan); +``` + +`NamedUpdate` (via `b.namedUpdate(...)`) is the analogous update operator, taking +transformation expressions and a row-selection condition. + +## Output remapping + +Every relation helper has an overload accepting a `Rel.Remap`, and the input +POJO builders accept `.remap(...)`. A remap reorders or filters the operator's +output columns by index: + +```java +Rel.Remap remap = b.remap(0, 1); // keep columns 0 and 1 +Sort sort = b.sort(rel -> b.sortFields(rel, 0), remap, scan); +``` + +## Building a plan + +Wrap the top relation in a `Plan.Root` and a `Plan`: + +```java +Plan.Root root = b.root(project, List.of("a", "b")); +Plan plan = b.plan(root); +``` + +See [Building plans](building-plans.md) for the full end-to-end flow and +[Serialization](serialization.md) to convert the result to protobuf. diff --git a/docs/core/serialization.md b/docs/core/serialization.md new file mode 100644 index 000000000..304fcac30 --- /dev/null +++ b/docs/core/serialization.md @@ -0,0 +1,122 @@ +# Serialization + +The POJO model and the Substrait protobuf wire format are two representations of +the same plan. The `:core` module converts between them in both directions, and +the generated protobuf classes (package `io.substrait.proto`) handle the final +encoding to bytes or JSON. + +## Plan: POJO to protobuf and back + +The two entry points live in `io.substrait.plan`: + +- `PlanProtoConverter.toProto(Plan)` — POJO `io.substrait.plan.Plan` to proto + `io.substrait.proto.Plan`. +- `ProtoPlanConverter.from(io.substrait.proto.Plan)` — proto back to POJO. + +```java +import io.substrait.plan.Plan; +import io.substrait.plan.PlanProtoConverter; +import io.substrait.plan.ProtoPlanConverter; + +// POJO -> proto +io.substrait.proto.Plan proto = new PlanProtoConverter().toProto(plan); + +// proto -> POJO +Plan roundTripped = new ProtoPlanConverter().from(proto); +``` + +Both converters default to `DefaultExtensionCatalog.DEFAULT_COLLECTION`. When +your plan references custom functions or types, pass a matching +`SimpleExtension.ExtensionCollection` (and, for advanced extensions, custom +extension converters) to the constructor: + +```java +PlanProtoConverter toProto = new PlanProtoConverter(myExtensions); +ProtoPlanConverter fromProto = new ProtoPlanConverter(myExtensions); +``` + +See [Function & type extensions](extensions.md) for building extension +collections. + +## Encoding to bytes + +The proto `Plan` is a standard protobuf message, so serialize and parse it with +the usual protobuf API: + +```java +// serialize to a byte array +byte[] bytes = proto.toByteArray(); + +// parse back from bytes +io.substrait.proto.Plan parsed = io.substrait.proto.Plan.parseFrom(bytes); + +// then convert to the POJO model +Plan plan = new ProtoPlanConverter().from(parsed); +``` + +This binary form is the canonical way to store or exchange plans between +Substrait producers and consumers. + +## Encoding to JSON + +For a human-readable form, use protobuf's `JsonFormat`: + +```java +import com.google.protobuf.util.JsonFormat; + +// proto -> JSON +String json = JsonFormat.printer().print(proto); + +// JSON -> proto +io.substrait.proto.Plan.Builder builder = io.substrait.proto.Plan.newBuilder(); +JsonFormat.parser().merge(json, builder); +io.substrait.proto.Plan fromJson = builder.build(); +``` + +!!! note + JSON is convenient for debugging, tests, and interop, but the binary form is + more compact and is what most tooling exchanges. + +## Lower-level converters + +`PlanProtoConverter` and `ProtoPlanConverter` delegate to per-layer converters +that you can use directly when working with a single relation, expression, or +type. The naming tells you the direction: `ProtoConverter` is POJO to +proto, `ProtoConverter` is proto to POJO. + +| Layer | POJO to proto | proto to POJO | +| --- | --- | --- | +| Relations | `RelProtoConverter` | `ProtoRelConverter` | +| Expressions | `ExpressionProtoConverter` | `ProtoExpressionConverter` | +| Types | `TypeProtoConverter` | `ProtoTypeConverter` | + +These converters thread an `ExtensionCollector` so that function and type +references discovered while walking the tree are gathered into the plan's +extension declarations. A minimal relation round-trip wires them together like +this: + +```java +import io.substrait.extension.ExtensionCollector; +import io.substrait.relation.RelProtoConverter; +import io.substrait.relation.ProtoRelConverter; + +ExtensionCollector collector = new ExtensionCollector(); +RelProtoConverter relToProto = new RelProtoConverter(collector); + +io.substrait.proto.Rel protoRel = relToProto.toProto(rel); + +ProtoRelConverter protoToRel = + new ProtoRelConverter(collector, DefaultExtensionCatalog.DEFAULT_COLLECTION); +io.substrait.relation.Rel back = protoToRel.from(protoRel); +``` + +!!! tip + Test code in `:core` extends `io.substrait.TestBase` and calls + `verifyRoundTrip(Rel)` / `verifyRoundTrip(Expression)` to assert + POJO to proto to POJO fidelity — a useful pattern to mirror in your own tests. + +## Related + +- Build the plans you serialize in [Building plans](building-plans.md). +- Serialize schema-bound standalone expressions via + [Extended expressions](extended-expressions.md). diff --git a/docs/core/types.md b/docs/core/types.md new file mode 100644 index 000000000..812686bbb --- /dev/null +++ b/docs/core/types.md @@ -0,0 +1,138 @@ +# Types + +Substrait types are POJOs under `io.substrait.type.Type`. Rather than building +them one by one, you create them through `TypeCreator`, a factory that fixes the +nullability up front and exposes constants for the scalar types plus builders for +the parameterized ones. + +## Nullability: REQUIRED vs NULLABLE + +Every Substrait type is either nullable or not. `TypeCreator` has two shared +instances, one for each: + +```java +import io.substrait.type.TypeCreator; + +TypeCreator.REQUIRED; // produces non-nullable types +TypeCreator.NULLABLE; // produces nullable types +``` + +A common idiom (used throughout the codebase and the rest of these docs) is to +alias them as `R` and `N`: + +```java +TypeCreator R = TypeCreator.REQUIRED; +TypeCreator N = TypeCreator.NULLABLE; +``` + +You can also select one dynamically with `TypeCreator.of(boolean)`: + +```java +TypeCreator t = TypeCreator.of(nullable); // NULLABLE if true, else REQUIRED +``` + +To flip the nullability of an existing type, use the static helpers: + +```java +io.substrait.type.Type nullableI32 = TypeCreator.asNullable(R.I32); +io.substrait.type.Type requiredI32 = TypeCreator.asNotNullable(N.I32); +``` + +## Scalar type constants + +Each `TypeCreator` instance exposes constants for the simple types, already at +its nullability: + +| Constant | Substrait type | +| --- | --- | +| `BOOLEAN` | boolean | +| `I8`, `I16`, `I32`, `I64` | 8/16/32/64-bit integers | +| `FP32`, `FP64` | single/double precision float | +| `STRING` | UTF-8 string | +| `BINARY` | variable-length binary | +| `DATE` | date | +| `INTERVAL_YEAR` | year-month interval | +| `UUID` | UUID | + +```java +TypeCreator R = TypeCreator.REQUIRED; + +io.substrait.type.Type i32 = R.I32; // non-nullable i32 +io.substrait.type.Type str = R.STRING; // non-nullable string +io.substrait.type.Type fp64 = R.FP64; // non-nullable fp64 +io.substrait.type.Type date = R.DATE; // non-nullable date +``` + +## Parameterized types + +Parameterized types are created with instance methods that carry the creator's +nullability: + +```java +TypeCreator R = TypeCreator.REQUIRED; + +// decimal(precision, scale) +io.substrait.type.Type dec = R.decimal(10, 2); + +// fixed- and variable-length character/binary +io.substrait.type.Type fchar = R.fixedChar(20); +io.substrait.type.Type vchar = R.varChar(255); +io.substrait.type.Type fbin = R.fixedBinary(16); + +// temporal types with fractional-second precision +io.substrait.type.Type ts = R.precisionTimestamp(6); +io.substrait.type.Type tstz = R.precisionTimestampTZ(6); +io.substrait.type.Type time = R.precisionTime(6); +io.substrait.type.Type iday = R.intervalDay(6); +``` + +### Structs, lists, and maps + +```java +TypeCreator R = TypeCreator.REQUIRED; + +// struct with i32 and string fields +io.substrait.type.Type.Struct struct = R.struct(R.I32, R.STRING); + +// list of strings +io.substrait.type.Type list = R.list(R.STRING); + +// map from string to i32 +io.substrait.type.Type map = R.map(R.STRING, R.I32); +``` + +`struct(...)` also accepts an `Iterable` or a `Stream`, which is +handy when building a schema from a computed set of field types. + +### User-defined types + +Types declared by an extension are referenced by URN and name: + +```java +io.substrait.type.Type udt = + TypeCreator.REQUIRED.userDefined("extension:my.org:my_types", "point"); +``` + +See [Function & type extensions](extensions.md) for how extension URNs work. + +## Named structs + +A relation's schema pairs field names with a struct type via +`io.substrait.type.NamedStruct`: + +```java +import io.substrait.type.NamedStruct; + +NamedStruct schema = + NamedStruct.of(List.of("a", "b"), TypeCreator.REQUIRED.struct(R.I32, R.STRING)); +``` + +The [`SubstraitBuilder`](building-plans.md) `namedScan` helper builds the +`NamedStruct` for you from parallel lists of column names and types. + +## Next steps + +- Use these types when creating literals and casts in + [Expressions & literals](expressions.md). +- See the full type hierarchy in the + [API reference](https://javadoc.io/doc/io.substrait/core). diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 000000000..432dffa62 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,109 @@ +# Getting started + +This page shows how to add substrait-java to your build and produce your first Substrait plan. + +## Requirements + +- **Java 17 or newer** to run applications that depend on substrait-java. +- A build tool such as Gradle or Maven. + +## Add a dependency + +substrait-java is published to Maven Central under the group `io.substrait`. Add the module you +need. Most applications start with **core**; add **isthmus** if you want SQL conversion, or a +**spark** variant for Spark integration. + +!!! tip "Check the latest version" + The examples below use `0.95.1`. Check + [Maven Central](https://central.sonatype.com/namespace/io.substrait) for the newest release + and substitute it. + +=== "Gradle (Kotlin DSL)" + + ```kotlin + dependencies { + implementation("io.substrait:core:0.95.1") + // Optional: SQL <-> Substrait conversion + implementation("io.substrait:isthmus:0.95.1") + } + ``` + +=== "Gradle (Groovy DSL)" + + ```groovy + dependencies { + implementation 'io.substrait:core:0.95.1' + // Optional: SQL <-> Substrait conversion + implementation 'io.substrait:isthmus:0.95.1' + } + ``` + +=== "Maven" + + ```xml + + io.substrait + core + 0.95.1 + + ``` + +The available artifacts are: + +| Module | Maven artifact | Purpose | +| --- | --- | --- | +| Core | `io.substrait:core` | Plan model + protobuf conversion | +| Isthmus | `io.substrait:isthmus` | SQL ⇄ Substrait (Calcite) | +| Spark | `io.substrait:spark34_2.12`, `spark35_2.12`, `spark40_2.13` | Spark ⇄ Substrait (per Spark/Scala version — see [Spark compatibility](spark/compatibility.md)) | + +!!! note "Logging" + Core uses the [SLF4J](https://www.slf4j.org/) logging API. If you want log output, add an + SLF4J provider to your runtime classpath; otherwise substrait-java runs fine but stays silent. + +## Your first plan + +The `SubstraitBuilder` DSL from the core module is the quickest way to assemble a plan. This +example builds a plan that scans an `orders` table and filters it, then serializes it to the +Substrait protobuf form. + +```java +import io.substrait.dsl.SubstraitBuilder; +import io.substrait.plan.Plan; +import io.substrait.plan.PlanProtoConverter; +import io.substrait.relation.Rel; +import io.substrait.type.TypeCreator; +import java.util.List; + +SubstraitBuilder builder = new SubstraitBuilder(); +TypeCreator R = TypeCreator.REQUIRED; + +// SELECT id, customer FROM orders WHERE id = 1 +Rel scan = + builder.namedScan( + List.of("orders"), + List.of("id", "customer"), + List.of(R.I32, R.STRING)); + +Rel filtered = + builder.filter( + input -> builder.equal(builder.fieldReference(input, 0), builder.i32(1)), + scan); + +Plan.Root root = builder.root(filtered, List.of("id", "customer")); +Plan plan = builder.plan(root); + +// Convert the POJO plan to the Substrait protobuf message +io.substrait.proto.Plan proto = new PlanProtoConverter().toProto(plan); +byte[] bytes = proto.toByteArray(); +``` + +That `byte[]` (or its JSON form) is a portable Substrait plan any Substrait-aware engine can +consume. See [Serialization](core/serialization.md) for the round trip back to a POJO and for the +JSON representation. + +## Where to go next + +- **[Core](core/index.md)** — build plans in Java, model types and expressions, and serialize. +- **[Isthmus](isthmus/index.md)** — convert SQL to and from Substrait. +- **[Isthmus CLI](isthmus-cli/index.md)** — do the same from the command line. +- **[Spark](spark/index.md)** — produce and consume plans with Apache Spark. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..5ee1e3235 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,57 @@ +# substrait-java + +**substrait-java** is the Java implementation of [Substrait](https://substrait.io/) — a +cross-language specification for relational query plans. It gives you an immutable Java object +model for plans, relations, expressions, and types, bidirectional conversion to and from the +Substrait protobuf wire format, and integrations with Apache Calcite (SQL) and Apache Spark. + +If you are new here, start with [Getting started](getting-started.md), then dive into the module +that matches your use case. + +## What is Substrait? + +Substrait defines a standard, engine-agnostic representation of data-manipulation operations +(scans, filters, projections, joins, aggregations, and so on) together with a catalogue of +functions and types. A plan produced by one system can be consumed by another. substrait-java +lets you produce and consume those plans from the JVM without hand-assembling protobuf. + +## Modules + +The project is organized into four modules. Each has its own section in this documentation. + +### Core + +The heart of the project: an immutable POJO model for plans, relations, expressions, and types, +plus converters to and from the Substrait protobuf format. Build plans directly in Java with the +`SubstraitBuilder` DSL, then serialize them for any Substrait-aware engine. + +→ [Core documentation](core/index.md) + +### Isthmus + +SQL ⇄ Substrait conversion built on Apache Calcite. Turn a SQL query into a Substrait plan, turn +a SQL expression into a Substrait extended expression, and convert plans back to SQL or to Calcite +relational trees. + +→ [Isthmus documentation](isthmus/index.md) + +### Isthmus CLI + +A command-line tool — the `isthmus` native binary — that converts SQL queries and expressions to +Substrait from the shell, without writing any Java. + +→ [Isthmus CLI documentation](isthmus-cli/index.md) + +### Spark + +Convert Apache Spark logical plans to and from Substrait. Produce a portable plan from a Spark +query and execute a Substrait plan on Spark. Published for multiple Spark and Scala versions. + +→ [Spark documentation](spark/index.md) + +## Getting help + +- [Substrait specification](https://substrait.io/) — the parent project. +- [Substrait community](https://substrait.io/community/) — how to get involved. +- [substrait-java on GitHub](https://github.com/substrait-io/substrait-java) — source, issues, + and releases. diff --git a/docs/isthmus-cli/examples.md b/docs/isthmus-cli/examples.md new file mode 100644 index 000000000..5dd41fd32 --- /dev/null +++ b/docs/isthmus-cli/examples.md @@ -0,0 +1,208 @@ +# Examples + +Worked examples of converting SQL to Substrait with the `isthmus` binary. The commands use +the built binary path (`./isthmus-cli/build/native/nativeCompile/isthmus`); substitute your +own binary location as needed — see [Install & build](install.md). + +Outputs below are `PROTOJSON` (the default) and are **abbreviated** — long, repeated +sections are elided with a `// ...` comment. See [Usage](usage.md) for the full flag +reference. + +## SQL query to a Substrait Plan + +Declare the catalog with `-c`/`--create`, then pass the query as the positional argument. +The tool prints a Substrait `Plan`: + +```bash +./isthmus-cli/build/native/nativeCompile/isthmus \ + -c "CREATE TABLE Persons ( firstName VARCHAR, lastName VARCHAR, zip INT )" \ + "SELECT lastName, firstName FROM Persons WHERE zip = 90210" +``` + +```json +{ + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }], + "extensions": [{ + "extensionFunction": { + "functionAnchor": 0, + "name": "equal:any1_any1", + "extensionUrnReference": 1 + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { "emit": { "outputMapping": [3, 4] } }, + "input": { + "filter": { + "input": { + "read": { + "baseSchema": { + "names": ["FIRSTNAME", "LASTNAME", "ZIP"] + // ... struct types elided ... + }, + "namedTable": { "names": ["PERSONS"] } + } + }, + "condition": { + "scalarFunction": { + "functionReference": 0, + "args": [ + { "selection": { "directReference": { "structField": { "field": 2 } } } }, + { "literal": { "i32": 90210 } } + ] + // ... outputType elided ... + } + } + } + }, + "expressions": [ + { "selection": { "directReference": { "structField": { "field": 1 } } } }, + { "selection": { "directReference": { "structField": { "field": 0 } } } } + ] + } + }, + "names": ["LASTNAME", "FIRSTNAME"] + } + }], + "expectedTypeUrls": [] +} +``` + +!!! note "Uppercased names" + The table and column names appear upper-cased (`PERSONS`, `FIRSTNAME`) because the + default `--unquotedcasing` policy is `TO_UPPER`. See [Usage](usage.md#unquoted-identifier-casing). + +## SQL expression to a Substrait ExtendedExpression + +Adding `-e`/`--expression` switches the tool to expression mode: it converts each +expression against the catalog and prints an `ExtendedExpression`. + +### Projection expression + +```bash +./isthmus-cli/build/native/nativeCompile/isthmus \ + -c "CREATE TABLE NATION (N_NATIONKEY BIGINT NOT NULL, N_NAME CHAR(25), N_REGIONKEY BIGINT NOT NULL, N_COMMENT VARCHAR(152))" \ + -e "N_REGIONKEY + 10" +``` + +```json +{ + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic" + }], + "extensions": [{ + "extensionFunction": { + "functionAnchor": 0, + "name": "add:i64_i64", + "extensionUrnReference": 1 + } + }], + "referredExpr": [{ + "expression": { + "scalarFunction": { + "functionReference": 0, + "outputType": { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, + "arguments": [ + { "value": { "selection": { "directReference": { "structField": { "field": 2 } } } } }, + { "value": { "cast": { "type": { "i64": {} }, "input": { "literal": { "i32": 10 } } } } } + ] + } + }, + "outputNames": ["new-column"] + }], + "baseSchema": { + "names": ["N_NATIONKEY", "N_NAME", "N_REGIONKEY", "N_COMMENT"] + // ... struct types elided ... + }, + "expectedTypeUrls": [] +} +``` + +### Filter expression + +A boolean expression produces the same `ExtendedExpression` shape, with a comparison +function extension: + +```bash +./isthmus-cli/build/native/nativeCompile/isthmus \ + -c "CREATE TABLE NATION (N_NATIONKEY BIGINT NOT NULL, N_NAME CHAR(25), N_REGIONKEY BIGINT NOT NULL, N_COMMENT VARCHAR(152))" \ + -e "N_REGIONKEY > 10" +``` + +```json +{ + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" + }], + "extensions": [{ + "extensionFunction": { + "functionAnchor": 0, + "name": "gt:any_any", + "extensionUrnReference": 1 + } + }], + "referredExpr": [{ + "expression": { + "scalarFunction": { + "functionReference": 0, + "outputType": { "bool": { "nullability": "NULLABILITY_REQUIRED" } }, + "arguments": [ + { "value": { "selection": { "directReference": { "structField": { "field": 2 } } } } }, + { "value": { "cast": { "type": { "i64": {} }, "input": { "literal": { "i32": 10 } } } } } + ] + } + }, + "outputNames": ["new-column"] + }], + "baseSchema": { + "names": ["N_NATIONKEY", "N_NAME", "N_REGIONKEY", "N_COMMENT"] + // ... struct types elided ... + }, + "expectedTypeUrls": [] +} +``` + +!!! tip "Multiple expressions at once" + Because `-e` has arity `1..*`, you can convert several expressions in one call — each + becomes an entry in `referredExpr`: + + ```bash + ./isthmus-cli/build/native/nativeCompile/isthmus \ + -c "$LINEITEM" \ + -e 'l_orderkey + 9888486986' 'l_orderkey * 2' 'l_orderkey > 10' 'l_orderkey in (10, 20)' + ``` + +## Smoke-test scripts + +The module ships two runnable scripts that exercise the binary against a range of inputs — +handy as ready-made examples. Both resolve the binary from the `ISTHMUS` environment +variable, defaulting to `build/native/nativeCompile/isthmus` (relative to the `isthmus-cli` +module), so set `ISTHMUS` to run them against a downloaded release binary. + +### `smoke.sh` + +Runs a spread of queries and expressions against a single `LINEITEM` table — simple +`SELECT`, a filter, an aggregate, and literal / reference / filter / projection +expressions (including converting several expressions in one call): + +```bash +./isthmus-cli/src/test/script/smoke.sh +``` + +### `tpch_smoke.sh` + +Generates Substrait plans for all 22 TPC-H queries, using the schema and query files from +the `isthmus` module's test resources: + +```bash +./isthmus-cli/src/test/script/tpch_smoke.sh +``` + +This is a good way to see what non-trivial plans look like across a well-known query set. diff --git a/docs/isthmus-cli/index.md b/docs/isthmus-cli/index.md new file mode 100644 index 000000000..efbbd28b8 --- /dev/null +++ b/docs/isthmus-cli/index.md @@ -0,0 +1,37 @@ +# Isthmus CLI + +The Isthmus CLI is a native command-line tool, named `isthmus`, that drives the +[Isthmus](../isthmus/index.md) library from the shell. It uses the Apache Calcite SQL +compiler to convert: + +- **SQL queries** into a Substrait [`Plan`](https://substrait.io/serialization/binary_serialization/), and +- **SQL expressions** into a Substrait [`ExtendedExpression`](https://substrait.io/expressions/extended_expression/). + +The binary is a [GraalVM](https://www.graalvm.org/) native image built with +[picocli](https://picocli.info/), so it starts instantly and has no JVM or classpath to +manage. It is ideal for experimenting with Substrait, inspecting the plan a given SQL +statement produces, and scripting conversions in CI or shell pipelines. + +## CLI vs. the Isthmus Java API + +The CLI is a thin wrapper around the same conversion classes documented under +[Isthmus](../isthmus/index.md) (`SqlToSubstrait` and `SqlExpressionToSubstrait`). Choose +based on how you want to work: + +- **Use the CLI** when you want a quick, zero-setup way to turn SQL into Substrait from a + terminal or shell script, to eyeball the generated plan, or to generate fixtures for + tests and documentation. +- **Use the [Isthmus Java API](../isthmus/index.md)** when you are embedding SQL-to-Substrait + conversion in an application, need the resulting POJO model for further processing, or + want to customize the conversion (custom type systems, function extension collections, + and so on). + +Both paths produce the same Substrait output for the same input. + +## On this page set + +| Page | What it covers | +| --- | --- | +| [Install & build](install.md) | Download a prebuilt release binary or build `isthmus` yourself with GraalVM. | +| [Usage](usage.md) | Every flag and argument, defaults, and how the tool decides between plan and expression conversion. | +| [Examples](examples.md) | Worked commands with (abbreviated) output, plus the runnable smoke-test scripts. | diff --git a/docs/isthmus-cli/install.md b/docs/isthmus-cli/install.md new file mode 100644 index 000000000..0adf78b44 --- /dev/null +++ b/docs/isthmus-cli/install.md @@ -0,0 +1,66 @@ +# Install & build + +There are two ways to get the `isthmus` binary: download a prebuilt native binary from a +GitHub release, or build it yourself from source with GraalVM. + +## Download a prebuilt binary + +Every [substrait-java release](https://github.com/substrait-io/substrait-java/releases) +attaches prebuilt native-image binaries for Linux and macOS. The assets are named after +the release version: + +- `isthmus-ubuntu-` — Linux (x86-64) +- `isthmus-macOS-` — macOS + +Download the asset that matches your platform, then make it executable and (optionally) +put it on your `PATH`: + +```bash +# Example: after downloading isthmus-ubuntu- for Linux +chmod +x isthmus-ubuntu- +mv isthmus-ubuntu- /usr/local/bin/isthmus + +isthmus --version +``` + +The binaries are standalone native images: no JVM or Java installation is required to run +them. + +## Build from source + +Isthmus is built as a native executable via GraalVM. From the repository root, run: + +```bash +./gradlew nativeCompile +``` + +The resulting binary is written to: + +```text +isthmus-cli/build/native/nativeCompile/isthmus +``` + +!!! warning "GraalVM 25 with `native-image` is required" + `nativeCompile` runs the GraalVM `native-image` tool with whatever JDK is running the + Gradle daemon. You must have a **GraalVM 25 JDK** (with the `native-image` component) + installed and its location set in the `GRAALVM_HOME` environment variable. This is a + different toolchain from the JDK 17 used to build the rest of the project. + +Verify the build: + +```bash +./isthmus-cli/build/native/nativeCompile/isthmus --version +``` + +which prints the binary and Substrait spec versions: + +```text +isthmus version +Substrait version +``` + +!!! tip "Referencing the binary in scripts" + The bundled smoke-test scripts resolve the binary from the `ISTHMUS` environment + variable, falling back to `build/native/nativeCompile/isthmus` (relative to the + `isthmus-cli` module). Set `ISTHMUS` to point at a downloaded binary if you want to run + the scripts against a release build. See [Examples](examples.md). diff --git a/docs/isthmus-cli/usage.md b/docs/isthmus-cli/usage.md new file mode 100644 index 000000000..f41a47a27 --- /dev/null +++ b/docs/isthmus-cli/usage.md @@ -0,0 +1,84 @@ +# Usage + +The `isthmus` command converts SQL queries and SQL expressions to Substrait. This page +documents every flag and argument, their defaults, and how the tool decides what to +produce. + +The examples below use the built binary path; substitute your own binary location (for +example a downloaded release binary on your `PATH`) as needed. See +[Install & build](install.md). + +## General form + +```bash +isthmus [-hV] [--outputformat=] [--unquotedcasing=] \ + [-c=]... [-e=...]... [] +``` + +Running `isthmus` with no arguments (and `-h`/`--help`) prints this usage help. + +## Arguments and options + +### Positional argument + +| Argument | Arity | Description | +| --- | --- | --- | +| `` | `0..1` | A single SQL query to convert to a Substrait `Plan`. | + +Because a query string often starts with words that could look like options, use the +`--` separator to mark the end of options when the query follows other flags: + +```bash +isthmus --create "$DDL" -- "SELECT * FROM lineitem" +``` + +### Options + +| Option | Arity | Default | Description | +| --- | --- | --- | --- | +| `-e`, `--expression=...` | `1..*` | — | One or more SQL expressions, e.g. `col + 1`. When present, the tool converts expressions instead of a query. | +| `-c`, `--create=` | repeatable | none | One or more `CREATE TABLE` statements defining the catalog, e.g. `CREATE TABLE T1(foo int, bar bigint)`. Repeat the flag to declare multiple tables. | +| `--outputformat=` | 1 | `PROTOJSON` | Output format for the generated message: `PROTOJSON`, `PROTOTEXT`, or `BINARY`. | +| `--unquotedcasing=` | 1 | `TO_UPPER` | Calcite's casing policy for unquoted identifiers: `UNCHANGED`, `TO_UPPER`, or `TO_LOWER`. | +| `-h`, `--help` | — | — | Show the help message and exit. | +| `-V`, `--version` | — | — | Print version information and exit. | + +!!! note "Enum values are case-insensitive" + Values for `--outputformat` and `--unquotedcasing` are matched case-insensitively, so + `--outputformat prototext` and `--outputformat PROTOTEXT` are equivalent. + +## What gets produced + +The tool operates in one of two modes, decided by whether `-e`/`--expression` is present: + +- **Expression mode** — if one or more `-e`/`--expression` values are given, the tool + converts them (against the catalog built from any `-c` statements) and prints a Substrait + `ExtendedExpression`. The `` positional argument is not used in this mode. +- **Query mode** (default) — otherwise, the tool builds a catalog from the `-c`/`--create` + statements and converts the `` query, printing a Substrait `Plan`. + +## Output formats + +`--outputformat` controls how the resulting protobuf message is serialized to standard +output: + +- **`PROTOJSON`** (default) — protobuf JSON. Default-valued fields are included, so the + output is verbose (you will see empty arrays such as `"expectedTypeUrls": []` and + zero-valued fields like `"typeVariationReference": 0`). This is the most readable format + for inspecting plans. +- **`PROTOTEXT`** — protobuf text format. +- **`BINARY`** — the raw protobuf binary wire format, written to stdout. Redirect it to a + file (for example `> plan.pb`) when you want to consume it programmatically. + +## Unquoted identifier casing + +`--unquotedcasing` maps directly to Calcite's SQL parser policy for identifiers that are +not double-quoted: + +- **`TO_UPPER`** (default) — unquoted identifiers are folded to upper case. This is why, in + the [examples](examples.md), a table declared as `Persons` with a column `firstName` + appears in the plan as `PERSONS` / `FIRSTNAME`. +- **`TO_LOWER`** — folded to lower case. +- **`UNCHANGED`** — preserved exactly as written. + +Choose the policy that matches the SQL dialect and catalog naming you are targeting. diff --git a/docs/isthmus/customization.md b/docs/isthmus/customization.md new file mode 100644 index 000000000..1ee671b82 --- /dev/null +++ b/docs/isthmus/customization.md @@ -0,0 +1,217 @@ +# Customization + +Every Isthmus converter — [`SqlToSubstrait`](sql-to-substrait.md), +[`SqlExpressionToSubstrait`](sql-expressions.md), +[`SubstraitToSql`](substrait-to-sql.md), and +[`SubstraitToCalcite`](substrait-to-calcite.md) — takes its configuration from a single +`ConverterProvider`. Customizing conversion behavior therefore means constructing (or +subclassing) a `ConverterProvider` and passing it to the converter. + +## `ConverterProvider` + +The no-argument constructor supplies system defaults: the standard Substrait extension +catalog (`DefaultExtensionCatalog.DEFAULT_COLLECTION`) and the Substrait type factory +(`SubstraitTypeSystem.TYPE_FACTORY`). + +```java +// Defaults +ConverterProvider provider = new ConverterProvider(); + +// Custom extension collection (functions/types), default type factory +ConverterProvider withExtensions = new ConverterProvider(myExtensions); + +// Full control: type factory, extensions, and the function/type converters +ConverterProvider full = + new ConverterProvider( + SubstraitTypeSystem.TYPE_FACTORY, + myExtensions, + scalarFunctionConverter, + aggregateFunctionConverter, + windowFunctionConverter, + typeConverter); +``` + +`ConverterProvider` exposes overridable methods for each configurable concern, +including: + +| Method | Controls | +| --- | --- | +| `getSqlParserConfig()` | Calcite SQL parsing (identifier casing, DDL parser, conformance) | +| `getCalciteConnectionConfig()` | table-name case sensitivity | +| `getSqlToRelConverterConfig()` | field trimming, sub-query expansion | +| `getSqlOperatorTable()` | which SQL operators/functions are valid | +| `getCallConverters()` | how Calcite `RexCall`s map to Substrait | +| `getScalarFunctionConverter()` / `getAggregateFunctionConverter()` / `getWindowFunctionConverter()` | function conversion | +| `getTypeConverter()` / `getTypeFactory()` / `getTypeSystem()` | type conversion | +| `getSchemaResolver()` | Substrait-to-Calcite schema inference | +| `getExecutionBehavior()` | the plan's execution behavior | + +The deepest customization is achieved by extending the class — which is exactly what +the two dynamic providers below do. + +## Custom SQL parser configuration + +By default Isthmus upper-cases unquoted identifiers (`Casing.TO_UPPER`), uses the DDL +parser factory, and applies lenient SQL conformance. Override `getSqlParserConfig()` to +change this — for example to keep unquoted identifiers in their original case: + +```java +import org.apache.calcite.avatica.util.Casing; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl; +import org.apache.calcite.sql.validate.SqlConformanceEnum; + +ConverterProvider provider = + new ConverterProvider() { + @Override + public SqlParser.Config getSqlParserConfig() { + return SqlParser.Config.DEFAULT + .withUnquotedCasing(Casing.UNCHANGED) + .withParserFactory(SqlDdlParserImpl.FACTORY) + .withConformance(SqlConformanceEnum.LENIENT); + } + }; + +Plan plan = new SqlToSubstrait(provider).convert(sql, catalog); +``` + +## Custom functions + +Isthmus knows how to translate the standard Substrait function set out of the box. To +teach it functions it does not know — custom functions defined in your own extension +YAML — you register additional signatures with the function converters and put the +resulting `ConverterProvider` to work. + +Load your extension YAML, merge it with the defaults if needed, build a +`ScalarFunctionConverter` (and/or aggregate/window converters) that carries the extra +signatures, and assemble a `ConverterProvider`: + +```java +import io.substrait.extension.SimpleExtension; +import io.substrait.isthmus.ConverterProvider; +import io.substrait.isthmus.SubstraitTypeSystem; +import io.substrait.isthmus.TypeConverter; +import io.substrait.isthmus.expression.FunctionMappings; +import io.substrait.isthmus.expression.ScalarFunctionConverter; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import java.util.List; + +// 1. Load the custom extension collection from YAML. +SimpleExtension.ExtensionCollection customExtensions = + SimpleExtension.load(myFunctionsYaml); + +// 2. Describe the matching Calcite operator and register its signature. +SqlFunction customScalarFn = + new SqlFunction( + "custom_scalar", + SqlKind.OTHER_FUNCTION, + ReturnTypes.explicit(SqlTypeName.VARCHAR), + null, + null, + SqlFunctionCategory.USER_DEFINED_FUNCTION); + +List additionalScalarSignatures = + List.of(FunctionMappings.s(customScalarFn)); + +// 3. Build a converter that carries those signatures. +ScalarFunctionConverter scalarFunctionConverter = + new ScalarFunctionConverter( + customExtensions.scalarFunctions(), + additionalScalarSignatures, + SubstraitTypeSystem.TYPE_FACTORY, + TypeConverter.DEFAULT); + +// 4. Assemble a ConverterProvider (aggregate/window converters analogous). +``` + +The same pattern applies to aggregate and window functions via +`AggregateFunctionConverter` and `WindowFunctionConverter`. `FunctionMappings.s(op)` +creates a signature entry linking a Calcite `SqlOperator` to a Substrait function name. + +### User-defined types + +Custom functions often involve custom types. Supply a `UserTypeMapper` to a +`TypeConverter` so that Isthmus can translate them in both directions: + +```java +import io.substrait.isthmus.TypeConverter; +import io.substrait.isthmus.UserTypeMapper; + +TypeConverter typeConverter = new TypeConverter(myUserTypeMapper); +``` + +`myUserTypeMapper` implements `toSubstrait(RelDataType)` and +`toCalcite(Type.UserDefined)`, returning `null` for types it does not handle. See +[Types & type system](types.md). + +## Dynamic providers for UDFs + +Registering a Calcite `SqlOperator` by hand for every function is tedious. Two provider +subclasses generate that wiring automatically from your extension collection. + +### `DynamicConverterProvider` + +`DynamicConverterProvider` treats any scalar function in the extension collection that +is **not** part of the known standard function mappings as a dynamic UDF. It generates +the SQL operators and call converters for those functions automatically, so SQL can +call them without manual configuration: + +```java +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.extension.SimpleExtension; +import io.substrait.isthmus.DynamicConverterProvider; +import io.substrait.isthmus.SqlToSubstrait; +import io.substrait.isthmus.sql.SubstraitCreateStatementParser; +import io.substrait.plan.Plan; +import java.util.List; +import org.apache.calcite.prepare.Prepare; + +// Merge custom UDFs with the default catalog. +SimpleExtension.ExtensionCollection extensions = + DefaultExtensionCatalog.DEFAULT_COLLECTION.merge( + SimpleExtension.load(List.of("/extensions/scalar_functions_custom.yaml"))); + +SqlToSubstrait converter = new SqlToSubstrait(new DynamicConverterProvider(extensions)); + +Prepare.CatalogReader catalog = + SubstraitCreateStatementParser.processCreateStatementsToCatalog( + "CREATE TABLE t(x VARCHAR NOT NULL)"); + +// regexp_extract_custom, format_text, ... are UDFs from the YAML above. +Plan plan = converter.convert("SELECT regexp_extract_custom(x, 'ab') FROM t", catalog); +``` + +### `AutomaticDynamicFunctionMappingConverterProvider` + +`AutomaticDynamicFunctionMappingConverterProvider` goes a step further: it inspects the +scalar, aggregate, **and** window functions of the extension collection, finds those +without an explicit mapping, and auto-generates operators and signatures for all three +kinds. This lets a standard extension function that simply has not been hand-mapped +(for example `strftime` from `functions_datetime.yaml`) be used from SQL without adding +it to the built-in mappings. + +```java +import io.substrait.isthmus.AutomaticDynamicFunctionMappingConverterProvider; +import io.substrait.isthmus.SqlToSubstrait; + +SqlToSubstrait converter = + new SqlToSubstrait(new AutomaticDynamicFunctionMappingConverterProvider()); +``` + +## Custom SQL dialects for output + +When rendering Substrait back to SQL, the Calcite `SqlDialect` you pass to +[`SubstraitToSql.convert`](substrait-to-sql.md) controls the generated text. Provide a +custom `SqlDialect` (for example subclassing `SparkSqlDialect` and overriding +`unparseCall`) to control how specific operators are rendered for a target engine. + +## Related + +- [Types & type system](types.md) — `TypeConverter`, `UserTypeMapper`, and the type + system. +- [core extensions](../core/extensions.md) — loading and merging extension YAMLs. +- [SQL to Substrait](sql-to-substrait.md) — where a customized provider is put to work. diff --git a/docs/isthmus/index.md b/docs/isthmus/index.md new file mode 100644 index 000000000..2693888ee --- /dev/null +++ b/docs/isthmus/index.md @@ -0,0 +1,81 @@ +# Isthmus + +Isthmus is the SQL bridge in substrait-java. It converts SQL queries and SQL +expressions into [Substrait](https://substrait.io/) plans and expressions, and +converts Substrait back into SQL, using [Apache Calcite](https://calcite.apache.org/) +as the SQL parser, validator, and relational-algebra engine. + +Under the hood every conversion path routes through Calcite's relational model: + +```text +SQL <-> Calcite RelNode / RexNode <-> Substrait POJO <-> Substrait protobuf +``` + +Isthmus wires up the Calcite parser, validator, type system, and operator table so +that the relational algebra it produces maps cleanly onto Substrait, and it supplies +the visitors that translate between Calcite's `RelNode`/`RexNode` trees and the +substrait-java [POJO model](../core/index.md). + +## Add the dependency + +```groovy +dependencies { + implementation "io.substrait:isthmus:0.95.1" +} +``` + +!!! tip "Check the latest version" + `0.95.1` is current at the time of writing. Check + [Maven Central](https://central.sonatype.com/artifact/io.substrait/isthmus) for + newer releases. The group is `io.substrait`. + +The same functionality is available from the command line without writing any Java; +see the [Isthmus CLI](../isthmus-cli/index.md). + +## The entry-point converters + +Each direction of conversion has a small, focused entry-point class. All of them are +configured by a single `ConverterProvider`, whose no-argument constructor supplies +sensible system defaults (the standard Substrait extension catalog and the Substrait +type system): + +```java +import io.substrait.isthmus.SqlToSubstrait; + +// Uses ConverterProvider defaults +SqlToSubstrait converter = new SqlToSubstrait(); +``` + +| Class | Direction | Returns | +| --- | --- | --- | +| `SqlToSubstrait` | SQL query -> Substrait | POJO `io.substrait.plan.Plan` | +| `SqlExpressionToSubstrait` | SQL expression -> Substrait | proto `io.substrait.proto.ExtendedExpression` | +| `SubstraitToSql` | Substrait -> SQL | one SQL `String` per plan root | +| `SubstraitToCalcite` | Substrait -> Calcite | `RelNode` / `RelRoot` | + +`ConverterProvider` is the single point of configuration shared by all four. Pass a +customized provider (or one of its subclasses, such as `DynamicConverterProvider`) to +change extensions, functions, the type factory, or the SQL parser configuration. See +[Customization](customization.md). + +## Pages in this section + +- [SQL to Substrait](sql-to-substrait.md) — convert one or more SQL statements into a + Substrait `Plan`, using a catalog built from `CREATE TABLE` statements. +- [SQL expressions](sql-expressions.md) — convert standalone SQL expressions into a + Substrait `ExtendedExpression`. +- [Substrait to SQL](substrait-to-sql.md) — render a Substrait `Plan` back to SQL in a + chosen dialect. +- [Substrait to Calcite](substrait-to-calcite.md) — convert Substrait relations into + Calcite `RelNode`/`RelRoot` trees. +- [Types & type system](types.md) — how Calcite and Substrait types map, and the + Substrait type system Isthmus relies on. +- [Customization](customization.md) — `ConverterProvider`, dynamic providers, custom + functions/UDFs, and parser configuration. +- [Supported SQL](supported-sql.md) — the breadth of SQL Isthmus translates, driven by + the TPC-H and TPC-DS test suites. + +## API reference + +The full Javadoc for every public class is published at +[javadoc.io/doc/io.substrait/isthmus](https://javadoc.io/doc/io.substrait/isthmus). diff --git a/docs/isthmus/sql-expressions.md b/docs/isthmus/sql-expressions.md new file mode 100644 index 000000000..ea74b4909 --- /dev/null +++ b/docs/isthmus/sql-expressions.md @@ -0,0 +1,93 @@ +# SQL expressions + +`SqlExpressionToSubstrait` converts standalone SQL expressions — not full queries — +into a Substrait [extended expression](../core/extended-expressions.md). An extended +expression is a self-contained payload that pairs one or more expressions with the +schema (base struct) they are evaluated against, which makes it a convenient way to +push a filter or projection expression to an engine without wrapping it in a plan. + +Unlike [`SqlToSubstrait`](sql-to-substrait.md), this converter returns the **protobuf** +type `io.substrait.proto.ExtendedExpression` directly. + +## The `convert` methods + +```java +public io.substrait.proto.ExtendedExpression convert( + String sqlExpression, List createStatements) throws SqlParseException; + +public io.substrait.proto.ExtendedExpression convert( + String[] sqlExpressions, List createStatements) throws SqlParseException; +``` + +- `sqlExpression` / `sqlExpressions` — one or more SQL expressions (not `SELECT` + statements), for example `L_ORDERKEY > 10`. +- `createStatements` — a list of `CREATE TABLE` statements that define the columns the + expressions may reference. Their columns become the base schema; referencing a column + not present in the schema fails validation. + +Each converted expression is added as an `ExpressionReference` with a generated output +name (`column-1`, `column-2`, …), and the combined base schema (a Substrait +`NamedStruct`) is attached to the result. + +!!! warning "Column names must be unique" + The columns from all the `createStatements` are flattened into a single name space + used to bind field references. Two columns with the same name (across one or more + tables) raise `IllegalArgumentException: There is no support for duplicate column + names`. + +## Example + +Define a schema, then convert a single expression: + +```java +import io.substrait.isthmus.SqlExpressionToSubstrait; +import io.substrait.proto.ExtendedExpression; +import java.util.List; + +List schema = + List.of("CREATE TABLE lineitem (L_ORDERKEY BIGINT, L_COMMENT VARCHAR)"); + +ExtendedExpression expr = + new SqlExpressionToSubstrait().convert("L_ORDERKEY > 10", schema); +``` + +The expression `L_ORDERKEY > 10` becomes a Substrait scalar-function call +(`greater_than`) over a field reference and an `i32` literal, carried in an +`ExtendedExpression` whose base schema has the `L_ORDERKEY` and `L_COMMENT` fields. + +## Supported expression kinds + +The following categories all convert (drawn directly from the Isthmus test suite): + +```sql +2 -- literal +L_ORDERKEY -- field reference +L_ORDERKEY > 10 -- comparison (scalar function) +L_ORDERKEY + 10 -- arithmetic (scalar function) +L_ORDERKEY IN (10, 20) -- IN +L_ORDERKEY IS NOT NULL -- IS NOT NULL +L_ORDERKEY IS NULL -- IS NULL +``` + +## Converting several expressions at once + +Pass an array to bundle multiple expressions into one `ExtendedExpression`. They share +the same base schema and are named `column-1`, `column-2`, … in order: + +```java +String[] expressions = { + "L_ORDERKEY", + "L_ORDERKEY > 10", + "L_ORDERKEY + 10", + "L_ORDERKEY IN (10, 20)", + "L_ORDERKEY IS NOT NULL" +}; + +ExtendedExpression expr = new SqlExpressionToSubstrait().convert(expressions, schema); +``` + +## Related + +- [core extended expressions](../core/extended-expressions.md) — the extended + expression model and its POJO <-> protobuf serialization. +- [SQL to Substrait](sql-to-substrait.md) — convert whole SQL statements into a `Plan`. diff --git a/docs/isthmus/sql-to-substrait.md b/docs/isthmus/sql-to-substrait.md new file mode 100644 index 000000000..194a2f989 --- /dev/null +++ b/docs/isthmus/sql-to-substrait.md @@ -0,0 +1,120 @@ +# SQL to Substrait + +`SqlToSubstrait` converts one or more SQL statements into a Substrait plan. Because +SQL references tables by name, the converter needs a *catalog* describing those tables. +Isthmus builds that catalog from `CREATE TABLE` statements, so a full conversion has +two steps: parse the schema into a catalog, then convert the query against it. + +The result is a substrait-java POJO `io.substrait.plan.Plan` — the same immutable model +described in the [core docs](../core/index.md). Serialize it to the protobuf wire +format when you need to send or persist it. + +## The `convert` methods + +```java +public Plan convert(String sqlStatements, Prepare.CatalogReader catalogReader) + throws SqlParseException; + +public Plan convert(String sqlStatements, Prepare.CatalogReader catalogReader, + SqlDialect sqlDialect) throws SqlParseException; +``` + +- `sqlStatements` — a string containing one or more SQL statements (separate multiple + statements with `;`). +- `catalogReader` — a Calcite `Prepare.CatalogReader` describing the tables the SQL + references (see below). +- `sqlDialect` — optional; supply a Calcite `SqlDialect` to control how the SQL is + *parsed* (for example a dialect's identifier-quoting and casing rules). Without it, + Isthmus uses the parser configuration from the `ConverterProvider`. + +Every root query in the input becomes one `Plan.Root`; the converter tags the plan with +a version whose producer is `"isthmus"`. + +## Building a catalog from CREATE statements + +`SubstraitCreateStatementParser.processCreateStatementsToCatalog(...)` parses SQL +`CREATE TABLE` statements and returns a `CalciteCatalogReader` you can hand straight to +`convert`. It accepts either a varargs of strings or a `List`: + +```java +public static CalciteCatalogReader processCreateStatementsToCatalog(String... createStatements) + throws SqlParseException; + +public static CalciteCatalogReader processCreateStatementsToCatalog(List createStatements) + throws SqlParseException; +``` + +Each string may itself contain several `CREATE TABLE` statements. Only `CREATE TABLE` +is accepted — `CREATE TABLE ... AS SELECT` (CTAS) is rejected. Primary-key and other +key constraints are parsed and ignored, so they are safe to include. + +## Worked example + +Define a schema, convert a query, and serialize the resulting plan to protobuf: + +```java +import io.substrait.isthmus.SqlToSubstrait; +import io.substrait.isthmus.sql.SubstraitCreateStatementParser; +import io.substrait.plan.Plan; +import io.substrait.plan.PlanProtoConverter; +import org.apache.calcite.prepare.Prepare; + +String[] createStatements = { + "CREATE TABLE users (id BIGINT, name VARCHAR, signup_date DATE)", + "CREATE TABLE orders (order_id BIGINT, user_id BIGINT, total DECIMAL(10, 2))" +}; + +// 1. Parse the schema into a catalog. +Prepare.CatalogReader catalog = + SubstraitCreateStatementParser.processCreateStatementsToCatalog(createStatements); + +// 2. Convert a query into a Substrait Plan POJO. +Plan plan = + new SqlToSubstrait() + .convert( + "SELECT u.name, o.total " + + "FROM users u JOIN orders o ON u.id = o.user_id " + + "WHERE o.total > 100.00", + catalog); + +// 3. Serialize the POJO Plan to the protobuf wire format. +io.substrait.proto.Plan proto = new PlanProtoConverter().toProto(plan); +``` + +`plan` is a fully-formed Substrait plan you can inspect, transform, or (as shown) +serialize. Converting the POJO `Plan` to protobuf is a core concern rather than an +Isthmus one — `PlanProtoConverter` lives in `:core`. See +[core serialization](../core/serialization.md) for the round trip and for reading a +proto plan back into a POJO with `ProtoPlanConverter`. + +!!! note "Identifier casing" + With the default parser configuration, unquoted identifiers are upper-cased and + table/column lookups are case-insensitive, so `users` and `USERS` resolve to the + same table. Quote identifiers to preserve their exact case. The parser behavior is + configurable via the `ConverterProvider`; see [Customization](customization.md). + +## Multiple statements + +Passing several statements in one call produces a plan with one root per statement: + +```java +Plan plan = + new SqlToSubstrait() + .convert( + "SELECT order_id FROM orders; " + + "SELECT user_id FROM orders WHERE total > 20;", + catalog); + +// plan.getRoots() has two entries, one per SELECT. +``` + +A trailing semicolon on a single statement is fine, and each statement is converted +independently. + +## Related + +- [SQL expressions](sql-expressions.md) — convert a bare SQL expression (rather than a + full query) into a Substrait `ExtendedExpression`. +- [Substrait to SQL](substrait-to-sql.md) — go the other way and render a plan back to + SQL. +- [core serialization](../core/serialization.md) — POJO `Plan` <-> protobuf. diff --git a/docs/isthmus/substrait-to-calcite.md b/docs/isthmus/substrait-to-calcite.md new file mode 100644 index 000000000..4b22e71be --- /dev/null +++ b/docs/isthmus/substrait-to-calcite.md @@ -0,0 +1,90 @@ +# Substrait to Calcite + +`SubstraitToCalcite` converts Substrait relations into Apache Calcite relational trees. +It is the step that [Substrait to SQL](substrait-to-sql.md) builds on, but it is also +useful on its own when you want to hand a Substrait plan to Calcite for optimization, +execution, or further inspection as `RelNode`s. + +## The `convert` methods + +```java +public RelNode convert(Rel rel); +public RelRoot convert(Plan.Root root); +``` + +- `convert(Rel)` returns a Calcite `RelNode` — the relational operator tree for a + single Substrait relation. +- `convert(Plan.Root)` returns a Calcite `RelRoot`, applying the root's final field + names to the output row type (including nested struct, array, and map fields) and + deriving the appropriate `SqlKind` (for example `INSERT`/`UPDATE`/`DELETE` for a + table modification, otherwise the query kind). + +## Constructing the converter + +```java +public SubstraitToCalcite(ConverterProvider converterProvider); +public SubstraitToCalcite(ConverterProvider converterProvider, Prepare.CatalogReader catalogReader); +``` + +Calcite needs a schema to resolve the tables a plan reads from. There are two +strategies: + +- **Supply a catalog** — pass a `Prepare.CatalogReader` (for example one built with + `SubstraitCreateStatementParser.processCreateStatementsToCatalog(...)`, as in + [SQL to Substrait](sql-to-substrait.md)). The converter resolves table names against + it. +- **Let Isthmus infer the schema** — with the single-argument constructor, the + `ConverterProvider`'s schema resolver walks the leaf (read) nodes of the plan and + synthesizes a Calcite schema on the fly. Override + `ConverterProvider#getSchemaResolver()` to customize this behavior. + +## Example + +Build a Substrait `Plan.Root` and convert it to a Calcite `RelRoot`: + +```java +import io.substrait.isthmus.ConverterProvider; +import io.substrait.isthmus.SubstraitToCalcite; +import io.substrait.plan.Plan.Root; +import io.substrait.type.Type; +import io.substrait.type.TypeCreator; +import java.util.List; +import org.apache.calcite.rel.RelRoot; + +Iterable types = List.of(TypeCreator.REQUIRED.I64, TypeCreator.REQUIRED.STRING); +Root root = + Root.builder() + .input(sb.namedScan(List.of("stores"), List.of("s_store_id", "s"), types)) + .addNames("s_store_id", "store") + .build(); + +SubstraitToCalcite substraitToCalcite = new SubstraitToCalcite(new ConverterProvider()); +RelRoot relRoot = substraitToCalcite.convert(root); + +// relRoot.fields carries the top-level output names: [s_store_id, store] +``` + +(`sb` here is a `io.substrait.dsl.SubstraitBuilder`; see +[building plans](../core/building-plans.md).) + +## Building the RelBuilder with the Substrait type system + +Internally, `convert` obtains a Calcite `RelBuilder` from the `ConverterProvider`, and +that builder is created with the **Substrait type system** +(`SubstraitTypeSystem.TYPE_SYSTEM`, via the provider's type factory). This is essential +for correctness, not a detail: Calcite's default type system caps `DECIMAL` precision at +19, while Substrait carries decimals at precision up to 38. If the `RelBuilder` used a +mismatched type system, Calcite's expression simplification would re-derive decimal +arithmetic at the lower precision and wrap results in a truncating +`CAST(... AS DECIMAL(19, 0))`. + +The default `ConverterProvider` already does the right thing. If you construct your own +`RelBuilder` for Substrait-to-Calcite work, be sure to set its type system to +`SubstraitTypeSystem.TYPE_SYSTEM`. See [Types & type system](types.md) for the full +explanation. + +## Related + +- [Substrait to SQL](substrait-to-sql.md) — renders the Calcite tree back to SQL text. +- [Types & type system](types.md) — the type mapping and the decimal-precision caveat. +- [Customization](customization.md) — customizing schema resolution and converters. diff --git a/docs/isthmus/substrait-to-sql.md b/docs/isthmus/substrait-to-sql.md new file mode 100644 index 000000000..0a53adb21 --- /dev/null +++ b/docs/isthmus/substrait-to-sql.md @@ -0,0 +1,83 @@ +# Substrait to SQL + +`SubstraitToSql` renders a Substrait plan back into SQL. It first converts the plan to +a Calcite `RelNode` tree (see [Substrait to Calcite](substrait-to-calcite.md)) and then +uses Calcite's `RelToSqlConverter` to emit SQL text in the dialect you choose. + +## The `convert` method + +```java +public List convert(Plan plan, SqlDialect dialect); +``` + +- `plan` — the Substrait POJO `io.substrait.plan.Plan` to render. +- `dialect` — the Calcite `SqlDialect` that controls the SQL that is generated + (keyword casing, quoting, function rendering, and so on). + +The method returns a `List` with **one SQL statement per `Plan.Root`**, in +order. Each root is converted to Calcite, projected onto its final field names, and +serialized with the given dialect. + +## Example + +```java +import io.substrait.isthmus.SubstraitToSql; +import io.substrait.isthmus.sql.SubstraitSqlDialect; +import io.substrait.plan.Plan; +import java.util.List; + +Plan plan = /* a Substrait plan, e.g. from SqlToSubstrait or a proto */; + +List sql = new SubstraitToSql().convert(plan, SubstraitSqlDialect.DEFAULT); +String firstStatement = sql.get(0); +``` + +Any Calcite `SqlDialect` works. `SubstraitSqlDialect.DEFAULT` is the Isthmus dialect +used internally; to target a specific engine, pass one of Calcite's built-in dialects +(for example `org.apache.calcite.sql.dialect.SparkSqlDialect.DEFAULT`). To change how +individual operators are rendered per engine, supply a custom dialect — see +[Customization](customization.md). + +## Round trip from a proto plan + +`SubstraitToSql` operates on the POJO `Plan`, so a plan received as protobuf must first +be read into a POJO with `ProtoPlanConverter` (from `:core`). The full +POJO -> proto -> POJO -> SQL round trip looks like this: + +```java +import io.substrait.plan.Plan; +import io.substrait.plan.PlanProtoConverter; +import io.substrait.plan.ProtoPlanConverter; +import io.substrait.isthmus.SubstraitToSql; +import io.substrait.isthmus.sql.SubstraitSqlDialect; + +// POJO Plan -> protobuf +io.substrait.proto.Plan proto = new PlanProtoConverter().toProto(plan); + +// protobuf -> POJO Plan +Plan restored = new ProtoPlanConverter().from(proto); + +// POJO Plan -> SQL +List sql = new SubstraitToSql().convert(restored, SubstraitSqlDialect.DEFAULT); +``` + +See [core serialization](../core/serialization.md) for the POJO <-> protobuf converters +in detail. + +!!! note "Lossy by nature" + Substrait is a lower-level algebra than SQL, and Calcite's `RelToSqlConverter` + reconstructs *a* SQL statement with the same semantics rather than the original + query text. Expect differences in aliases, parenthesization, and the exact shape of + the generated SQL. + +## Converting a single relation to Calcite + +If you only need the Calcite side, `substraitRelToCalciteRel` converts a Substrait +`Rel` to a Calcite `RelNode` given a catalog: + +```java +public RelNode substraitRelToCalciteRel(Rel relRoot, Prepare.CatalogReader catalog); +``` + +For converting whole plan roots to Calcite `RelRoot`s, use +[`SubstraitToCalcite`](substrait-to-calcite.md) directly. diff --git a/docs/isthmus/supported-sql.md b/docs/isthmus/supported-sql.md new file mode 100644 index 000000000..53819c1fe --- /dev/null +++ b/docs/isthmus/supported-sql.md @@ -0,0 +1,89 @@ +# Supported SQL + +Isthmus supports a broad slice of analytical SQL. The clearest measure of that breadth +is the industry-standard benchmark suites that run as tests on every build: + +- **All 22 [TPC-H](https://www.tpc.org/tpch/) queries** convert to Substrait and back + to SQL. +- **Most of the 99 [TPC-DS](https://www.tpc.org/tpcds/) queries** convert; a small + number use alternate query forms in the suite. + +These are exercised query-by-query in `TpchQueryTest` and `TpcdsQueryTest`, which run +each query through the pipeline `SQL -> Substrait POJO -> protobuf -> SQL`. `TpcdsQueryTest` +covers queries 1–99, substituting alternate forms for a handful of them (queries 27, 36, +70, and 86). + +!!! note "What the suites assert" + The TPC-H and TPC-DS tests assert that each query converts through the full pipeline + *without error*; they do not (yet) assert semantic equivalence of the round-tripped + plan. Stronger fidelity checks — asserting POJO equality across round trips — are + made by the plan-level tests such as `Substrait2SqlTest` and `SimplePlansTest`, which + use the `assertFullRoundTrip` harnesses in `PlanTestBase`. + +## Supported categories + +Drawn from the test suites, Isthmus translates at least the following: + +- **Projection** — column selection, computed columns, aliases. +- **Filtering** — `WHERE` with comparisons, `AND`/`OR`, `IN`, `IS [NOT] NULL`, + `BETWEEN`, `LIKE`, and `CASE`/`WHEN` expressions. +- **Arithmetic and scalar functions** — numeric, string (`substring`, `lower`, `upper`, + …), and date/time functions (`extract`, `month`, `year`, `current_timestamp`, + `current_date`, interval arithmetic). +- **Joins** — inner, `LEFT`, `RIGHT`, and `FULL` outer joins, plus comma (cross) joins. +- **Aggregation** — `GROUP BY`, aggregate functions (`sum`, `count`, `avg`, + `approx_count_distinct`, …), `DISTINCT` aggregates, `FILTER (WHERE ...)` on aggregates, + and grouped extensions: `GROUPING SETS`, `ROLLUP`, and `GROUP_ID()`. +- **Window functions** — `OVER (...)` window aggregates. +- **Set operations** — `UNION` / `UNION ALL`, `INTERSECT`, `EXCEPT`, and their + variants. +- **Ordering and limiting** — `ORDER BY` (with `ASC`/`DESC` and `NULLS FIRST`/`NULLS + LAST`), `LIMIT`, and `OFFSET`. +- **VALUES / virtual tables** — literal row sets, including `SELECT` with no `FROM` + (e.g. `SELECT 1`). +- **DDL** — `CREATE TABLE` statements, parsed into a catalog with + `SubstraitCreateStatementParser` (see [SQL to Substrait](sql-to-substrait.md)). +- **DML** — table modifications (`INSERT`/`UPDATE`/`DELETE`), which map to Substrait + write relations. + +## Example round-trip queries + +These are representative queries verified by `Substrait2SqlTest` via +`assertFullRoundTrip`: + +```sql +-- Join with arithmetic and a decimal literal +SELECT l_partkey + l_orderkey, l_extendedprice * 0.1 + 100.0, o_orderkey +FROM lineitem +JOIN orders ON l_orderkey = o_orderkey +WHERE l_shipdate < date '1998-01-01'; + +-- Aggregation with DISTINCT +SELECT l_partkey, count(l_tax), COUNT(distinct l_discount) +FROM lineitem +GROUP BY l_partkey; + +-- Grouping sets +SELECT sum(l_discount) +FROM lineitem +GROUP BY grouping sets ((l_orderkey, l_commitdate), l_shipdate); + +-- CASE expression +SELECT case when p_size > 100 then 'large' + when p_size > 50 then 'medium' + else 'small' end +FROM part; +``` + +## Extending coverage + +Functions and types outside the standard Substrait set are supported by registering +custom functions, dynamic UDFs, or user-defined types. See +[Customization](customization.md). + +## Related + +- [SQL to Substrait](sql-to-substrait.md) — the conversion entry point. +- [Substrait to SQL](substrait-to-sql.md) — the reverse direction used by the round-trip + tests. +- [Customization](customization.md) — extending the supported function/type set. diff --git a/docs/isthmus/types.md b/docs/isthmus/types.md new file mode 100644 index 000000000..7c57e9f78 --- /dev/null +++ b/docs/isthmus/types.md @@ -0,0 +1,116 @@ +# Types & type system + +Converting between SQL and Substrait means converting between Calcite's type model +(`RelDataType`) and Substrait's (`io.substrait.type.Type`). Isthmus does this with two +pieces: + +- **`SubstraitTypeSystem`** — a Calcite `RelDataTypeSystem` whose precision and scale + rules match Substrait's, plus the type factory built on top of it. +- **`TypeConverter`** — the bidirectional mapper between Calcite and Substrait types. + +## `SubstraitTypeSystem` + +`SubstraitTypeSystem` extends Calcite's default type system and adjusts the limits that +matter for Substrait: + +| Setting | Value | +| --- | --- | +| Max `DECIMAL` precision | 38 | +| Max `DECIMAL` scale | 38 | +| Max precision for `TIME`, `TIMESTAMP`, `TIMESTAMP_WITH_LOCAL_TIME_ZONE`, and the year/day interval types | 6 (microseconds) | +| `shouldConvertRaggedUnionTypesToVarying()` | `true` | + +Two shared singletons are exposed: + +```java +public static final RelDataTypeSystem TYPE_SYSTEM; // the type system +public static final RelDataTypeFactory TYPE_FACTORY; // a JavaTypeFactoryImpl over it +``` + +`ConverterProvider` uses `TYPE_FACTORY` by default, so all Isthmus conversions run with +this type system unless you override the type factory. + +!!! note "Why the public no-arg constructor exists" + Prefer the `TYPE_SYSTEM` singleton. The public no-argument constructor is kept + because Calcite's `Frameworks`/Avatica machinery re-instantiates a type system from + its class name (via a default constructor) when it is supplied to a + `FrameworkConfig`. The type system is stateless, so any instance is equivalent to + the singleton — do not remove the constructor. + +## `TypeConverter` + +`TypeConverter` maps types in both directions: + +```java +// Calcite -> Substrait +Type toSubstrait(RelDataType type); +NamedStruct toNamedStruct(RelDataType rowType); + +// Substrait -> Calcite +RelDataType toCalcite(RelDataTypeFactory factory, TypeExpression typeExpression); +``` + +`TypeConverter.DEFAULT` handles all the built-in types and does not map user-defined +types. To support user-defined types, construct one with a `UserTypeMapper` +(see [Customization](customization.md)). + +### Type mapping + +| Substrait | Calcite `SqlTypeName` | +| --- | --- | +| `bool` | `BOOLEAN` | +| `i8` | `TINYINT` | +| `i16` | `SMALLINT` | +| `i32` | `INTEGER` | +| `i64` | `BIGINT` | +| `fp32` | `REAL` | +| `fp64` | `DOUBLE` (`FLOAT` also maps to `fp64`) | +| `decimal(p, s)` | `DECIMAL(p, s)` | +| `string` | `VARCHAR` (unbounded) | +| `varchar(n)` | `VARCHAR(n)` | +| `fixedchar(n)` | `CHAR(n)` | +| `binary` | `VARBINARY` | +| `fixedbinary(n)` | `BINARY(n)` | +| `date` | `DATE` | +| `precision_time(p)` | `TIME(p)` | +| `precision_timestamp(p)` | `TIMESTAMP(p)` | +| `precision_timestamp_tz(p)` | `TIMESTAMP_WITH_LOCAL_TIME_ZONE(p)` | +| `interval_year` | year-to-month interval | +| `interval_day` | day-to-second interval | +| `struct` | `ROW` | +| `list` | `ARRAY` | +| `map` | `MAP` | +| `user-defined` | via `UserTypeMapper` | + +Nullability is preserved in both directions. A `DECIMAL` with precision greater than 38 +is rejected with `UnsupportedOperationException`, and a `precision_time` / +`precision_timestamp` whose precision exceeds the type system's maximum (6) is rejected +with `IllegalArgumentException`. + +## Decimal precision caveat (important) + +When converting **Substrait to Calcite**, the Calcite `RelBuilder` must be created with +`SubstraitTypeSystem.TYPE_SYSTEM`. + +Calcite's default type system caps `DECIMAL` precision at 19, while Substrait — and the +expressions Isthmus produces — carry decimals at precision up to 38. If a `RelBuilder` +built on the *default* type system processes a converted plan, the precision-38 types on +the expressions disagree with the type system's precision-19 ceiling. Calcite's +expression simplification (`RexSimplify`, run by `RelBuilder.project`) then re-derives +decimal arithmetic at precision 19 and wraps the result in a truncating +`CAST(... AS DECIMAL(19, 0))`, silently discarding scale and precision. Using the +Substrait type system keeps Calcite's type derivation aligned with the types the +expressions actually carry, so no truncating cast is inserted. + +The default `ConverterProvider` already builds its `RelBuilder` with the Substrait type +system (its `getRelBuilder` supplies `getTypeSystem()`, which is derived from the +Substrait type factory). This caveat matters only if you build a `RelBuilder` yourself +for Substrait-to-Calcite conversion — always set its type system to +`SubstraitTypeSystem.TYPE_SYSTEM`. + +## Related + +- [Substrait to Calcite](substrait-to-calcite.md) — where the `RelBuilder` is used. +- [Customization](customization.md) — supplying a `UserTypeMapper` and a custom + `TypeConverter`. +- [core types](../core/types.md) — the Substrait type model itself. diff --git a/docs/spark/compatibility.md b/docs/spark/compatibility.md new file mode 100644 index 000000000..2489b2e4a --- /dev/null +++ b/docs/spark/compatibility.md @@ -0,0 +1,118 @@ +# Compatibility & dependencies + +The `spark` module is published as **several independent artifacts**, one per supported +Spark/Scala combination. There is deliberately **no single cross-version jar**: each artifact is +compiled against a specific Spark release on a specific Scala binary version, and mixing them with +a different runtime will fail at load time. Pick the artifact that matches the Spark and Scala +versions your application already runs on. + +## Variant matrix + +| Variant | Spark | Scala | Maven artifact (`groupId:artifactId`) | Gradle subproject | +| --- | --- | --- | --- | --- | +| Spark 3.4 | 3.4.4 | 2.12 | `io.substrait:spark34_2.12` | `:spark:spark-3.4_2.12` | +| Spark 3.5 | 3.5.4 | 2.12 | `io.substrait:spark35_2.12` | `:spark:spark-3.5_2.12` | +| Spark 4.0 | 4.0.2 | 2.13 | `io.substrait:spark40_2.13` | `:spark:spark-4.0_2.13` | + +The artifact id encodes both the Spark major/minor version and the Scala binary version +(`spark_`), matching the usual Scala convention. The **Gradle +subproject** column is only relevant when building from source in this repository; the shared Scala +source lives in `spark/src` and each subproject compiles it against its own Spark/Scala versions. + +## Adding the dependency + +The current release is **0.95.1** (group `io.substrait`). Add the single variant matching your +runtime. + +### Spark 3.5 (Scala 2.12) + +=== "Gradle" + + ```kotlin + dependencies { + implementation("io.substrait:spark35_2.12:0.95.1") + } + ``` + +=== "Maven" + + ```xml + + io.substrait + spark35_2.12 + 0.95.1 + + ``` + +### Spark 3.4 (Scala 2.12) + +=== "Gradle" + + ```kotlin + dependencies { + implementation("io.substrait:spark34_2.12:0.95.1") + } + ``` + +=== "Maven" + + ```xml + + io.substrait + spark34_2.12 + 0.95.1 + + ``` + +### Spark 4.0 (Scala 2.13) + +=== "Gradle" + + ```kotlin + dependencies { + implementation("io.substrait:spark40_2.13:0.95.1") + } + ``` + +=== "Maven" + + ```xml + + io.substrait + spark40_2.13 + 0.95.1 + + ``` + +!!! tip "Match Scala binary versions" + On Spark 3.4/3.5 use the Scala **2.12** variant; on Spark 4.0 use the Scala **2.13** variant. + The Scala binary version must line up with the rest of your Spark application's dependencies — + a 2.12 artifact will not load in a 2.13 runtime, and vice versa. + +## Spark 4.0 package caveat + +Spark 4.0 split the "classic" (`RDD`/Catalyst-backed) `SparkSession` and `Dataset` out into a new +package. The types you import differ by Spark version: + +| Spark version | `SparkSession` / `Dataset` package | +| --- | --- | +| 3.4, 3.5 | `org.apache.spark.sql` | +| 4.0 | `org.apache.spark.sql.classic` | + +!!! warning "Import the right `Dataset` / `SparkSession` on Spark 4.0" + On Spark 4.0, the classic API used by these workflows — including the + `Dataset.ofRows(spark, plan)` call that executes a converted plan (see + [Consuming plans](consuming-plans.md)) — lives in + `org.apache.spark.sql.classic.{Dataset, SparkSession}`. + + On Spark 3.4/3.5 the same classes are in `org.apache.spark.sql.{Dataset, SparkSession}`. Code + that compiles against one variant will need its imports adjusted for the other. This is one of + the reasons the artifacts are version-specific rather than shared. + +## API reference + +The published Javadoc is per variant. For example: + +- [`spark34_2.12`](https://javadoc.io/doc/io.substrait/spark34_2.12) +- [`spark35_2.12`](https://javadoc.io/doc/io.substrait/spark35_2.12) +- [`spark40_2.13`](https://javadoc.io/doc/io.substrait/spark40_2.13) diff --git a/docs/spark/consuming-plans.md b/docs/spark/consuming-plans.md new file mode 100644 index 000000000..774188cd1 --- /dev/null +++ b/docs/spark/consuming-plans.md @@ -0,0 +1,92 @@ +# Consuming plans + +Consuming reverses [producing](producing-plans.md): starting from serialized Substrait bytes, you +rebuild a Spark logical plan and execute it. The pipeline is: + +1. Parse the protobuf bytes into a protobuf `io.substrait.proto.Plan`. +2. Convert that to an `io.substrait.plan.Plan` POJO with core's `ProtoPlanConverter` — passing the + **Spark extension collection**. +3. Rebuild a Spark `LogicalPlan` with `ToLogicalPlan`. +4. Execute it with `Dataset.ofRows`. + +## Step 1: parse the protobuf bytes + +```java +byte[] buffer = Files.readAllBytes(Paths.get("spark_substrait.plan")); +io.substrait.proto.Plan proto = io.substrait.proto.Plan.parseFrom(buffer); +``` + +!!! tip "Name your variables for the two `Plan` types" + There are two distinct `Plan` classes in play: the protobuf message + `io.substrait.proto.Plan` and the high-level POJO `io.substrait.plan.Plan`. Using fully + qualified names (or clearly named variables) keeps the two directions of conversion readable. + +## Step 2: convert to a Substrait `Plan` POJO + +`ProtoPlanConverter` turns the protobuf message back into the `io.substrait.plan.Plan` POJO. When +the plan was produced by Spark, deserialize it with `SparkExtension.COLLECTION`: + +```java +import io.substrait.plan.Plan; +import io.substrait.plan.ProtoPlanConverter; +import io.substrait.spark.SparkExtension; + +Plan plan = new ProtoPlanConverter(SparkExtension.COLLECTION()).from(proto); +``` + +!!! warning "Pass `SparkExtension.COLLECTION` for Spark-produced plans" + `SparkExtension.COLLECTION` is the standard Substrait function extensions **merged with the + Spark-specific extensions** the producer may have used (for example `date_add`, or the + aggregate/window functions declared in the Spark dialect). The default no-argument + `ProtoPlanConverter()` only knows the standard extensions, so it will fail to resolve any + Spark-specific function reference in the plan. Constructing the converter with + `SparkExtension.COLLECTION()` ensures every function the plan references can be looked up. + + `SparkExtension` is a Scala `object`; from Java its `COLLECTION` value is reached as the static + method call `SparkExtension.COLLECTION()`. + +## Step 3: rebuild the Spark logical plan + +`ToLogicalPlan` is constructed with the target `SparkSession` and converts the `Plan` POJO into a +Catalyst `LogicalPlan`. It needs the live session to resolve `NamedScan` table references against +the catalog and to build file-backed relations. + +```java +import io.substrait.spark.logical.ToLogicalPlan; + +ToLogicalPlan toSpark = new ToLogicalPlan(spark); +LogicalPlan sparkPlan = toSpark.convert(plan); +``` + +`convert(Plan)` also reapplies the plan's root output names, adding a final projection (with casts +if needed) so the executed plan's schema matches the names carried in the Substrait root. The +resulting plan is fully resolved and ready to run. (`ToLogicalPlan` also has a +`convert(io.substrait.relation.Rel)` overload for a bare relation tree without root names.) + +## Step 4: execute + +Hand the rebuilt logical plan to Spark for execution with `Dataset.ofRows`: + +```java +Dataset.ofRows(spark, sparkPlan).show(); +``` + +!!! warning "Spark 4.0 package" + On Spark 4.0, `Dataset` (and the `SparkSession` you pass) come from + `org.apache.spark.sql.classic`, whereas on Spark 3.4/3.5 they come from `org.apache.spark.sql`. + See the [Spark 4.0 package caveat](compatibility.md#spark-40-package-caveat). + +## Where the data comes from + +What the plan reads from depends on which read relation the producer emitted: + +- **`LocalFiles`** — the plan carries concrete file URIs (e.g. + `file:///opt/spark-data/tests.csv`) plus format and read options. The consuming engine reads + those files directly, so the paths must be valid where the plan runs. +- **`NamedScan`** — the plan carries a table name such as `[spark_catalog, default, vehicles]` and + the expected schema, but no data location. The referenced table must already exist in the + consuming session's catalog, otherwise execution fails. + +This distinction matters most when moving plans between engines; see the +[end-to-end example](end-to-end.md#cross-engine-consumption) for how other engines handle each +case. diff --git a/docs/spark/end-to-end.md b/docs/spark/end-to-end.md new file mode 100644 index 000000000..6479a1ec4 --- /dev/null +++ b/docs/spark/end-to-end.md @@ -0,0 +1,149 @@ +# End-to-end example + +The repository ships a runnable example, [`examples/substrait-spark`](https://github.com/substrait-io/substrait-java/tree/main/examples/substrait-spark), +that walks the full loop: build a small Spark application, start a Spark cluster in Docker, produce +a Substrait `.plan` protobuf file from a query, then load that file back and execute it. This page +follows that example; see its `README.md` for the complete narrative and sample output. + +## What the example contains + +The application lives under `examples/substrait-spark/src/main/java/io/substrait/examples/` and has +three entry points (dispatched by `App` on the first command-line argument): + +| Class | Role | +| --- | --- | +| `SparkSQL` | Builds a plan from the **SQL** API, writes `spark_sql_substrait.plan` | +| `SparkDataset` | Builds a plan from the **DataFrame/Dataset** API, writes `spark_dataset_substrait.plan` | +| `SparkConsumeSubstrait` | **Loads** a `.plan` file and executes it on Spark | + +`SparkHelper` holds the shared constants — notably `ROOT_DIR = /opt/spark-data` (the in-container +data directory) and the `vehicles`/`tests` table and CSV names. Two CSV datasets are provided under +`src/main/resources/`. `SparkSQL` and `SparkDataset` both express the same query (count vehicles by +colour that passed a safety test), and both convert `queryExecution().optimizedPlan()` — the +[optimized plan](producing-plans.md) — so they emit structurally identical Substrait. + +The example uses a [`just`](https://github.com/casey/just) task runner (`justfile`) to wrap the +Docker and `spark-submit` commands. + +## 1. Build the application + +`just buildapp` compiles the app and stages the artifacts the Docker cluster expects: it runs the +Gradle build, creates `_apps/` and `_data/`, copies the built jar to `_apps/app.jar`, and copies +the CSV datasets into `_data/`. + +```bash +cd examples/substrait-spark +just buildapp +``` + +!!! note "Why `_data` is group-writable" + `buildapp` runs `chmod g+w _data` so that the Spark process inside the container (running as a + different user) can **write the output `.plan` file** back into the mounted directory. `_data` + is mounted into the containers at `/opt/spark-data` (which is `ROOT_DIR`), and `_apps` at + `/opt/spark-apps`. + +## 2. Start the Spark cluster + +`just spark` brings up a small Spark cluster (a master and one worker, using the `bitnami/spark` +image) via `docker compose`. Run it in its own terminal and leave it running. + +```bash +just spark +``` + +Under the hood this is `docker compose up`, with your uid/gid exported so the container can write to +the mounted `_data` volume. + +## 3. Produce a `.plan` file + +With the cluster up, run either query. Each does `spark-submit` of `app.jar` inside the master +container and writes its `.plan` file into `_data/`. + +=== "SQL" + + ```bash + just sql # writes _data/spark_sql_substrait.plan + ``` + +=== "DataFrame / Dataset" + + ```bash + just dataset # writes _data/spark_dataset_substrait.plan + ``` + +Internally the example converts and serializes exactly as described in +[Producing plans](producing-plans.md): + +```java +ToSubstraitRel toSubstrait = new ToSubstraitRel(); +io.substrait.plan.Plan plan = toSubstrait.convert(optimised); + +byte[] buffer = new PlanProtoConverter().toProto(plan).toByteArray(); +Files.write(Paths.get(ROOT_DIR, "spark_sql_substrait.plan"), buffer); +``` + +Both entry points also print a human-readable rendering of the plan using the example's +`SubstraitStringify` utility — a good illustration of walking the Substrait POJO model with the +visitor pattern. + +## 4. Load and execute the plan + +`just consume ` runs `SparkConsumeSubstrait`, which reads the `.plan` file from `_data/`, +rebuilds the Spark logical plan, and executes it — producing the same result table as the original +query. + +```bash +just consume spark_sql_substrait.plan +``` + +The consuming code follows [Consuming plans](consuming-plans.md): + +```java +byte[] buffer = Files.readAllBytes(Paths.get(ROOT_DIR, arg)); +io.substrait.proto.Plan proto = io.substrait.proto.Plan.parseFrom(buffer); + +Plan plan = new ProtoPlanConverter(SparkExtension.COLLECTION()).from(proto); + +ToLogicalPlan toSpark = new ToLogicalPlan(spark); +LogicalPlan sparkPlan = toSpark.convert(plan); + +Dataset.ofRows(spark, sparkPlan).show(); +``` + +!!! warning "Spark 4.0 imports" + In the example, `SparkConsumeSubstrait` and `SparkHelper` import `Dataset` and `SparkSession` + from `org.apache.spark.sql.classic` — the Spark 4.0 location. On Spark 3.4/3.5 these come from + `org.apache.spark.sql`. See the [Spark 4.0 package caveat](compatibility.md#spark-40-package-caveat). + +Run `just` with no arguments to list all recipes (`buildapp`, `spark`, `sql`, `dataset`, +`consume`). + +## Cross-engine consumption + +Because the serialized plan is engine-neutral protobuf, engines other than Spark can consume it — +but what they need from a plan differs, and the example README explores this. The behavior depends +on which read relation Spark emitted: + +- **DuckDB** (`connection.from_substrait(plan_bytes)`) is happy to load files itself, but the + `LocalFiles` URIs are coupled to the machine that produced the plan — so it fails if the file + paths do not exist locally. For a `NamedScan`, DuckDB cannot index its catalog with Spark's + three-part name such as `spark_catalog.default.vehicles`. +- **PyArrow** (`substrait.run_query(plan_bytes, table_provider=...)`) delegates data loading back + to the caller via a `table_provider` callback, which receives the requested table names and + expected schema. It rejects Spark's non-default `LocalFiles` length field, so the `NamedScan` + (table-reference) form is the better fit there. + +```python +# PyArrow: the caller resolves table names to datasets +def table_provider(self, names, schema): + if names[-1] == "vehicles": + return self.vehicles + elif names[-1] == "tests": + return self.tests + raise Exception(f"Unrecognized table name {names}") +``` + +The takeaway: transferring plans between engines is powerful, but the consuming engine must share an +understanding of how source data is referenced — file URIs versus catalog names. See +[Consuming plans](consuming-plans.md#where-the-data-comes-from) for how Spark itself handles each +read type. diff --git a/docs/spark/index.md b/docs/spark/index.md new file mode 100644 index 000000000..d6c0bf86e --- /dev/null +++ b/docs/spark/index.md @@ -0,0 +1,74 @@ +# Spark + +The `spark` module bridges [Apache Spark](https://spark.apache.org/) and Substrait. It converts a +Spark **logical plan** into a Substrait [`Plan`](../core/building-plans.md) and, in the other +direction, rebuilds a Spark logical plan from a Substrait plan so that a Spark session can execute +it. This lets a query authored in one Spark cluster be serialized as an engine-neutral Substrait +plan, moved elsewhere, and run — or handed to a different engine entirely. + +The module is written in Scala and published for several Spark/Scala combinations. Because the +public API is small and mostly plain method calls, it is comfortable to drive from Java as well; +the examples in these pages are Java calling the Scala API. + +## The two entry points + +Everything centres on two classes in the `io.substrait.spark.logical` package: + +| Direction | Class | Key method | +| --- | --- | --- | +| Spark → Substrait | `ToSubstraitRel` | `convert(LogicalPlan): io.substrait.plan.Plan` | +| Substrait → Spark | `ToLogicalPlan` | `convert(io.substrait.plan.Plan): LogicalPlan` | + +### `ToSubstraitRel` (Spark → Substrait) + +`ToSubstraitRel` is a visitor over Spark's Catalyst `LogicalPlan` tree. Its `convert` method walks +the plan and produces an `io.substrait.plan.Plan` POJO, tagging the plan with the producer name +`substrait-spark`. From there you serialize to the canonical protobuf wire format with core's +`PlanProtoConverter`. See [Producing plans](producing-plans.md). + +```java +ToSubstraitRel toSubstrait = new ToSubstraitRel(); +io.substrait.plan.Plan plan = toSubstrait.convert(optimizedPlan); +``` + +### `ToLogicalPlan` (Substrait → Spark) + +`ToLogicalPlan` is a Substrait `RelVisitor` that rebuilds a Catalyst `LogicalPlan`. It is +constructed with an active `SparkSession` (it needs the session to resolve `NamedScan` tables and +build file relations), and its `convert` method accepts either a Substrait `Plan` or a bare `Rel`. +See [Consuming plans](consuming-plans.md). + +```java +ToLogicalPlan toSpark = new ToLogicalPlan(spark); +LogicalPlan sparkPlan = toSpark.convert(plan); +``` + +## Key convention: convert the optimized plan + +!!! warning "Convert the optimized plan, not the raw logical plan" + Always feed `ToSubstraitRel` the plan returned by + `queryExecution().optimizedPlan()`, not `queryExecution().logical()`. + + Spark's raw logical plan still contains constructs that have no Substrait counterpart — + `SubqueryAlias`, `View`, unresolved references, and Spark-internal rewrites. The Catalyst + optimizer lowers these into the small set of relations and expressions the converter + understands (projects, filters, joins, aggregates, and so on), so the optimized plan is the + reliable starting point. This holds whether the query originated from SQL or the + DataFrame/Dataset API — both produce the same optimized plan. + +## Pages in this section + +- [Compatibility & dependencies](compatibility.md) — the per-variant artifacts (there is no single + cross-version jar) and how to depend on the one matching your Spark and Scala runtime. +- [Producing plans](producing-plans.md) — turn a Spark SQL or DataFrame query into a Substrait + plan and serialize it. +- [Consuming plans](consuming-plans.md) — deserialize a Substrait plan and execute it on Spark. +- [Supported features](supported-features.md) — the types, expressions, relations, and functions + the converters understand. +- [End-to-end example](end-to-end.md) — the runnable `substrait-spark` example: build, run a Spark + cluster in Docker, produce a `.plan` file, then load and execute it. + +!!! note "API reference" + These pages cover the entry points and the common workflow. For the full API — every visitor, + type mapping, and helper — see the published Javadoc for the variant you depend on, e.g. + [`spark35_2.12`](https://javadoc.io/doc/io.substrait/spark35_2.12). diff --git a/docs/spark/producing-plans.md b/docs/spark/producing-plans.md new file mode 100644 index 000000000..813d83bfc --- /dev/null +++ b/docs/spark/producing-plans.md @@ -0,0 +1,109 @@ +# Producing plans + +Producing a Substrait plan from Spark is a three-step pipeline: + +1. Get Spark to build the query's **optimized** logical plan. +2. Convert that plan to an `io.substrait.plan.Plan` with `ToSubstraitRel`. +3. Serialize the plan to protobuf bytes with core's `PlanProtoConverter`. + +The query can start from either the SQL API or the DataFrame/Dataset API — both funnel into the +same optimized plan, so the conversion code is identical from step 2 onward. + +## Step 1: get the optimized logical plan + +!!! warning "Use `optimizedPlan()`" + `ToSubstraitRel` expects the **optimized** logical plan + (`queryExecution().optimizedPlan()`). The raw logical plan still contains `SubqueryAlias`, + `View`, and unresolved nodes that the converter cannot translate; the optimizer rewrites these + into the relations and expressions Substrait understands. See the + [overview](index.md#key-convention-convert-the-optimized-plan). + +=== "SQL API" + + ```java + // A DataFrame from a SQL string; tables/views must already be registered + Dataset result = spark.sql( + "SELECT vehicles.colour, count(*) AS colourcount" + + " FROM vehicles" + + " INNER JOIN tests ON vehicles.vehicle_id = tests.vehicle_id" + + " WHERE tests.test_result = 'P'" + + " GROUP BY vehicles.colour" + + " ORDER BY count(*)"); + + LogicalPlan optimised = result.queryExecution().optimizedPlan(); + ``` + +=== "DataFrame / Dataset API" + + ```java + Dataset joined = + dsVehicles + .join(dsTests, dsVehicles.col("vehicle_id").equalTo(dsTests.col("vehicle_id"))) + .filter(dsTests.col("test_result").equalTo("P")) + .groupBy(dsVehicles.col("colour")) + .count() + .orderBy("count"); + + LogicalPlan optimised = joined.queryExecution().optimizedPlan(); + ``` + +Structurally the two optimized plans are identical, so the Substrait plan produced from each is the +same. + +## Step 2: convert to a Substrait `Plan` + +`ToSubstraitRel.convert` walks the Catalyst plan and returns an `io.substrait.plan.Plan` POJO. The +plan is stamped with the producer name `substrait-spark`. + +```java +import io.substrait.spark.logical.ToSubstraitRel; + +ToSubstraitRel toSubstrait = new ToSubstraitRel(); +io.substrait.plan.Plan plan = toSubstrait.convert(optimised); +``` + +`io.substrait.plan.Plan` is a high-level, immutable POJO. You can inspect or transform it in memory, +but most often you will serialize it. + +!!! tip "Truncating in-memory (RDD) sources" + When the plan reads from an in-memory `LogicalRDD` (for example a DataFrame created from a local + collection), `ToSubstraitRel` captures the rows as a Substrait `VirtualTableScan`. To keep plans + bounded it takes at most `rddLimit` rows (default `100`) and logs a warning if there are more. + Adjust it before converting: + + ```java + ToSubstraitRel toSubstrait = new ToSubstraitRel(); + toSubstrait.rddLimit_$eq(1000); // Scala setter, seen from Java + ``` + +## Step 3: serialize to protobuf + +The canonical Substrait serialization is protobuf. Core's `PlanProtoConverter` turns the POJO plan +into a protobuf `io.substrait.proto.Plan`, from which you get the wire bytes: + +```java +import io.substrait.plan.PlanProtoConverter; + +byte[] buffer = new PlanProtoConverter().toProto(plan).toByteArray(); + +// e.g. persist the plan to a file +Files.write(Paths.get("spark_substrait.plan"), buffer); +``` + +Those bytes are the portable intermediate representation: store them, ship them to another engine, +or reload them into Spark. See [Serialization](../core/serialization.md) for the full round-trip +details and [Consuming plans](consuming-plans.md) for the reverse direction. + +!!! note "Shortcut: `toProtoSubstrait`" + `ToSubstraitRel` also exposes `toProtoSubstrait(LogicalPlan): byte[]`, which performs the + convert-and-serialize in one call. It emits a bare relation tree (via `RelProtoConverter`) + rather than a full `Plan` with root output names, so `convert` followed by `PlanProtoConverter` + is preferred when you need the complete plan — for example to round-trip through + [`ToLogicalPlan`](consuming-plans.md). + +## Not everything converts + +The converter supports the common relations, expressions, and functions — enough that every TPC-H +query round-trips — but it is not exhaustive. Unsupported nodes raise +`UnsupportedOperationException` (for example union-by-name, or a file format other than +CSV/Parquet/ORC). See [Supported features](supported-features.md) for the full list. diff --git a/docs/spark/supported-features.md b/docs/spark/supported-features.md new file mode 100644 index 000000000..56826f2b7 --- /dev/null +++ b/docs/spark/supported-features.md @@ -0,0 +1,113 @@ +# Supported features + +The converters translate the common core of Spark's logical model — enough that **every TPC-H +query round-trips** through Substrait and back. Coverage is not exhaustive, though: **TPC-DS has +known gaps** that prevent some of its queries from being translated, and any unsupported node, +type, or function raises `UnsupportedOperationException` (or a resolution error) rather than being +silently dropped. + +This page summarizes what is supported. The type and function mappings are declared in the Spark +dialect (`spark/spark_dialect.yaml`), and the Spark ⇄ Substrait function signatures are wired up in +`FunctionMappings`. + +## Types + +Substrait types map to Spark SQL types as follows (both directions): + +| Substrait type | Spark type | +| --- | --- | +| `I8` | `ByteType` | +| `I16` | `ShortType` | +| `I32` | `IntegerType` | +| `I64` | `LongType` | +| `FP32` | `FloatType` | +| `FP64` | `DoubleType` | +| `DECIMAL` | `DecimalType` | +| `DATE` | `DateType` | +| `STRING` | `StringType` | +| `VARCHAR` | `StringType` | +| `FIXED_CHAR` | `StringType` | +| `BINARY` | `BinaryType` | +| `BOOL` | `BooleanType` | +| `PRECISION_TIMESTAMP` (max precision 9) | `TimestampNTZType` | +| `PRECISION_TIMESTAMP_TZ` (max precision 9) | `TimestampType` | +| `INTERVAL_DAY` (max precision 9) | `DayTimeIntervalType` | +| `INTERVAL_YEAR` | `YearMonthIntervalType` | +| `LIST` | `ArrayType` | +| `MAP` | `MapType` | +| `STRUCT` | `StructType` | + +## Expressions + +- **Literals** +- **Field references** (selections) +- **Scalar functions** (see [Functions](#functions)) +- **`IF`/`CASE`** (if-then) +- **`IN` lists** (singular-or-list) +- **Casts** +- **Subqueries** — scalar subqueries and `IN` predicate (semi-join) subqueries + +## Relations + +- **Project** +- **Filter** +- **Aggregate** — a single grouping set; `Rollup`/`GroupingSets` are not yet supported. A `Project` + is layered on top so grouping keys and aggregate results can be reordered and combined with + scalar expressions. +- **Sort** +- **Fetch** — `LIMIT` and `OFFSET` (and their combination) +- **Join** — `INNER`, `LEFT`/`RIGHT` outer, `OUTER` (full), `LEFT_SEMI`, `LEFT_ANTI`. Spark's + internal `ExistenceJoin` is modelled with a Substrait `InPredicate` inside a filter. +- **Cross** — a cross/Cartesian join (an inner join with a trivially-true condition is emitted as a + `Cross`) +- **Set** — `UNION ALL` (union-by-name is not supported) +- **Window** — consistent-partition window functions (see [Functions](#functions)) +- **Expand** — for multi-projection expansions (switching-field form) +- **Read** — virtual tables (in-memory/`LocalRelation`), local files (CSV, Parquet, ORC), and + named tables (catalog scans) +- **Write** — insert into files or Hive tables, and CTAS (create-table-as-select) +- **DDL** — `CREATE TABLE` / `DROP TABLE` on named objects + +!!! note "File formats" + File-backed reads and writes support **CSV, Parquet, and ORC**. CSV carries its delimiter, + quote, escape, header, and null-value options through the plan. Other formats raise + `UnsupportedOperationException`. + +## Functions + +Function names below are the Substrait function names; each maps to the corresponding Spark +Catalyst expression. + +### Scalar functions + +- **Arithmetic:** `add`, `subtract`, `multiply`, `divide`, `abs`, `modulus`, `power`, `exp`, + `sqrt`, `sin`, `cos`, `tan`, `asin`, `acos`, `atan`, `atan2`, `sinh`, `cosh`, `tanh`, `asinh`, + `acosh`, `atanh` +- **Logarithmic:** `ln`, `log10` +- **Rounding:** `round`, `floor`, `ceil` +- **Boolean:** `and`, `or`, `not` +- **Comparison:** `equal`, `is_not_distinct_from`, `lt`, `lte`, `gt`, `gte`, `is_null`, + `is_not_null` +- **String:** `substring`, `upper`, `lower`, `lpad`, `rpad`, `concat`, `like`, `contains`, + `starts_with`, `ends_with`, `trim`, `ltrim`, `rtrim` +- **Null handling:** `coalesce` +- **Bitwise:** `bitwise_and`, `bitwise_or`, `bitwise_xor`, `shift_left`, `shift_right`, + `shift_right_unsigned` +- **Date/time:** `date_add` (Spark extension), and `extract` — Spark's `Year`, `Quarter`, `Month`, + and `DayOfMonth` map to the Substrait `extract` function with the appropriate enum argument + +### Aggregate functions + +`sum`, `avg`, `count`, `min`, `max`, `any_value` (Spark `First`), `approx_count_distinct` +(HyperLogLog++), and `std_dev` (sample standard deviation). + +### Window functions + +`row_number`, `rank`, `dense_rank`, `percent_rank`, `cume_dist`, `ntile`, `lead`, `lag`, and +`nth_value`. + +## Benchmark coverage + +- **TPC-H:** all queries round-trip Spark → Substrait → Spark. +- **TPC-DS:** most queries work, but there are **known gaps** — some queries use constructs + (such as rollups/grouping sets) that are not yet supported and therefore do not translate. diff --git a/isthmus-cli/README.md b/isthmus-cli/README.md index c5af2b2f7..99b11797b 100644 --- a/isthmus-cli/README.md +++ b/isthmus-cli/README.md @@ -57,15 +57,15 @@ Convert SQL Queries and SQL Expressions to Substrait "SELECT lastName, firstName FROM Persons WHERE zip = 90210" { - "extensionUris": [{ - "extensionUriAnchor": 1, - "uri": "/functions_comparison.yaml" + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" }], "extensions": [{ "extensionFunction": { - "extensionUriReference": 1, "functionAnchor": 0, - "name": "equal:any1_any1" + "name": "equal:any1_any1", + "extensionUrnReference": 1 } }], "relations": [{ @@ -187,15 +187,15 @@ $ ./isthmus-cli/build/native/nativeCompile/isthmus -c "CREATE TABLE NATION (N_NA -e "N_REGIONKEY + 10" { - "extensionUris": [{ - "extensionUriAnchor": 1, - "uri": "/functions_arithmetic.yaml" + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_arithmetic" }], "extensions": [{ "extensionFunction": { - "extensionUriReference": 1, "functionAnchor": 0, - "name": "add:i64_i64" + "name": "add:i64_i64", + "extensionUrnReference": 1 } }], "referredExpr": [{ @@ -287,15 +287,15 @@ $ ./isthmus-cli/build/native/nativeCompile/isthmus -c "CREATE TABLE NATION (N_NA -e "N_REGIONKEY > 10" { - "extensionUris": [{ - "extensionUriAnchor": 1, - "uri": "/functions_comparison.yaml" + "extensionUrns": [{ + "extensionUrnAnchor": 1, + "urn": "extension:io.substrait:functions_comparison" }], "extensions": [{ "extensionFunction": { - "extensionUriReference": 1, "functionAnchor": 0, - "name": "gt:any_any" + "name": "gt:any_any", + "extensionUrnReference": 1 } }], "referredExpr": [{ diff --git a/pixi.lock b/pixi.lock new file mode 100644 index 000000000..3f561fec6 --- /dev/null +++ b/pixi.lock @@ -0,0 +1,1381 @@ +version: 6 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + options: + pypi-prerelease-mode: if-necessary-or-explicit + packages: {} + docs: + channels: + - url: https://conda.anaconda.org/conda-forge/ + indexes: + - https://pypi.org/simple + options: + pypi-prerelease-mode: if-necessary-or-explicit + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_9.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.6.17-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45.1-default_hbd61a6d_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.8.1-hecca717_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h3435931_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_19.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_19.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_19.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.3-hb03c661_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.53.3-h0c1763c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.42.2-h5347b49_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.2-h25fd6f3_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.3-h35e630c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h366c992_103.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/51/25/2a75b47cb057b1e164c604fb81ab690a6cdb5e2260ce651194eae90f64a3/deepmerge-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: git+https://github.com/squidfunk/mike.git#2d4ad799442f4592db8ad53b179bfb33db8c69ac + - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a4/ce/3b6fee91c85626eaf769d617f1be9d2e15c1cca027bbdeb2e0d751469355/verspec-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/33/9b/1f5065c664afde6a63e0ada8c14eae75f9e6960df597a5ca28c07340f938/zensical-0.0.50-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + linux-aarch64: + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_9.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.6.17-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-78.3-hcab7f73_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45.1-default_h1979696_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.8.1-hfae3067_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-h376a255_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_19.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_19.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_19.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.3-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.1-h86ecc28_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.53.3-h10b116e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-hef695bb_19.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.42.2-h1022ec0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.2-hdc9db2a_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.6-hf8d1292_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.6.3-h546c87b_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.12.13-h91f4b29_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.3-hb682ff5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h0dc03b3_103.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-h85ac4a6_6.conda + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/51/25/2a75b47cb057b1e164c604fb81ab690a6cdb5e2260ce651194eae90f64a3/deepmerge-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: git+https://github.com/squidfunk/mike.git#2d4ad799442f4592db8ad53b179bfb33db8c69ac + - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/a4/ce/3b6fee91c85626eaf769d617f1be9d2e15c1cca027bbdeb2e0d751469355/verspec-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/38/f5/16c087635680efb39e1d7b7aa3c2cca548305aa29ad51dee04f37b32b65f/zensical-0.0.50-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + osx-64: + - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_9.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.6.17-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.3-h25d91c4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.8.1-hcc62823_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-hd1f9c09_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/liblzma-5.8.3-hbb4bfdb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.53.3-h8f8c405_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.2-hbb4bfdb_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.6-hcc0dc9a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.6.3-hc881268_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.12.13-ha9537fe_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h7142dee_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/51/25/2a75b47cb057b1e164c604fb81ab690a6cdb5e2260ce651194eae90f64a3/deepmerge-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl + - pypi: git+https://github.com/squidfunk/mike.git#2d4ad799442f4592db8ad53b179bfb33db8c69ac + - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a4/ce/3b6fee91c85626eaf769d617f1be9d2e15c1cca027bbdeb2e0d751469355/verspec-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c2/d2/e126d56642a5fd437eeb5a067b7c0d1f766c08e22d6a708bb923986f1d44/zensical-0.0.50-cp310-abi3-macosx_10_12_x86_64.whl + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_9.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.6.17-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.8.1-hf6b4638_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-hcf2aa1b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.3-h8088a28_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.53.3-h1b79a29_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.6-h1d4f5a5_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.3-hd24854e_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-h8561d8f_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h010d191_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/51/25/2a75b47cb057b1e164c604fb81ab690a6cdb5e2260ce651194eae90f64a3/deepmerge-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl + - pypi: git+https://github.com/squidfunk/mike.git#2d4ad799442f4592db8ad53b179bfb33db8c69ac + - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a4/ce/3b6fee91c85626eaf769d617f1be9d2e15c1cca027bbdeb2e0d751469355/verspec-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/bd/9b/e1a22fb4ea8fc3e8377d7b394ee06cf0349e7f9a64ecb4c9873dee79b102/zensical-0.0.50-cp310-abi3-macosx_11_0_arm64.whl + win-64: + - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_9.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.6.17-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.8.1-hac47afa_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h3d046cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.3-hfd05255_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.53.3-hf5d6505_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.2-hfd05255_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.3-hf411b9b_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.12.13-h0159041_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h6ed50ae_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.5-h1b7c187_39.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.51.36231-h1b9f54f_39.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.51.36231-h1b9f54f_39.conda + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/51/25/2a75b47cb057b1e164c604fb81ab690a6cdb5e2260ce651194eae90f64a3/deepmerge-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl + - pypi: git+https://github.com/squidfunk/mike.git#2d4ad799442f4592db8ad53b179bfb33db8c69ac + - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/a4/ce/3b6fee91c85626eaf769d617f1be9d2e15c1cca027bbdeb2e0d751469355/verspec-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e3/e1/4babb30544d7465c95a6a93abc0680b1c51752411a225c2b7f2cb44e80c7/zensical-0.0.50-cp310-abi3-win_amd64.whl +packages: +- conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda + build_number: 20 + sha256: 1dd3fffd892081df9726d7eb7e0dea6198962ba775bd88842135a4ddb4deb3c9 + md5: a9f577daf3de00bca7c3c76c0ecbd1de + depends: + - __glibc >=2.17,<3.0.a0 + - libgomp >=7.5.0 + constrains: + - openmp_impl <0.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 28948 + timestamp: 1770939786096 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda + build_number: 20 + sha256: a2527b1d81792a0ccd2c05850960df119c2b6d8f5fdec97f2db7d25dc23b1068 + md5: 468fd3bb9e1f671d36c2cbc677e56f1d + depends: + - libgomp >=7.5.0 + constrains: + - openmp_impl <0.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 28926 + timestamp: 1770939656741 +- conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_9.conda + sha256: 0b75d45f0bba3e95dc693336fa51f40ea28c980131fec438afb7ce6118ed05f6 + md5: d2ffd7602c02f2b316fd921d39876885 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + size: 260182 + timestamp: 1771350215188 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_9.conda + sha256: b3495077889dde6bb370938e7db82be545c73e8589696ad0843a32221520ad4c + md5: 840d8fc0d7b3209be93080bc20e07f2d + depends: + - libgcc >=14 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + size: 192412 + timestamp: 1771350241232 +- conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_9.conda + sha256: 9f242f13537ef1ce195f93f0cc162965d6cc79da578568d6d8e50f70dd025c42 + md5: 4173ac3b19ec0a4f400b4f782910368b + depends: + - __osx >=10.13 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + size: 133427 + timestamp: 1771350680709 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_9.conda + sha256: 540fe54be35fac0c17feefbdc3e29725cce05d7367ffedfaaa1bdda234b019df + md5: 620b85a3f45526a8bc4d23fd78fc22f0 + depends: + - __osx >=11.0 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + size: 124834 + timestamp: 1771350416561 +- conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_9.conda + sha256: 76dfb71df5e8d1c4eded2dbb5ba15bb8fb2e2b0fe42d94145d5eed4c75c35902 + md5: 4cb8e6b48f67de0b018719cdf1136306 + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + size: 56115 + timestamp: 1771350256444 +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.6.17-h4c7d964_0.conda + sha256: 7f458e4a82514d7bebbfef23d92817794a16aaf1c748a15f04870d4fb49aeab2 + md5: b9696b2cf00dfeec138c70cee38ed192 + depends: + - __win + license: ISC + purls: [] + size: 129352 + timestamp: 1781709016515 +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.6.17-hbd8a1cb_0.conda + sha256: f8e3c730fa14ee3f170493779f06522c4acf89169f43db4f039727709b6419cf + md5: a9965dd99f683c5f444428f896635716 + depends: + - __unix + license: ISC + purls: [] + size: 128866 + timestamp: 1781708962055 +- pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + name: click + version: 8.4.2 + sha256: e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + requires_dist: + - colorama ; sys_platform == 'win32' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl + name: colorama + version: 0.4.6 + sha256: 4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*' +- pypi: https://files.pythonhosted.org/packages/51/25/2a75b47cb057b1e164c604fb81ab690a6cdb5e2260ce651194eae90f64a3/deepmerge-2.1.0-py3-none-any.whl + name: deepmerge + version: 2.1.0 + sha256: 8f148339a91d680a75ecb74ade235d9e759a93df373a0b04e9d31c8666cfeb75 + requires_dist: + - typing-extensions ; python_full_version < '3.10' + - validate-pyproject[all] ; extra == 'dev' + - pyupgrade ; extra == 'dev' + - black ; extra == 'dev' + - mypy ; extra == 'dev' + - pytest ; extra == 'dev' + - build ; extra == 'dev' + - twine ; extra == 'dev' + - sphinx ; extra == 'dev' + - sphinx-rtd-theme ; extra == 'dev' + requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-78.3-hcab7f73_0.conda + sha256: 49ba6aed2c6b482bb0ba41078057555d29764299bc947b990708617712ef6406 + md5: 546da38c2fa9efacf203e2ad3f987c59 + depends: + - libgcc >=14 + - libstdcxx >=14 + license: MIT + license_family: MIT + purls: [] + size: 12837286 + timestamp: 1773822650615 +- conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.3-h25d91c4_0.conda + sha256: 1294117122d55246bb83ad5b589e2a031aacdf2d0b1f99fd338aa4394f881735 + md5: 627eca44e62e2b665eeec57a984a7f00 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + size: 12273764 + timestamp: 1773822733780 +- pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl + name: jinja2 + version: 3.1.6 + sha256: 85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + requires_dist: + - markupsafe>=2.0 + - babel>=2.7 ; extra == 'i18n' + requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45.1-default_hbd61a6d_102.conda + sha256: 3d584956604909ff5df353767f3a2a2f60e07d070b328d109f30ac40cd62df6c + md5: 18335a698559cdbcd86150a48bf54ba6 + depends: + - __glibc >=2.17,<3.0.a0 + - zstd >=1.5.7,<1.6.0a0 + constrains: + - binutils_impl_linux-64 2.45.1 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 728002 + timestamp: 1774197446916 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45.1-default_h1979696_102.conda + sha256: 7abd913d81a9bf00abb699e8987966baa2065f5132e37e815f92d90fc6bba530 + md5: a21644fc4a83da26452a718dc9468d5f + depends: + - zstd >=1.5.7,<1.6.0a0 + constrains: + - binutils_impl_linux-aarch64 2.45.1 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 875596 + timestamp: 1774197520746 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.8.1-hecca717_1.conda + sha256: 16feffd9ddbbe5b718515d38ee376c685ba95491cd901244e24671d20b952a77 + md5: b24d3c612f71e7aa74158d92106318b2 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - expat 2.8.1.* + license: MIT + license_family: MIT + purls: [] + size: 77856 + timestamp: 1781203599810 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.8.1-hfae3067_1.conda + sha256: 20a5726bc8705d91437c9e6ef83b30da64a1719b869656d20a1ee818333ea5ac + md5: fac3b65a605cd253037fdf3daf2de8d9 + depends: + - libgcc >=14 + constrains: + - expat 2.8.1.* + license: MIT + license_family: MIT + purls: [] + size: 77649 + timestamp: 1781203572523 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.8.1-hcc62823_1.conda + sha256: 9c96cc05e056e1bba5b545cbbd57b6e01db622dc2c82934caaaa25cfb22fe666 + md5: dcfdea7b7013beef0a4d744d776ea38f + depends: + - __osx >=11.0 + constrains: + - expat 2.8.1.* + license: MIT + license_family: MIT + purls: [] + size: 76020 + timestamp: 1781204303305 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.8.1-hf6b4638_1.conda + sha256: 5af74261101e3c777399c6294b2b5d290e508153268eb2e9ff99c4d69834612f + md5: a915151d5d3c5bf039f5ccc8402a436f + depends: + - __osx >=11.0 + constrains: + - expat 2.8.1.* + license: MIT + license_family: MIT + purls: [] + size: 69362 + timestamp: 1781203631990 +- conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.8.1-hac47afa_1.conda + sha256: 1a54d874addda73b6f7164d5f3905821277a1831bcc05edd74b3085391688571 + md5: ccc490c81ffe14181861beac0e8f3169 + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + constrains: + - expat 2.8.1.* + license: MIT + license_family: MIT + purls: [] + size: 71631 + timestamp: 1781203724164 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h3435931_0.conda + sha256: 31f19b6a88ce40ebc0d5a992c131f57d919f73c0b92cd1617a5bec83f6e961e6 + md5: a360c33a5abe61c07959e449fa1453eb + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: MIT + license_family: MIT + purls: [] + size: 58592 + timestamp: 1769456073053 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-h376a255_0.conda + sha256: 3df4c539449aabc3443bbe8c492c01d401eea894603087fca2917aa4e1c2dea9 + md5: 2f364feefb6a7c00423e80dcb12db62a + depends: + - libgcc >=14 + license: MIT + license_family: MIT + purls: [] + size: 55952 + timestamp: 1769456078358 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-hd1f9c09_0.conda + sha256: 951958d1792238006fdc6fce7f71f1b559534743b26cc1333497d46e5903a2d6 + md5: 66a0dc7464927d0853b590b6f53ba3ea + depends: + - __osx >=10.13 + license: MIT + license_family: MIT + purls: [] + size: 53583 + timestamp: 1769456300951 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-hcf2aa1b_0.conda + sha256: 6686a26466a527585e6a75cc2a242bf4a3d97d6d6c86424a441677917f28bec7 + md5: 43c04d9cb46ef176bb2a4c77e324d599 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + size: 40979 + timestamp: 1769456747661 +- conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h3d046cb_0.conda + sha256: 59d01f2dfa8b77491b5888a5ab88ff4e1574c9359f7e229da254cdfe27ddc190 + md5: 720b39f5ec0610457b725eb3f396219a + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: MIT + license_family: MIT + purls: [] + size: 45831 + timestamp: 1769456418774 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_19.conda + sha256: 8e0a3b5e41272e5678499b5dfc4cddb673f9e935de01eb0767ce857001229f46 + md5: 57736f29cc2b0ec0b6c2952d3f101b6a + depends: + - __glibc >=2.17,<3.0.a0 + - _openmp_mutex >=4.5 + constrains: + - libgcc-ng ==15.2.0=*_19 + - libgomp 15.2.0 he0feb66_19 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 1041084 + timestamp: 1778269013026 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_19.conda + sha256: 4592b096e553f67799ae70d4b6167eeda3ec74587d68c7aecbf4e7b1df136681 + md5: f35b3f52d0a2ec4ffe3c89ba135cdb9a + depends: + - _openmp_mutex >=4.5 + constrains: + - libgomp 15.2.0 h8acb6b2_19 + - libgcc-ng ==15.2.0=*_19 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 622462 + timestamp: 1778268755949 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_19.conda + sha256: 9dcf54adfaa5e861123c2da4f2f0451a685464ea7e5a41ad91cf67b31d658d98 + md5: 331ee9b72b9dff570d56b1302c5ab37d + depends: + - libgcc 15.2.0 he0feb66_19 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 27694 + timestamp: 1778269016987 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_19.conda + sha256: 1137f93f477f56199ded24117430045a0c02cbe8b10031beac3b9ad2138539d3 + md5: 770cf892e5530f43e63cadc673e85653 + depends: + - libgcc 15.2.0 h8acb6b2_19 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 27738 + timestamp: 1778268759211 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_19.conda + sha256: 5abe4ab9d93f6c9757d654f1969ae2267d4505315c1f2f8fe705fd60af084f1b + md5: faac990cb7aedc7f3a2224f2c9b0c26c + depends: + - __glibc >=2.17,<3.0.a0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 603817 + timestamp: 1778268942614 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_19.conda + sha256: 2370ef0ffcbae5bede3c4bf136add4abc257245eb91f724c99bb4a43116c5a83 + md5: c5e8a379c4a2ec2aea4ba22758c001d9 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 587387 + timestamp: 1778268674393 +- conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.3-hb03c661_0.conda + sha256: ec30e52a3c1bf7d0425380a189d209a52baa03f22fb66dd3eb587acaa765bd6d + md5: b88d90cad08e6bc8ad540cb310a761fb + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - xz 5.8.3.* + license: 0BSD + purls: [] + size: 113478 + timestamp: 1775825492909 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.3-he30d5cf_0.conda + sha256: d61962b9cd54c3554361550203c64d5b65b71e3058a285b66e4b04b9769f0a5c + md5: 76298a9e6d71ee6e832a8d0d7373b261 + depends: + - libgcc >=14 + constrains: + - xz 5.8.3.* + license: 0BSD + purls: [] + size: 126102 + timestamp: 1775828008518 +- conda: https://conda.anaconda.org/conda-forge/osx-64/liblzma-5.8.3-hbb4bfdb_0.conda + sha256: d9e2006051529aec5578c6efeb13bb6a7200a014b2d5a77a579e83a8049d5f3c + md5: becdfbfe7049fa248e52aa37a9df09e2 + depends: + - __osx >=11.0 + constrains: + - xz 5.8.3.* + license: 0BSD + purls: [] + size: 105724 + timestamp: 1775826029494 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.3-h8088a28_0.conda + sha256: 34878d87275c298f1a732c6806349125cebbf340d24c6c23727268184bba051e + md5: b1fd823b5ae54fbec272cea0811bd8a9 + depends: + - __osx >=11.0 + constrains: + - xz 5.8.3.* + license: 0BSD + purls: [] + size: 92472 + timestamp: 1775825802659 +- conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.3-hfd05255_0.conda + sha256: d636d1a25234063642f9c531a7bb58d84c1c496411280a36ea000bd122f078f1 + md5: 8f83619ab1588b98dd99c90b0bfc5c6d + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + constrains: + - xz 5.8.3.* + license: 0BSD + purls: [] + size: 106486 + timestamp: 1775825663227 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + sha256: 927fe72b054277cde6cb82597d0fcf6baf127dcbce2e0a9d8925a68f1265eef5 + md5: d864d34357c3b65a4b731f78c0801dc4 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: LGPL-2.1-only + license_family: GPL + purls: [] + size: 33731 + timestamp: 1750274110928 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.1-h86ecc28_1.conda + sha256: c0dc4d84198e3eef1f37321299e48e2754ca83fd12e6284754e3cb231357c3a5 + md5: d5d58b2dc3e57073fe22303f5fed4db7 + depends: + - libgcc >=13 + license: LGPL-2.1-only + license_family: GPL + purls: [] + size: 34831 + timestamp: 1750274211 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.53.3-h0c1763c_0.conda + sha256: 365376f4815e5e80def2b3462a2419708b7c292da0da85278386c2618621fff4 + md5: 4aed8e657e9ff156bdbe849b4df44389 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libzlib >=1.3.2,<2.0a0 + license: blessing + purls: [] + size: 962119 + timestamp: 1782519076616 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.53.3-h10b116e_0.conda + sha256: a835400072fb638fb582ee9fc2271169da84cbcad664d28b852610201116027e + md5: 2cd50877f494b34383af22560ced8b04 + depends: + - icu >=78.3,<79.0a0 + - libgcc >=14 + - libzlib >=1.3.2,<2.0a0 + license: blessing + purls: [] + size: 968420 + timestamp: 1782519054102 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.53.3-h8f8c405_0.conda + sha256: 84d4af77021ca28e02ff60f396575b921ed1747e459838daa0e73b4724b47953 + md5: 72d9eaef59342a3614c83d57a32f578b + depends: + - __osx >=11.0 + - icu >=78.3,<79.0a0 + - libzlib >=1.3.2,<2.0a0 + license: blessing + purls: [] + size: 1012648 + timestamp: 1782519619230 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.53.3-h1b79a29_0.conda + sha256: a73a8acd97a6599fd6e561514db9f101ca7fd984cdc0cfd91ba74c8aa9dbe067 + md5: 7184d95871a58b8258a8ea124ed5aabc + depends: + - __osx >=11.0 + - libzlib >=1.3.2,<2.0a0 + license: blessing + purls: [] + size: 924912 + timestamp: 1782519136322 +- conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.53.3-hf5d6505_0.conda + sha256: 692dfb73a22c873656d5e393b8f1e2b019a3c8a6486c97cb6900552e64e38c25 + md5: 051f1b2228e7517a2ef8cca5146c8967 + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: blessing + purls: [] + size: 1315909 + timestamp: 1782519131898 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-hef695bb_19.conda + sha256: 1dadc45e599f510dd5f97141dddcdbb9844d9f1430c1f3a38075cf1c58f87b4e + md5: 543fbc8d71f2a0baf04cf88ce96cb8bb + depends: + - libgcc 15.2.0 h8acb6b2_19 + constrains: + - libstdcxx-ng ==15.2.0=*_19 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 5546559 + timestamp: 1778268777463 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.42.2-h5347b49_0.conda + sha256: 9b1bdce27a7e31f7d241aeecff67a1f3101d52a2b1e33ccc2cdf2613072bf81f + md5: 01bb81d12c957de066ea7362007df642 + depends: + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 40017 + timestamp: 1781625522462 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.42.2-h1022ec0_0.conda + sha256: 7663489f97c104ae3814db10f384932c74b439f3c1fd4247e4fe3599830c090a + md5: 58fa42bc4bc71fc329889497ec15effb + depends: + - libgcc >=14 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 43248 + timestamp: 1781625528371 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + sha256: 6ae68e0b86423ef188196fff6207ed0c8195dd84273cb5623b85aa08033a410c + md5: 5aa797f8787fe7a17d1b0821485b5adc + depends: + - libgcc-ng >=12 + license: LGPL-2.1-or-later + purls: [] + size: 100393 + timestamp: 1702724383534 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda + sha256: 6b46c397644091b8a26a3048636d10b989b1bf266d4be5e9474bf763f828f41f + md5: b4df5d7d4b63579d081fd3a4cf99740e + depends: + - libgcc-ng >=12 + license: LGPL-2.1-or-later + purls: [] + size: 114269 + timestamp: 1702724369203 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.2-h25fd6f3_2.conda + sha256: 55044c403570f0dc26e6364de4dc5368e5f3fc7ff103e867c487e2b5ab2bcda9 + md5: d87ff7921124eccd67248aa483c23fec + depends: + - __glibc >=2.17,<3.0.a0 + constrains: + - zlib 1.3.2 *_2 + license: Zlib + license_family: Other + purls: [] + size: 63629 + timestamp: 1774072609062 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.2-hdc9db2a_2.conda + sha256: eb111e32e5a7313a5bf799c7fb2419051fa2fe7eff74769fac8d5a448b309f7f + md5: 502006882cf5461adced436e410046d1 + constrains: + - zlib 1.3.2 *_2 + license: Zlib + license_family: Other + purls: [] + size: 69833 + timestamp: 1774072605429 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.2-hbb4bfdb_2.conda + sha256: 4c6da089952b2d70150c74234679d6f7ac04f4a98f9432dec724968f912691e7 + md5: 30439ff30578e504ee5e0b390afc8c65 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.2 *_2 + license: Zlib + license_family: Other + purls: [] + size: 59000 + timestamp: 1774073052242 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_2.conda + sha256: 361415a698514b19a852f5d1123c5da746d4642139904156ddfca7c922d23a05 + md5: bc5a5721b6439f2f62a84f2548136082 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.2 *_2 + license: Zlib + license_family: Other + purls: [] + size: 47759 + timestamp: 1774072956767 +- conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.2-hfd05255_2.conda + sha256: 88609816e0cc7452bac637aaf65783e5edf4fee8a9f8e22bdc3a75882c536061 + md5: dbabbd6234dea34040e631f87676292f + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + constrains: + - zlib 1.3.2 *_2 + license: Zlib + license_family: Other + purls: [] + size: 58347 + timestamp: 1774072851498 +- pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl + name: markdown + version: 3.10.2 + sha256: e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36 + requires_dist: + - coverage ; extra == 'testing' + - pyyaml ; extra == 'testing' + - mkdocs>=1.6 ; extra == 'docs' + - mkdocs-nature>=0.6 ; extra == 'docs' + - mdx-gh-links>=0.2 ; extra == 'docs' + - mkdocstrings[python]>=0.28.3 ; extra == 'docs' + - mkdocs-gen-files ; extra == 'docs' + - mkdocs-section-index ; extra == 'docs' + - mkdocs-literate-nav ; extra == 'docs' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + name: markupsafe + version: 3.0.3 + sha256: 3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: markupsafe + version: 3.0.3 + sha256: d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl + name: markupsafe + version: 3.0.3 + sha256: d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl + name: markupsafe + version: 3.0.3 + sha256: 1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl + name: markupsafe + version: 3.0.3 + sha256: 26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c + requires_python: '>=3.9' +- pypi: git+https://github.com/squidfunk/mike.git#2d4ad799442f4592db8ad53b179bfb33db8c69ac + name: mike + version: 2.2.0+zensical.0.1.0 + requires_dist: + - jinja2>=2.7 + - pyparsing>=3.0 + - verspec + - zensical>=0.0.30 + - ruff ; extra == 'dev' + - shtab ; extra == 'dev' + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_0.conda + sha256: fc89f74bbe362fb29fa3c037697a89bec140b346a2469a90f7936d1d7ea4d8a3 + md5: fc21868a1a5aacc937e7a18747acb8a5 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: X11 AND BSD-3-Clause + purls: [] + size: 918956 + timestamp: 1777422145199 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.6-hf8d1292_0.conda + sha256: 369db85c5cd8d99dde364ce70725d76511d9c8199e5b820c740414091bf5bcca + md5: b2a43456aa56fe80c2477a5094899eff + depends: + - libgcc >=14 + license: X11 AND BSD-3-Clause + purls: [] + size: 960036 + timestamp: 1777422174534 +- conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.6-hcc0dc9a_0.conda + sha256: f5f7e006ff4271305ab4cc08eedd855c67a571793c3d18aff73f645f088a8cae + md5: 31b8740cf1b2588d4e61c81191004061 + depends: + - __osx >=11.0 + license: X11 AND BSD-3-Clause + purls: [] + size: 831711 + timestamp: 1777423052277 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.6-h1d4f5a5_0.conda + sha256: 4ea6c620b87bd1d42bb2ccc2c87cd2483fa2d7f9e905b14c223f11ff3f4c455d + md5: 343d10ed5b44030a2f67193905aea159 + depends: + - __osx >=11.0 + license: X11 AND BSD-3-Clause + purls: [] + size: 805509 + timestamp: 1777423252320 +- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.3-h35e630c_0.conda + sha256: d48f5c22b9897c01e4dff3680f1f57ceb02711ab9c62f74339b080419dfad34b + md5: 79dd2074b5cd5c5c6b2930514a11e22d + depends: + - __glibc >=2.17,<3.0.a0 + - ca-certificates + - libgcc >=14 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 3159683 + timestamp: 1781069855778 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.6.3-h546c87b_0.conda + sha256: da4a5df42614166b69c2f6d8602fc1425f7aaa699f77c3bafb5c7fe69b3d9fb7 + md5: fa6260b3e6eababf6ca85a7eb3336383 + depends: + - ca-certificates + - libgcc >=14 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 3704664 + timestamp: 1781069675555 +- conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.6.3-hc881268_0.conda + sha256: 819d4368d6b5b298fa40d4bc836c1250842489002cacf3fb918a13ee2033b7c6 + md5: 46be42ab403712fd349d007d763bf767 + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + purls: [] + size: 2775300 + timestamp: 1781071391999 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.3-hd24854e_0.conda + sha256: b3e3ca895c336d4eb91c5d2f244a312bdb59a0de8cfa0cc4c179225ab2f6bbfb + md5: 8187a86242741725bfa74785fe812979 + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + purls: [] + size: 3102584 + timestamp: 1781069820667 +- conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.3-hf411b9b_0.conda + sha256: cb6e7ba0d010ee0d3249ce9886de3d7613d26d9965d4c95666fa66b9c4c31001 + md5: e99f95734a326c0fd4d02bbd995150d4 + depends: + - ca-certificates + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 9414790 + timestamp: 1781071745579 +- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + name: pygments + version: 2.20.0 + sha256: 81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + requires_dist: + - colorama>=0.4.6 ; extra == 'windows-terminal' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl + name: pymdown-extensions + version: 11.0.1 + sha256: db3943a62bab7e03af1364f0c4083e64b91fb097675a4b6cceccfbe9a77e5eb2 + requires_dist: + - markdown>=3.6 + - pyyaml + - pygments>=2.19.1 ; extra == 'extra' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl + name: pyparsing + version: 3.3.2 + sha256: 850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d + requires_dist: + - railroad-diagrams ; extra == 'diagrams' + - jinja2 ; extra == 'diagrams' + requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda + sha256: a44655c1c3e1d43ed8704890a91e12afd68130414ea2c0872e154e5633a13d7e + md5: 7eccb41177e15cc672e1babe9056018e + depends: + - __glibc >=2.17,<3.0.a0 + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.7.4,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.2,<6.0a0 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.51.2,<4.0a0 + - libuuid >=2.41.3,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.5,<4.0a0 + - readline >=8.3,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + purls: [] + size: 31608571 + timestamp: 1772730708989 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.12.13-h91f4b29_0_cpython.conda + sha256: 61933478813f5fd96c89a00dd964201b0266d71d2e3bc4dd5679354056e46948 + md5: 8aed8fdbbc03a5c9f455d20ce75a9dce + depends: + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-aarch64 >=2.36.1 + - libexpat >=2.7.4,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.2,<6.0a0 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.51.2,<4.0a0 + - libuuid >=2.41.3,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.5,<4.0a0 + - readline >=8.3,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + purls: [] + size: 13757191 + timestamp: 1772728951853 +- conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.12.13-ha9537fe_0_cpython.conda + sha256: fb592ceb1bc247d19247d5535083da4a79721553e29e1290f5d81c07d4f086b5 + md5: ec05996c0d914a4e98ee3c7d789083f8 + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.4,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.2,<6.0a0 + - libsqlite >=3.51.2,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.5,<4.0a0 + - readline >=8.3,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + purls: [] + size: 13672169 + timestamp: 1772730464626 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-h8561d8f_0_cpython.conda + sha256: e658e647a4a15981573d6018928dec2c448b10c77c557c29872043ff23c0eb6a + md5: 8e7608172fa4d1b90de9a745c2fd2b81 + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.4,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.2,<6.0a0 + - libsqlite >=3.51.2,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.5,<4.0a0 + - readline >=8.3,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + purls: [] + size: 12127424 + timestamp: 1772730755512 +- conda: https://conda.anaconda.org/conda-forge/win-64/python-3.12.13-h0159041_0_cpython.conda + sha256: a02b446d8b7b167b61733a3de3be5de1342250403e72a63b18dac89e99e6180e + md5: 2956dff38eb9f8332ad4caeba941cfe7 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.4,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.2,<6.0a0 + - libsqlite >=3.51.2,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - openssl >=3.5.5,<4.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + purls: [] + size: 15840187 + timestamp: 1772728877265 +- pypi: https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl + name: pyyaml + version: 6.0.3 + sha256: 5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl + name: pyyaml + version: 6.0.3 + sha256: fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: pyyaml + version: 6.0.3 + sha256: ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl + name: pyyaml + version: 6.0.3 + sha256: 7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + name: pyyaml + version: 6.0.3 + sha256: 9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 + requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + sha256: 12ffde5a6f958e285aa22c191ca01bbd3d6e710aa852e00618fa6ddc59149002 + md5: d7d95fc8287ea7bf33e0e7116d2b95ec + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 345073 + timestamp: 1765813471974 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.3-hb682ff5_0.conda + sha256: fe695f9d215e9a2e3dd0ca7f56435ab4df24f5504b83865e3d295df36e88d216 + md5: 3d49cad61f829f4f0e0611547a9cda12 + depends: + - libgcc >=14 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 357597 + timestamp: 1765815673644 +- conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda + sha256: 4614af680aa0920e82b953fece85a03007e0719c3399f13d7de64176874b80d5 + md5: eefd65452dfe7cce476a519bece46704 + depends: + - __osx >=10.13 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 317819 + timestamp: 1765813692798 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + sha256: a77010528efb4b548ac2a4484eaf7e1c3907f2aec86123ed9c5212ae44502477 + md5: f8381319127120ce51e081dce4865cf4 + depends: + - __osx >=11.0 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 313930 + timestamp: 1765813902568 +- conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h366c992_103.conda + sha256: cafeec44494f842ffeca27e9c8b0c27ed714f93ac77ddadc6aaf726b5554ebac + md5: cffd3bdd58090148f4cfcd831f4b26ab + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + constrains: + - xorg-libx11 >=1.8.12,<2.0a0 + license: TCL + license_family: BSD + purls: [] + size: 3301196 + timestamp: 1769460227866 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h0dc03b3_103.conda + sha256: e25c314b52764219f842b41aea2c98a059f06437392268f09b03561e4f6e5309 + md5: 7fc6affb9b01e567d2ef1d05b84aa6ed + depends: + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + constrains: + - xorg-libx11 >=1.8.12,<2.0a0 + license: TCL + license_family: BSD + purls: [] + size: 3368666 + timestamp: 1769464148928 +- conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h7142dee_3.conda + sha256: 7f0d9c320288532873e2d8486c331ec6d87919c9028208d3f6ac91dc8f99a67b + md5: 6e6efb7463f8cef69dbcb4c2205bf60e + depends: + - __osx >=10.13 + - libzlib >=1.3.1,<2.0a0 + license: TCL + license_family: BSD + purls: [] + size: 3282953 + timestamp: 1769460532442 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h010d191_3.conda + sha256: 799cab4b6cde62f91f750149995d149bc9db525ec12595e8a1d91b9317f038b3 + md5: a9d86bc62f39b94c4661716624eb21b0 + depends: + - __osx >=11.0 + - libzlib >=1.3.1,<2.0a0 + license: TCL + license_family: BSD + purls: [] + size: 3127137 + timestamp: 1769460817696 +- conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h6ed50ae_3.conda + sha256: 0e79810fae28f3b69fe7391b0d43f5474d6bd91d451d5f2bde02f55ae481d5e3 + md5: 0481bfd9814bf525bd4b3ee4b51494c4 + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: TCL + license_family: BSD + purls: [] + size: 3526350 + timestamp: 1769460339384 +- pypi: https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: tomli + version: 2.4.1 + sha256: 136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl + name: tomli + version: 2.4.1 + sha256: 52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + name: tomli + version: 2.4.1 + sha256: ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl + name: tomli + version: 2.4.1 + sha256: c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl + name: tomli + version: 2.4.1 + sha256: 7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085 + requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + sha256: 1d30098909076af33a35017eed6f2953af1c769e273a0626a04722ac4acaba3c + md5: ad659d0a2b3e47e38d829aa8cad2d610 + license: LicenseRef-Public-Domain + purls: [] + size: 119135 + timestamp: 1767016325805 +- conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda + sha256: 3005729dce6f3d3f5ec91dfc49fc75a0095f9cd23bab49efb899657297ac91a5 + md5: 71b24316859acd00bdb8b38f5e2ce328 + constrains: + - vc14_runtime >=14.29.30037 + - vs2015_runtime >=14.29.30037 + license: LicenseRef-MicrosoftWindowsSDK10 + purls: [] + size: 694692 + timestamp: 1756385147981 +- conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.5-h1b7c187_39.conda + sha256: 17693b60cb54f80c60275f003f3bfc1b128af56dbfd65c4fae37c64eeb755ce1 + md5: 2eacea63f545b97342da520df6854276 + depends: + - vc14_runtime >=14.51.36231 + track_features: + - vc14 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 20362 + timestamp: 1781320968457 +- conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.51.36231-h1b9f54f_39.conda + sha256: 8153ed849c92e891eacac0f2f8d7ecb79f9b5fd7f7917fbb896f252a60a40390 + md5: 06a5bf5a1ca16cce0df6eaa91fc42bc2 + depends: + - ucrt >=10.0.20348.0 + - vcomp14 14.51.36231 h1b9f54f_39 + constrains: + - vs2015_runtime 14.51.36231.* *_39 + license: LicenseRef-MicrosoftVisualCpp2015-2022Runtime + license_family: Proprietary + purls: [] + size: 737434 + timestamp: 1781320964561 +- conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.51.36231-h1b9f54f_39.conda + sha256: 07fb14713c4bc62e2533a2e23a363abfb0e65650681fba0ae4c840e2219350f3 + md5: 8b53a83fda40ec679e4d63fa32fae989 + depends: + - ucrt >=10.0.20348.0 + constrains: + - vs2015_runtime 14.51.36231.* *_39 + license: LicenseRef-MicrosoftVisualCpp2015-2022Runtime + license_family: Proprietary + purls: [] + size: 120684 + timestamp: 1781320948530 +- pypi: https://files.pythonhosted.org/packages/a4/ce/3b6fee91c85626eaf769d617f1be9d2e15c1cca027bbdeb2e0d751469355/verspec-0.1.0-py3-none-any.whl + name: verspec + version: 0.1.0 + sha256: 741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31 + requires_dist: + - coverage ; extra == 'test' + - flake8>=3.7 ; extra == 'test' + - mypy ; extra == 'test' + - pretend ; extra == 'test' + - pytest ; extra == 'test' +- pypi: https://files.pythonhosted.org/packages/33/9b/1f5065c664afde6a63e0ada8c14eae75f9e6960df597a5ca28c07340f938/zensical-0.0.50-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: zensical + version: 0.0.50 + sha256: a64f957f94f7d8e8847a7f1e8205509ba4b188c93c157c5e7be49bcb8556a127 + requires_dist: + - click>=8.1.8 + - deepmerge>=2.0 + - jinja2>=3.1 + - markdown>=3.7 + - pygments>=2.20 + - pymdown-extensions>=10.21.3 + - pyyaml>=6.0.2 + - tomli>=2.4.0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/38/f5/16c087635680efb39e1d7b7aa3c2cca548305aa29ad51dee04f37b32b65f/zensical-0.0.50-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + name: zensical + version: 0.0.50 + sha256: 2bbf3034a2cb1a9d2a72a5ee7047688717c93da0d8a90d25dd871db8d5f33a6d + requires_dist: + - click>=8.1.8 + - deepmerge>=2.0 + - jinja2>=3.1 + - markdown>=3.7 + - pygments>=2.20 + - pymdown-extensions>=10.21.3 + - pyyaml>=6.0.2 + - tomli>=2.4.0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/bd/9b/e1a22fb4ea8fc3e8377d7b394ee06cf0349e7f9a64ecb4c9873dee79b102/zensical-0.0.50-cp310-abi3-macosx_11_0_arm64.whl + name: zensical + version: 0.0.50 + sha256: 376887def6470c57509b48519870f74e2b987e30509cf5c819fcf372771f9403 + requires_dist: + - click>=8.1.8 + - deepmerge>=2.0 + - jinja2>=3.1 + - markdown>=3.7 + - pygments>=2.20 + - pymdown-extensions>=10.21.3 + - pyyaml>=6.0.2 + - tomli>=2.4.0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/c2/d2/e126d56642a5fd437eeb5a067b7c0d1f766c08e22d6a708bb923986f1d44/zensical-0.0.50-cp310-abi3-macosx_10_12_x86_64.whl + name: zensical + version: 0.0.50 + sha256: 26dda9dbacab84db2743726f83202798e1893fbfddecb6a845812d7a331043ad + requires_dist: + - click>=8.1.8 + - deepmerge>=2.0 + - jinja2>=3.1 + - markdown>=3.7 + - pygments>=2.20 + - pymdown-extensions>=10.21.3 + - pyyaml>=6.0.2 + - tomli>=2.4.0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e3/e1/4babb30544d7465c95a6a93abc0680b1c51752411a225c2b7f2cb44e80c7/zensical-0.0.50-cp310-abi3-win_amd64.whl + name: zensical + version: 0.0.50 + sha256: 2f054769bf96ae46353de3eca2463ad4db6df1da9fde515c6d7fc54c3608e615 + requires_dist: + - click>=8.1.8 + - deepmerge>=2.0 + - jinja2>=3.1 + - markdown>=3.7 + - pygments>=2.20 + - pymdown-extensions>=10.21.3 + - pyyaml>=6.0.2 + - tomli>=2.4.0 + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + sha256: 68f0206ca6e98fea941e5717cec780ed2873ffabc0e1ed34428c061e2c6268c7 + md5: 4a13eeac0b5c8e5b8ab496e6c4ddd829 + depends: + - __glibc >=2.17,<3.0.a0 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 601375 + timestamp: 1764777111296 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-h85ac4a6_6.conda + sha256: 569990cf12e46f9df540275146da567d9c618c1e9c7a0bc9d9cfefadaed20b75 + md5: c3655f82dcea2aa179b291e7099c1fcc + depends: + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 614429 + timestamp: 1764777145593 diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 000000000..abc85eb4f --- /dev/null +++ b/pixi.toml @@ -0,0 +1,34 @@ +# pixi workspace for the substrait-java documentation site. +# +# The docs are built with Zensical (https://zensical.org) from the Markdown +# sources under `docs/`, configured by `zensical.toml`. Multi-version publishing +# uses the Zensical-compatible fork of `mike`. This mirrors the tooling used by +# substrait-python so the two repositories stay consistent. +# +# Local usage: +# pixi run docs-serve # live-reloading preview at http://localhost:8000 +# pixi run docs-build # build the static site into ./site +# +# Versioned publishing is done in CI (.github/workflows/docs-deploy.yml) via +# `pixi run -e docs mike ...`. + +[workspace] +channels = ["conda-forge"] +platforms = ["linux-64", "osx-64", "osx-arm64", "win-64", "linux-aarch64"] + +[feature.docs.dependencies] +# conda-provided interpreter that the PyPI dependencies below resolve against +python = "3.12.*" + +[feature.docs.pypi-dependencies] +zensical = "*" +# Zensical-compatible fork of mike (not published on PyPI). Pin to a commit SHA +# here if an upstream change ever breaks the build. +mike = { git = "https://github.com/squidfunk/mike.git" } + +[feature.docs.tasks] +docs-build = "zensical build --clean" +docs-serve = "zensical serve" + +[environments] +docs = ["docs"] diff --git a/readme.md b/readme.md index bfe9ffd6b..5f0404c48 100644 --- a/readme.md +++ b/readme.md @@ -6,6 +6,21 @@ Substrait Java is a project that makes it easier to build [Substrait](https://su 3) **Spark** is the module that provides an API for translating a Substrait plan to and from a Spark query plan. The most commonly used logical relations and functions are supported, including those generated from all of the TPC-H and TPC-DS queries. The supported features are formally specified in the Substrait dialect file [spark_dialect.yaml](spark/spark_dialect.yaml). +## Documentation + +User-facing documentation for all modules is published at +****. + +The documentation sources live under [`docs/`](docs) and are built with +[Zensical](https://zensical.org). To preview the site locally you need [pixi](https://pixi.sh): + +```bash +pixi run docs-serve # live-reloading preview at http://localhost:8000 +pixi run docs-build # build the static site into ./site +``` + +See [CONTRIBUTING.md](CONTRIBUTING.md#documentation) for more detail. + ## Building After you've cloned the project through git, Substrait Java is built with a tool called [Gradle](https://gradle.org/). To build, execute the following: ``` diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 000000000..3d7ded687 --- /dev/null +++ b/zensical.toml @@ -0,0 +1,142 @@ +# Zensical configuration for the substrait-java documentation. +# Docs: https://zensical.org/docs/ +# +# Build: pixi run docs-build (or: zensical build) +# Serve: pixi run docs-serve (or: zensical serve) + +[project] +site_name = "substrait-java" +site_description = "Java implementation of Substrait: build, convert, and serialize relational query plans." +site_author = "Substrait contributors" +# Canonical hosting URL. Required for multi-version docs: each version is +# published as a subdirectory of this URL. Adjust if the site is served +# elsewhere (custom domain, or a fork's *.github.io while testing). +site_url = "https://substrait-io.github.io/substrait-java/" +repo_url = "https://github.com/substrait-io/substrait-java" +repo_name = "substrait-io/substrait-java" +copyright = "Copyright © 2026 Substrait contributors" + +nav = [ + { "Home" = "index.md" }, + { "Getting started" = "getting-started.md" }, + { "Core" = [ + "core/index.md", + { "Building plans" = "core/building-plans.md" }, + { "Types" = "core/types.md" }, + { "Expressions & literals" = "core/expressions.md" }, + { "Relations" = "core/relations.md" }, + { "Serialization" = "core/serialization.md" }, + { "Extended expressions" = "core/extended-expressions.md" }, + { "Function & type extensions" = "core/extensions.md" }, + ] }, + { "Isthmus (SQL)" = [ + "isthmus/index.md", + { "SQL to Substrait" = "isthmus/sql-to-substrait.md" }, + { "SQL expressions" = "isthmus/sql-expressions.md" }, + { "Substrait to SQL" = "isthmus/substrait-to-sql.md" }, + { "Substrait to Calcite" = "isthmus/substrait-to-calcite.md" }, + { "Types & type system" = "isthmus/types.md" }, + { "Customization" = "isthmus/customization.md" }, + { "Supported SQL" = "isthmus/supported-sql.md" }, + ] }, + { "Isthmus CLI" = [ + "isthmus-cli/index.md", + { "Install & build" = "isthmus-cli/install.md" }, + { "Usage" = "isthmus-cli/usage.md" }, + { "Examples" = "isthmus-cli/examples.md" }, + ] }, + { "Spark" = [ + "spark/index.md", + { "Compatibility & dependencies" = "spark/compatibility.md" }, + { "Producing plans" = "spark/producing-plans.md" }, + { "Consuming plans" = "spark/consuming-plans.md" }, + { "Supported features" = "spark/supported-features.md" }, + { "End-to-end example" = "spark/end-to-end.md" }, + ] }, +] + +# -------------------------------------------------------------------------- +# Multi-version documentation (via the Zensical-compatible `mike` fork). +# Publishing is done by .github/workflows/docs-deploy.yml, which commits each +# version to the `gh-pages` branch as a subdirectory of site_url. Setting the +# provider here renders the version selector in the header. +# -------------------------------------------------------------------------- +[project.extra.version] +provider = "mike" # enables the selector; reads mike's versions.json on gh-pages +default = "latest" # the alias the site root redirects to +alias = true + +# -------------------------------------------------------------------------- +# Theme (Material-style; ships with Zensical). +# -------------------------------------------------------------------------- +[project.theme] +language = "en" +features = [ + "content.code.annotate", + "content.code.copy", + "content.tabs.link", + "navigation.footer", + "navigation.indexes", + "navigation.instant", + "navigation.sections", + "navigation.top", + "navigation.tracking", + "search.highlight", + "toc.follow", +] + +[[project.theme.palette]] +scheme = "default" +primary = "indigo" +accent = "indigo" +toggle.icon = "lucide/sun" +toggle.name = "Switch to dark mode" + +[[project.theme.palette]] +scheme = "slate" +primary = "indigo" +accent = "indigo" +toggle.icon = "lucide/moon" +toggle.name = "Switch to light mode" + +[[project.extra.social]] +icon = "fontawesome/brands/github" +link = "https://github.com/substrait-io/substrait-java" + +[[project.extra.social]] +icon = "fontawesome/brands/java" +link = "https://central.sonatype.com/namespace/io.substrait" + +# -------------------------------------------------------------------------- +# Markdown extensions (Zensical defaults + what the guide relies on). +# -------------------------------------------------------------------------- +[project.markdown_extensions.abbr] +[project.markdown_extensions.admonition] +[project.markdown_extensions.attr_list] +[project.markdown_extensions.def_list] +[project.markdown_extensions.footnotes] +[project.markdown_extensions.md_in_html] +[project.markdown_extensions.toc] +permalink = true +[project.markdown_extensions.pymdownx.betterem] +[project.markdown_extensions.pymdownx.caret] +[project.markdown_extensions.pymdownx.details] +[project.markdown_extensions.pymdownx.emoji] +emoji_generator = "zensical.extensions.emoji.to_svg" +emoji_index = "zensical.extensions.emoji.twemoji" +[project.markdown_extensions.pymdownx.highlight] +anchor_linenums = true +line_spans = "__span" +pygments_lang_class = true +[project.markdown_extensions.pymdownx.inlinehilite] +[project.markdown_extensions.pymdownx.keys] +[project.markdown_extensions.pymdownx.magiclink] +[project.markdown_extensions.pymdownx.mark] +[project.markdown_extensions.pymdownx.smartsymbols] +[project.markdown_extensions.pymdownx.superfences] +[project.markdown_extensions.pymdownx.tabbed] +alternate_style = true +combine_header_slug = true +[project.markdown_extensions.pymdownx.tasklist] +custom_checkbox = true +[project.markdown_extensions.pymdownx.tilde]