From 4bcdc1e3da2a2ecd885c3359a406ed3e54495f39 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Thu, 25 Jun 2026 17:34:46 +0800 Subject: [PATCH 1/6] [core] Support append writes for MAP shared-shredding --- LICENSE | 3 + .../ArrowSchemaMetadataCompatibilityTest.java | 125 +- .../paimon/append/AppendOnlyWriter.java | 28 +- .../MapSharedShreddingCoreUtils.java | 157 +++ .../MapSharedShreddingWritePlanFactory.java | 133 ++ .../paimon/io/FileMetadataFinalizer.java | 29 + .../paimon/io/RowDataFileWritePlan.java | 72 ++ .../io/RowDataFileWritePlanFactory.java | 25 + .../apache/paimon/io/RowDataFileWriter.java | 37 +- .../paimon/io/RowDataRollingFileWriter.java | 154 ++- .../apache/paimon/io/RowDataTransform.java | 32 + .../apache/paimon/io/SingleFileWriter.java | 3 + .../operation/BaseAppendFileStoreWrite.java | 13 +- .../paimon/schema/SchemaValidation.java | 5 + .../paimon/append/AppendOnlyWriterTest.java | 1105 ++++++++++++++++- .../MapSharedShreddingCoreUtilsTest.java | 300 +++++ .../paimon/io/KeyValueFileReadWriteTest.java | 3 +- .../paimon/schema/SchemaValidationTest.java | 8 + .../paimon/format/ArrowSchemaMetadata.java | 26 +- .../paimon/format/FormatMetadataUtils.java | 1 + .../format/FormatMetadataUtilsTest.java | 22 +- .../format/orc/OrcFormatReadWriteTest.java | 6 +- 22 files changed, 2227 insertions(+), 60 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingWritePlanFactory.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/io/FileMetadataFinalizer.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWritePlan.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWritePlanFactory.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/io/RowDataTransform.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtilsTest.java diff --git a/LICENSE b/LICENSE index 16447645b8d7..aa42bed7e693 100644 --- a/LICENSE +++ b/LICENSE @@ -272,6 +272,9 @@ paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java from https://parquet.apache.org/ version 1.14.0 +paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java +from https://arrow.apache.org/ version 15.0.0 + paimon-common/src/main/java/org/apache/paimon/data/variant/GenericVariant.java paimon-common/src/main/java/org/apache/paimon/data/variant/GenericVariantBuilder.java paimon-common/src/main/java/org/apache/paimon/data/variant/GenericVariantUtil.java diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java index 165383bc34da..73b8a2b28a4d 100644 --- a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java +++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java @@ -59,18 +59,41 @@ public void testFormatMetadataCanBeReadByArrowJava() { .containsAllEntriesOf(tagsMetadata); } + @Test + public void testRichFormatMetadataSchemaMatchesArrowUtils() { + // Keep this case broad because Arrow schema metadata is used as an IPC-compatible + // format contract. It verifies that format-generated bytes deserialize to the same + // field tree as ArrowUtils, and that Arrow Java official serialization can be parsed + // back by format metadata utilities with the same top-level field metadata. + RowType rowType = richRowType(); + Map nestedMetadata = new LinkedHashMap<>(); + nestedMetadata.put("paimon.test.nested", "enabled"); + Map> fieldMetadata = new LinkedHashMap<>(); + fieldMetadata.put("nested", nestedMetadata); + + byte[] schemaBytes = + FormatMetadataUtils.buildArrowSchemaMetadata( + rowType, fieldMetadata, FormatMetadataUtils.PARQUET_FIELD_ID_KEY); + Schema schema = Schema.deserializeMessage(ByteBuffer.wrap(schemaBytes)); + List expectedFields = arrowFields(rowType, fieldMetadata); + + assertThat(schema.getFields()).hasSize(expectedFields.size()); + for (int i = 0; i < expectedFields.size(); i++) { + assertFieldEquals(schema.getFields().get(i), expectedFields.get(i)); + } + + byte[] arrowJavaSchemaBytes = new Schema(expectedFields).serializeAsMessage(); + assertThat(FormatMetadataUtils.readFieldMetadata(arrowJavaSchemaBytes)) + .isEqualTo(FormatMetadataUtils.readFieldMetadata(schemaBytes)); + } + @Test public void testArrowJavaSchemaCanBeReadByFormatMetadata() { RowType rowType = rowType(); Map tagsMetadata = tagsMetadata(); - List fields = new ArrayList<>(); - for (DataField field : rowType.getFields()) { - Field arrowField = ArrowUtils.toArrowField(field.name(), field.id(), field.type(), 0); - if ("tags".equals(field.name())) { - arrowField = withMetadata(arrowField, tagsMetadata); - } - fields.add(arrowField); - } + Map> fieldMetadata = new LinkedHashMap<>(); + fieldMetadata.put("tags", tagsMetadata); + List fields = arrowFields(rowType, fieldMetadata); byte[] schemaBytes = new Schema(fields).serializeAsMessage(); Map> metadata = @@ -89,6 +112,64 @@ private static RowType rowType() { DataTypes.FIELD(1, "tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))); } + private static RowType richRowType() { + return DataTypes.ROW( + DataTypes.FIELD(0, "boolean_field", DataTypes.BOOLEAN().notNull()), + DataTypes.FIELD(1, "tinyint_field", DataTypes.TINYINT()), + DataTypes.FIELD(2, "smallint_field", DataTypes.SMALLINT()), + DataTypes.FIELD(3, "int_field", DataTypes.INT()), + DataTypes.FIELD(4, "bigint_field", DataTypes.BIGINT()), + DataTypes.FIELD(5, "float_field", DataTypes.FLOAT()), + DataTypes.FIELD(6, "double_field", DataTypes.DOUBLE()), + DataTypes.FIELD(7, "decimal_field", DataTypes.DECIMAL(20, 3)), + DataTypes.FIELD(8, "date_field", DataTypes.DATE()), + DataTypes.FIELD(9, "time_field", DataTypes.TIME(6)), + DataTypes.FIELD(10, "timestamp_second_field", DataTypes.TIMESTAMP(0)), + DataTypes.FIELD(11, "timestamp_milli_field", DataTypes.TIMESTAMP(3)), + DataTypes.FIELD(12, "timestamp_micro_field", DataTypes.TIMESTAMP(6)), + DataTypes.FIELD(13, "timestamp_nano_field", DataTypes.TIMESTAMP(9)), + DataTypes.FIELD( + 14, + "timestamp_ltz_second_field", + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0)), + DataTypes.FIELD( + 15, + "timestamp_ltz_milli_field", + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3)), + DataTypes.FIELD( + 16, + "timestamp_ltz_micro_field", + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6)), + DataTypes.FIELD( + 17, + "timestamp_ltz_nano_field", + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(9)), + DataTypes.FIELD(18, "char_field", DataTypes.CHAR(10)), + DataTypes.FIELD(19, "varchar_field", DataTypes.VARCHAR(20)), + DataTypes.FIELD(20, "binary_field", DataTypes.BINARY(16)), + DataTypes.FIELD(21, "varbinary_field", DataTypes.VARBINARY(32)), + DataTypes.FIELD(22, "array_field", DataTypes.ARRAY(DataTypes.INT())), + DataTypes.FIELD( + 23, + "map_field", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.ARRAY(DataTypes.DECIMAL(10, 2)))), + DataTypes.FIELD( + 24, + "nested", + DataTypes.ROW( + DataTypes.FIELD(25, "name", DataTypes.STRING()), + DataTypes.FIELD( + 26, + "scores", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + 27, "score", DataTypes.INT())))))), + DataTypes.FIELD(28, "vector_field", DataTypes.VECTOR(3, DataTypes.FLOAT())), + DataTypes.FIELD(29, "variant_field", DataTypes.VARIANT())); + } + private static Map tagsMetadata() { Map metadata = new LinkedHashMap<>(); metadata.put("paimon.test.tags", "enabled"); @@ -96,6 +177,20 @@ private static Map tagsMetadata() { return metadata; } + private static List arrowFields( + RowType rowType, Map> fieldMetadata) { + List fields = new ArrayList<>(); + for (DataField field : rowType.getFields()) { + Field arrowField = ArrowUtils.toArrowField(field.name(), field.id(), field.type(), 0); + Map metadata = fieldMetadata.get(field.name()); + if (metadata != null) { + arrowField = withMetadata(arrowField, metadata); + } + fields.add(arrowField); + } + return fields; + } + private static Field withMetadata(Field field, Map metadata) { Map merged = new LinkedHashMap<>(); merged.putAll(metadata); @@ -110,4 +205,18 @@ private static Field withMetadata(Field field, Map metadata) { merged), field.getChildren()); } + + private static void assertFieldEquals(Field actual, Field expected) { + assertThat(actual.getName()).isEqualTo(expected.getName()); + assertThat(actual.getFieldType().isNullable()) + .isEqualTo(expected.getFieldType().isNullable()); + assertThat(actual.getFieldType().getType()).isEqualTo(expected.getFieldType().getType()); + assertThat(actual.getFieldType().getDictionary()) + .isEqualTo(expected.getFieldType().getDictionary()); + assertThat(actual.getMetadata()).isEqualTo(expected.getMetadata()); + assertThat(actual.getChildren()).hasSize(expected.getChildren().size()); + for (int i = 0; i < expected.getChildren().size(); i++) { + assertFieldEquals(actual.getChildren().get(i), expected.getChildren().get(i)); + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java index 66dcc75613f6..74c1344b4f9f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java @@ -23,6 +23,9 @@ import org.apache.paimon.compact.CompactManager; import org.apache.paimon.compression.CompressOptions; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.shredding.MapSharedShreddingContext; +import org.apache.paimon.data.shredding.MapSharedShreddingCoreUtils; +import org.apache.paimon.data.shredding.MapSharedShreddingWritePlanFactory; import org.apache.paimon.disk.IOManager; import org.apache.paimon.disk.RowBuffer; import org.apache.paimon.fileindex.FileIndexOptions; @@ -87,6 +90,7 @@ public class AppendOnlyWriter implements BatchRecordWriter, MemoryOwner { private final boolean statsDenseStore; @Nullable private final FileFormat rowSidecarFileFormat; @Nullable private final BlobFileContext blobContext; + @Nullable private final MapSharedShreddingContext sharedShreddingContext; private final List newFiles; private final List deletedFiles; private final List compactBefore; @@ -131,7 +135,8 @@ public AppendOnlyWriter( boolean statsDenseStore, boolean dataEvolutionEnabled, @Nullable FileFormat rowSidecarFileFormat, - @Nullable BlobFileContext blobContext) { + @Nullable BlobFileContext blobContext, + @Nullable MapSharedShreddingContext sharedShreddingContext) { this.fileIO = fileIO; this.schemaId = schemaId; this.fileFormat = fileFormat; @@ -149,6 +154,7 @@ public AppendOnlyWriter( this.statsDenseStore = statsDenseStore; this.rowSidecarFileFormat = dataEvolutionEnabled ? rowSidecarFileFormat : null; this.blobContext = blobContext; + this.sharedShreddingContext = sharedShreddingContext; this.newFiles = new ArrayList<>(); this.deletedFiles = new ArrayList<>(); this.compactBefore = new ArrayList<>(); @@ -211,6 +217,11 @@ public void write(InternalRow rowData) throws Exception { @Override public void writeBundle(BundleRecords bundle) throws Exception { + if (sharedShreddingContext != null && !sharedShreddingContext.isEmpty()) { + // TODO (xinyu.lxy): Support bundle writes for MAP shared-shredding. + throw new UnsupportedOperationException( + "Bundle write is not supported for MAP shared-shredding."); + } if (sinkWriter instanceof BufferedSinkWriter) { for (InternalRow row : bundle) { write(row); @@ -313,8 +324,15 @@ public void toBufferedWriter() throws Exception { } private RollingFileWriter createRollingRowWriter() { + boolean sharedShredding = + sharedShreddingContext != null && !sharedShreddingContext.isEmpty(); if (blobContext != null || !fieldsInVectorFile(writeSchema, vectorFileFormat != null).isEmpty()) { + if (sharedShredding) { + // TODO (xinyu.lxy): Support blob with MAP shared-shredding. + throw new UnsupportedOperationException( + "MAP shared-shredding does not support blob or vector file writes yet."); + } return new DedicatedFormatRollingFileWriter( fileIO, schemaId, @@ -348,7 +366,13 @@ private RollingFileWriter createRollingRowWriter() { asyncFileWrite, statsDenseStore, writeCols, - rowSidecarFileFormat); + rowSidecarFileFormat, + sharedShredding + ? new MapSharedShreddingWritePlanFactory( + writeSchema, + sharedShreddingContext, + MapSharedShreddingCoreUtils.sharedShreddingFieldIdKey(fileFormat)) + : null); } private void trySyncLatestCompaction(boolean blocking) diff --git a/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java new file mode 100644 index 000000000000..5f895999c476 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data.shredding; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.format.FileFormat; +import org.apache.paimon.format.FileFormatDiscover; +import org.apache.paimon.format.FormatMetadataUtils; +import org.apache.paimon.format.FormatReaderContext; +import org.apache.paimon.format.FormatReaderFactory; +import org.apache.paimon.format.SupportsReaderFieldMetadata; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataFilePathFactory; +import org.apache.paimon.reader.FileRecordReader; +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Core utilities for shared-shredding MAP write and restore flows. */ +public class MapSharedShreddingCoreUtils { + + private static final RowType METADATA_READER_ROW_TYPE = new RowType(Collections.emptyList()); + + private MapSharedShreddingCoreUtils() {} + + @Nullable + public static String sharedShreddingFieldIdKey(FileFormat fileFormat) { + switch (fileFormat.getFormatIdentifier()) { + case "parquet": + return FormatMetadataUtils.PARQUET_FIELD_ID_KEY; + case "orc": + return FormatMetadataUtils.ORC_FIELD_ID_KEY; + default: + return null; + } + } + + @Nullable + public static MapSharedShreddingContext createAndRestoreContext( + RowType writeType, + List restoredFiles, + DataFilePathFactory pathFactory, + CoreOptions options, + FileIO fileIO) { + List shreddingFieldNames = + MapSharedShreddingUtils.detectShreddingColumns(writeType, options); + if (shreddingFieldNames.isEmpty()) { + return null; + } + + MapSharedShreddingContext context = + new MapSharedShreddingContext( + MapSharedShreddingUtils.buildColumnToNumColumns( + shreddingFieldNames, options)); + restoreRecentFileStats( + context, + shreddingFieldNames, + restoredFiles, + pathFactory, + fileIO, + FileFormatDiscover.of(options)); + return context; + } + + private static void restoreRecentFileStats( + MapSharedShreddingContext context, + List shreddingFieldNames, + List restoredFiles, + DataFilePathFactory pathFactory, + FileIO fileIO, + FileFormatDiscover fileFormatDiscover) { + Set pendingFieldNames = new HashSet<>(shreddingFieldNames); + if (pendingFieldNames.isEmpty()) { + return; + } + + for (int i = restoredFiles.size() - 1; i >= 0 && !pendingFieldNames.isEmpty(); i--) { + DataFileMeta file = restoredFiles.get(i); + List candidateFields = + candidateFields(file.writeCols(), shreddingFieldNames, pendingFieldNames); + if (candidateFields.isEmpty()) { + continue; + } + + FormatReaderFactory readerFactory = + fileFormatDiscover + .discover(file.fileFormat()) + .createReaderFactory( + METADATA_READER_ROW_TYPE, METADATA_READER_ROW_TYPE, null); + try (FileRecordReader reader = + readerFactory.createReader( + new FormatReaderContext( + fileIO, pathFactory.toPath(file), file.fileSize()))) { + if (!(reader instanceof SupportsReaderFieldMetadata)) { + continue; + } + + Map> fieldMetadata = + ((SupportsReaderFieldMetadata) reader).readFieldMetadata(); + for (String fieldName : candidateFields) { + Map metadata = fieldMetadata.get(fieldName); + if (!MapSharedShreddingUtils.hasShreddingMetadata(metadata)) { + continue; + } + + MapSharedShreddingFieldMeta fieldMeta = + MapSharedShreddingUtils.deserializeMetadata( + metadata, MapSharedShreddingDefine.DEFAULT_DICT_COMPRESSION); + context.reportFileStats(fieldName, fieldMeta.maxRowWidth()); + pendingFieldNames.remove(fieldName); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + private static List candidateFields( + @Nullable List writeCols, + List shreddingFieldNames, + Set pendingFieldNames) { + List fields = new ArrayList<>(); + for (String fieldName : shreddingFieldNames) { + if (pendingFieldNames.contains(fieldName) + && (writeCols == null || writeCols.contains(fieldName))) { + fields.add(fieldName); + } + } + return fields; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingWritePlanFactory.java b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingWritePlanFactory.java new file mode 100644 index 000000000000..6bedabbbba6b --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingWritePlanFactory.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data.shredding; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.format.FormatMetadataUtils; +import org.apache.paimon.format.FormatWriter; +import org.apache.paimon.format.SupportsWriterMetadata; +import org.apache.paimon.io.FileMetadataFinalizer; +import org.apache.paimon.io.RowDataFileWritePlan; +import org.apache.paimon.io.RowDataFileWritePlanFactory; +import org.apache.paimon.io.RowDataTransform; +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Creates per-file write plans for MAP shared-shredding row data files. */ +public class MapSharedShreddingWritePlanFactory implements RowDataFileWritePlanFactory { + + private final RowType logicalType; + private final MapSharedShreddingContext context; + @Nullable private final String fieldIdKey; + + public MapSharedShreddingWritePlanFactory( + RowType logicalType, MapSharedShreddingContext context, @Nullable String fieldIdKey) { + this.logicalType = logicalType; + this.context = context; + this.fieldIdKey = fieldIdKey; + } + + @Override + public RowDataFileWritePlan create() { + MapSharedShreddingRowConverter converter = + new MapSharedShreddingRowConverter(logicalType, context.computeNextK()); + return new SharedShreddingWritePlan(converter); + } + + private class SharedShreddingWritePlan implements RowDataFileWritePlan { + + private final MapSharedShreddingRowConverter converter; + private final RowDataTransform rowTransform; + private final FileMetadataFinalizer metadataFinalizer; + + private SharedShreddingWritePlan(MapSharedShreddingRowConverter converter) { + this.converter = converter; + this.rowTransform = + new RowDataTransform() { + @Override + public RowType physicalType() { + return converter.physicalType(); + } + + @Override + public InternalRow transform(InternalRow row) { + return converter.convert(row); + } + }; + this.metadataFinalizer = new SharedShreddingMetadataFinalizer(converter); + } + + @Override + public RowType physicalType() { + return converter.physicalType(); + } + + @Nullable + @Override + public RowDataTransform rowTransform() { + return rowTransform; + } + + @Nullable + @Override + public FileMetadataFinalizer metadataFinalizer() { + return metadataFinalizer; + } + } + + private class SharedShreddingMetadataFinalizer implements FileMetadataFinalizer { + + private final MapSharedShreddingRowConverter converter; + + private SharedShreddingMetadataFinalizer(MapSharedShreddingRowConverter converter) { + this.converter = converter; + } + + @Override + public void beforeClose(FormatWriter writer) throws IOException { + if (!(writer instanceof SupportsWriterMetadata)) { + throw new UnsupportedOperationException( + "MAP shared-shredding requires a format writer supporting metadata."); + } + + Map> fieldMetadata = new LinkedHashMap<>(); + for (String fieldName : converter.shreddingFieldNames()) { + MapSharedShreddingFieldMeta fieldMeta = converter.buildFieldMeta(fieldName); + Map metadata = new LinkedHashMap<>(); + MapSharedShreddingUtils.serializeMetadata( + fieldMeta, MapSharedShreddingDefine.DEFAULT_DICT_COMPRESSION, metadata); + fieldMetadata.put(fieldName, metadata); + context.reportFileStats(fieldName, fieldMeta.maxRowWidth()); + } + + ((SupportsWriterMetadata) writer) + .addMetadata( + Collections.singletonMap( + FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY, + FormatMetadataUtils.buildArrowSchemaMetadata( + converter.physicalType(), fieldMetadata, fieldIdKey))); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/FileMetadataFinalizer.java b/paimon-core/src/main/java/org/apache/paimon/io/FileMetadataFinalizer.java new file mode 100644 index 000000000000..057b1a38a77e --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/io/FileMetadataFinalizer.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +import org.apache.paimon.format.FormatWriter; + +import java.io.IOException; + +/** Adds file-level metadata after rows are written and before the format writer closes. */ +public interface FileMetadataFinalizer { + + void beforeClose(FormatWriter writer) throws IOException; +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWritePlan.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWritePlan.java new file mode 100644 index 000000000000..153ca4e189f6 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWritePlan.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +/** + * Per-file physical write plan for row data files. + * + *

The physical row may use a different nested layout from the logical input row, but it must + * keep the same top-level field names and arity as the logical write schema. File-level statistics, + * index writers, and commit metadata still use logical top-level fields. + */ +public interface RowDataFileWritePlan { + + /** Returns the physical row type written to the file. */ + RowType physicalType(); + + /** + * Returns a transform from logical input rows to physical file rows, or {@code null} if rows + * can be written directly. + */ + @Nullable + RowDataTransform rowTransform(); + + /** + * Returns a metadata finalizer to add file metadata before the writer is closed, or {@code + * null} if no extra metadata is needed. + */ + @Nullable + FileMetadataFinalizer metadataFinalizer(); + + /** Creates a plan that writes rows directly with the given row type and no extra metadata. */ + static RowDataFileWritePlan direct(RowType rowType) { + return new RowDataFileWritePlan() { + @Override + public RowType physicalType() { + return rowType; + } + + @Nullable + @Override + public RowDataTransform rowTransform() { + return null; + } + + @Nullable + @Override + public FileMetadataFinalizer metadataFinalizer() { + return null; + } + }; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWritePlanFactory.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWritePlanFactory.java new file mode 100644 index 000000000000..325a033991cf --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWritePlanFactory.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +/** Creates file-local write plans for rolling row data writers. */ +public interface RowDataFileWritePlanFactory { + + RowDataFileWritePlan create(); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java index 02dff0fd8233..07675cd3fde3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java @@ -21,6 +21,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.fileindex.FileIndexOptions; import org.apache.paimon.format.FileFormat; +import org.apache.paimon.format.FormatWriter; import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; @@ -56,6 +57,7 @@ public class RowDataFileWriter extends StatsCollectingSingleFileWriter writeCols; private final RowDataFileSequenceNumberTracker sequenceNumberTracker; + private final RowDataFileWritePlan writePlan; public RowDataFileWriter( FileIO fileIO, @@ -75,6 +77,7 @@ public RowDataFileWriter( context, path, writeSchema, + RowDataFileWritePlan.direct(writeSchema), schemaId, seqNumCounterSupplier, fileIndexOptions, @@ -91,7 +94,8 @@ public RowDataFileWriter( FileIO fileIO, FileWriterContext context, Path path, - RowType writeSchema, + RowType logicalWriteSchema, + RowDataFileWritePlan writePlan, long schemaId, Supplier seqNumCounterSupplier, FileIndexOptions fileIndexOptions, @@ -102,18 +106,20 @@ public RowDataFileWriter( @Nullable List writeCols, @Nullable FileFormat rowSidecarFormat, @Nullable Path rowSidecarPath) { - super(fileIO, context, path, Function.identity(), writeSchema, asyncFileWrite); + super(fileIO, context, path, Function.identity(), writePlan.physicalType(), asyncFileWrite); if ((rowSidecarFormat == null) != (rowSidecarPath == null)) { throw new IllegalArgumentException( "Row sidecar format and path should be both null or both non-null."); } this.schemaId = schemaId; this.isExternalPath = isExternalPath; - this.statsArraySerializer = new SimpleStatsConverter(writeSchema, statsDenseStore); + this.statsArraySerializer = + new SimpleStatsConverter(writePlan.physicalType(), statsDenseStore); List auxiliaryFileWriters = new ArrayList<>(); Path fileIndexPath = dataFileToFileIndexPath(path); DataFileIndexWriter dataFileIndexWriter = - DataFileIndexWriter.create(fileIO, fileIndexPath, writeSchema, fileIndexOptions); + DataFileIndexWriter.create( + fileIO, fileIndexPath, logicalWriteSchema, fileIndexOptions); if (dataFileIndexWriter != null) { auxiliaryFileWriters.add( new DataFileIndexAuxiliaryWriter(dataFileIndexWriter, fileIO, fileIndexPath)); @@ -122,7 +128,7 @@ public RowDataFileWriter( auxiliaryFileWriters.add( new RowSidecarAuxiliaryWriter( fileIO, - rowSidecarFormat.createWriterFactory(writeSchema), + rowSidecarFormat.createWriterFactory(logicalWriteSchema), rowSidecarPath, context.compression(), asyncFileWrite)); @@ -130,14 +136,17 @@ public RowDataFileWriter( this.auxiliaryFileWriters = Collections.unmodifiableList(auxiliaryFileWriters); this.fileSource = fileSource; this.writeCols = writeCols; + this.writePlan = writePlan; this.sequenceNumberTracker = new RowDataFileSequenceNumberTracker( - writeSchema, seqNumCounterSupplier, super::recordCount); + logicalWriteSchema, seqNumCounterSupplier, super::recordCount); } @Override public void write(InternalRow row) throws IOException { - super.write(row); + RowDataTransform rowTransform = writePlan.rowTransform(); + InternalRow physicalRow = rowTransform == null ? row : rowTransform.transform(row); + super.write(physicalRow); for (DataFileAuxiliaryWriter auxiliaryFileWriter : auxiliaryFileWriters) { auxiliaryFileWriter.write(row); } @@ -146,6 +155,12 @@ public void write(InternalRow row) throws IOException { @Override public void writeBundle(BundleRecords bundle) throws IOException { + if (writePlan.rowTransform() != null) { + // TODO (xinyu.lxy): Support transformed bundle writes by applying row transformation to + // bundles. + throw new UnsupportedOperationException( + "Bundle write is not supported for transformed row data writer."); + } for (InternalRow row : bundle) { write(row); } @@ -188,6 +203,14 @@ public Optional abortExecutor() { return Optional.of(new CompoundFileWriterAbortExecutor(fileIO, path, abortExecutors)); } + @Override + protected void beforeCloseFormatWriter(FormatWriter writer) throws IOException { + FileMetadataFinalizer metadataFinalizer = writePlan.metadataFinalizer(); + if (metadataFinalizer != null) { + metadataFinalizer.beforeClose(writer); + } + } + @Override public DataFileMeta result() throws IOException { long fileSize = outputBytes(); diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java index 0da1badf6229..6c73a7a6d862 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java @@ -36,6 +36,40 @@ /** {@link RollingFileWriterImpl} for data files containing {@link InternalRow}. */ public class RowDataRollingFileWriter extends RollingFileWriterImpl { + public RowDataRollingFileWriter( + FileIO fileIO, + long schemaId, + FileFormat fileFormat, + long targetFileSize, + RowType writeSchema, + DataFilePathFactory pathFactory, + Supplier seqNumCounterSupplier, + String fileCompression, + SimpleColStatsCollector.Factory[] statsCollectors, + FileIndexOptions fileIndexOptions, + FileSource fileSource, + boolean asyncFileWrite, + boolean statsDenseStore, + @Nullable List writeCols) { + this( + fileIO, + schemaId, + fileFormat, + targetFileSize, + writeSchema, + pathFactory, + seqNumCounterSupplier, + fileCompression, + statsCollectors, + fileIndexOptions, + fileSource, + asyncFileWrite, + statsDenseStore, + writeCols, + null, + null); + } + public RowDataRollingFileWriter( FileIO fileIO, long schemaId, @@ -52,30 +86,104 @@ public RowDataRollingFileWriter( boolean statsDenseStore, @Nullable List writeCols, @Nullable FileFormat rowSidecarFormat) { + this( + fileIO, + schemaId, + fileFormat, + targetFileSize, + writeSchema, + pathFactory, + seqNumCounterSupplier, + fileCompression, + statsCollectors, + fileIndexOptions, + fileSource, + asyncFileWrite, + statsDenseStore, + writeCols, + rowSidecarFormat, + null); + } + + public RowDataRollingFileWriter( + FileIO fileIO, + long schemaId, + FileFormat fileFormat, + long targetFileSize, + RowType writeSchema, + DataFilePathFactory pathFactory, + Supplier seqNumCounterSupplier, + String fileCompression, + SimpleColStatsCollector.Factory[] statsCollectors, + FileIndexOptions fileIndexOptions, + FileSource fileSource, + boolean asyncFileWrite, + boolean statsDenseStore, + @Nullable List writeCols, + @Nullable FileFormat rowSidecarFormat, + @Nullable RowDataFileWritePlanFactory writePlanFactory) { super( - () -> { - Path dataPath = pathFactory.newPath(); - Path rowSidecarPath = - rowSidecarFormat == null - ? null - : new Path(dataPath.getParent(), dataPath.getName() + ".row"); - return new RowDataFileWriter( - fileIO, - RollingFileWriter.createFileWriterContext( - fileFormat, writeSchema, statsCollectors, fileCompression), - dataPath, - writeSchema, - schemaId, - seqNumCounterSupplier, - fileIndexOptions, - fileSource, - asyncFileWrite, - statsDenseStore, - pathFactory.isExternalPath(), - writeCols, - rowSidecarFormat, - rowSidecarPath); - }, + () -> + createFileWriter( + fileIO, + schemaId, + fileFormat, + writeSchema, + pathFactory, + seqNumCounterSupplier, + fileCompression, + statsCollectors, + fileIndexOptions, + fileSource, + asyncFileWrite, + statsDenseStore, + writeCols, + rowSidecarFormat, + writePlanFactory), targetFileSize); } + + private static RowDataFileWriter createFileWriter( + FileIO fileIO, + long schemaId, + FileFormat fileFormat, + RowType writeSchema, + DataFilePathFactory pathFactory, + Supplier seqNumCounterSupplier, + String fileCompression, + SimpleColStatsCollector.Factory[] statsCollectors, + FileIndexOptions fileIndexOptions, + FileSource fileSource, + boolean asyncFileWrite, + boolean statsDenseStore, + @Nullable List writeCols, + @Nullable FileFormat rowSidecarFormat, + @Nullable RowDataFileWritePlanFactory writePlanFactory) { + RowDataFileWritePlan writePlan = + writePlanFactory == null + ? RowDataFileWritePlan.direct(writeSchema) + : writePlanFactory.create(); + Path dataPath = pathFactory.newPath(); + Path rowSidecarPath = + rowSidecarFormat == null + ? null + : new Path(dataPath.getParent(), dataPath.getName() + ".row"); + return new RowDataFileWriter( + fileIO, + RollingFileWriter.createFileWriterContext( + fileFormat, writePlan.physicalType(), statsCollectors, fileCompression), + dataPath, + writeSchema, + writePlan, + schemaId, + seqNumCounterSupplier, + fileIndexOptions, + fileSource, + asyncFileWrite, + statsDenseStore, + pathFactory.isExternalPath(), + writeCols, + rowSidecarFormat, + rowSidecarPath); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataTransform.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataTransform.java new file mode 100644 index 000000000000..45bd03d0cf7f --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataTransform.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.types.RowType; + +/** Transforms logical row data to the physical row data written into one data file. */ +public interface RowDataTransform { + + /** Returns the physical row type produced by this transform. */ + RowType physicalType(); + + /** Converts a logical row to the physical row written into the data file. */ + InternalRow transform(InternalRow row); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java index 473acecea9ed..5926e042c55d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java @@ -193,6 +193,7 @@ public void close() throws IOException { try { if (writer != null) { + beforeCloseFormatWriter(writer); writer.close(); writerMetadata = writer.writerMetadata(); writer = null; @@ -219,6 +220,8 @@ protected long outputBytes() throws IOException { return outputBytes; } + protected void beforeCloseFormatWriter(FormatWriter writer) throws IOException {} + /** * Returns cached writer metadata from the format writer. Available after {@link #close()} is * called. Can be used by stats extractors to avoid re-reading the file from object storage. diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/BaseAppendFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/BaseAppendFileStoreWrite.java index 574143192e19..44ce3cfb329b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/BaseAppendFileStoreWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/BaseAppendFileStoreWrite.java @@ -26,6 +26,8 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BlobConsumer; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.shredding.MapSharedShreddingContext; +import org.apache.paimon.data.shredding.MapSharedShreddingCoreUtils; import org.apache.paimon.deletionvectors.BucketedDvMaintainer; import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.fileindex.FileIndexOptions; @@ -33,6 +35,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.io.BundleRecords; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataFilePathFactory; import org.apache.paimon.io.RowDataRollingFileWriter; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.reader.RecordReaderIterator; @@ -127,6 +130,11 @@ protected RecordWriter createWriter( ExecutorService compactExecutor, @Nullable BucketedDvMaintainer dvMaintainer, boolean ignorePreviousFiles) { + DataFilePathFactory dataPathFactory = + pathFactory.createDataFilePathFactory(partition, bucket); + MapSharedShreddingContext sharedShreddingContext = + MapSharedShreddingCoreUtils.createAndRestoreContext( + writeType, restoredFiles, dataPathFactory, options, fileIO); return new AppendOnlyWriter( fileIO, ioManager, @@ -143,7 +151,7 @@ protected RecordWriter createWriter( // it is only for new files, no dv files -> createFilesIterator(partition, bucket, files, null), options.commitForceCompact(), - pathFactory.createDataFilePathFactory(partition, bucket), + dataPathFactory, restoreIncrement, options.useWriteBufferForAppend() || forceBufferSpill, options.writeBufferSpillable() || forceBufferSpill, @@ -156,7 +164,8 @@ protected RecordWriter createWriter( options.statsDenseStore(), options.dataEvolutionEnabled(), rowSidecarFileFormat(), - blobContext); + blobContext, + sharedShreddingContext); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index fb4f5862685a..3835db635b9f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -695,6 +695,11 @@ private static void validateFileIndex(TableSchema schema) { + "Only CHAR/VARCHAR/STRING is supported.", columnName, keyType); + checkArgument( + options.mapStorageLayout(columnName) != MapStorageLayout.SHARED_SHREDDING, + "Column '%s' is configured with map.storage-layout=shared-shredding, " + + "but MAP shared-shredding does not support nested file index.", + columnName); } for (String indexType : entry.getValue().keySet()) { diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java index 55965234bb59..34b11498c28e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java @@ -19,21 +19,33 @@ package org.apache.paimon.append; import org.apache.paimon.CoreOptions; +import org.apache.paimon.compact.NoopCompactManager; import org.apache.paimon.compression.CompressOptions; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryRowWriter; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.BinaryVector; import org.apache.paimon.data.BlobData; +import org.apache.paimon.data.GenericMap; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.data.shredding.MapSharedShreddingContext; +import org.apache.paimon.data.shredding.MapSharedShreddingCoreUtils; +import org.apache.paimon.data.shredding.MapSharedShreddingDefine; +import org.apache.paimon.data.shredding.MapSharedShreddingFieldMeta; +import org.apache.paimon.data.shredding.MapSharedShreddingUtils; import org.apache.paimon.disk.ChannelWithMeta; import org.apache.paimon.disk.ExternalBuffer; import org.apache.paimon.disk.IOManager; import org.apache.paimon.disk.RowBuffer; import org.apache.paimon.fileindex.FileIndexOptions; import org.apache.paimon.format.FileFormat; +import org.apache.paimon.format.FormatReaderContext; import org.apache.paimon.format.SimpleColStats; +import org.apache.paimon.format.SupportsReaderFieldMetadata; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; @@ -45,6 +57,8 @@ import org.apache.paimon.operation.BlobFileContext; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; +import org.apache.paimon.reader.FileRecordIterator; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.SimpleStatsConverter; @@ -54,6 +68,7 @@ import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; import org.apache.paimon.types.VarCharType; import org.apache.paimon.utils.CommitIncrement; @@ -66,6 +81,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.File; import java.io.IOException; @@ -74,10 +91,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -407,6 +427,838 @@ public void testSpillWorksAndMoreSmallFilesGenerated() throws Exception { }); } + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"parquet", "orc"}) + public void testWriteSharedShreddingMapFieldContent(String fileFormat) throws Exception { + RowType writeType = sharedShreddingTagsWriteType(); + Options rawOptions = sharedShreddingOptions("tags", 3); + + assertSharedShreddingAppendWrite( + fileFormat, + writeType, + rawOptions, + expectedSharedShreddingFile( + Arrays.asList( + sharedShreddingRow(1, "a", 10L, "b", 20L), + sharedShreddingRow(2, "c", 30L, "a", 40L, "b", 50L), + sharedShreddingRow(3, "a", 60L)), + shreddingColumns("tags", 3), + Arrays.asList( + expectedPhysicalRow( + 1, + sharedShreddingField( + "tags", + mapping(0, 1, -1), + values(10L, 20L, null), + null)), + expectedPhysicalRow( + 2, + sharedShreddingField( + "tags", + mapping(2, 0, 1), + values(30L, 40L, 50L), + null)), + expectedPhysicalRow( + 3, + sharedShreddingField( + "tags", + mapping(0, -1, -1), + values(60L, null, null), + null))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("a", 0, "b", 1, "c", 2), + fieldToColumns( + 0, columns(0, 1), + 1, columns(1, 2), + 2, columns(0)), + overflowFields(), + 3, + 3)))); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"parquet", "orc"}) + public void testWriteSharedShreddingWithFileIndex(String fileFormat) throws Exception { + RowType writeType = sharedShreddingTagsWriteType(); + Options rawOptions = sharedShreddingOptions("tags", 3); + rawOptions.setString("file-index.bloom-filter.columns", "id"); + SharedShreddingAppendContext context = + createSharedShreddingAppendContext(fileFormat, writeType, rawOptions); + AppendOnlyWriter writer = + createSharedShreddingAppendWriter( + writeType, context, SCHEMA_ID, -1L, new FileIndexOptions(context.options)); + + writer.write(sharedShreddingRow(1, "a", 10L)); + writer.write(sharedShreddingRow(2, "b", 20L)); + CommitIncrement increment = writer.prepareCommit(true); + writer.close(); + + DataFileMeta file = assertSingleNewFile(increment); + assertThat(file.embeddedIndex() != null || !file.extraFiles().isEmpty()).isTrue(); + assertSharedShreddingDataFile( + context, + writeType, + file, + expectedSharedShreddingFile( + Arrays.asList( + sharedShreddingRow(1, "a", 10L), sharedShreddingRow(2, "b", 20L)), + shreddingColumns("tags", 3), + Arrays.asList( + expectedPhysicalRow( + 1, + sharedShreddingField( + "tags", + mapping(0, -1, -1), + values(10L, null, null), + null)), + expectedPhysicalRow( + 2, + sharedShreddingField( + "tags", + mapping(1, -1, -1), + values(20L, null, null), + null))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("a", 0, "b", 1), + fieldToColumns( + 0, columns(0), + 1, columns(0)), + overflowFields(), + 3, + 1)))); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"parquet", "orc"}) + public void testSharedShreddingMapAllEmptyFirstFile(String fileFormat) throws Exception { + RowType writeType = sharedShreddingTagsWriteType(); + Options rawOptions = sharedShreddingOptions("tags", 3); + + assertSharedShreddingAppendWrite( + fileFormat, + writeType, + rawOptions, + expectedSharedShreddingFile( + Arrays.asList(sharedShreddingRow(1), sharedShreddingRow(2)), + shreddingColumns("tags", 3), + Arrays.asList( + expectedPhysicalRow( + 1, + sharedShreddingField( + "tags", + mapping(-1, -1, -1), + values(null, null, null), + null)), + expectedPhysicalRow( + 2, + sharedShreddingField( + "tags", + mapping(-1, -1, -1), + values(null, null, null), + null))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId(), fieldToColumns(), overflowFields(), 3, 0)))); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"parquet", "orc"}) + public void testSharedShreddingMapAllNullThenAllEmptyFiles(String fileFormat) throws Exception { + RowType writeType = sharedShreddingTagsWriteType(); + Options rawOptions = sharedShreddingOptions("tags", 3); + + assertSharedShreddingAppendWrite( + fileFormat, + writeType, + rawOptions, + expectedSharedShreddingFile( + Arrays.asList( + sharedShreddingLogicalRow(1, null), + sharedShreddingLogicalRow(2, null)), + shreddingColumns("tags", 3), + Arrays.asList( + expectedPhysicalRow(1, sharedShreddingNullField("tags")), + expectedPhysicalRow(2, sharedShreddingNullField("tags"))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId(), fieldToColumns(), overflowFields(), 3, 0))), + expectedSharedShreddingFile( + Arrays.asList(sharedShreddingRow(3), sharedShreddingRow(4)), + shreddingColumns("tags", 1), + Arrays.asList( + expectedPhysicalRow( + 3, + sharedShreddingField( + "tags", mapping(-1), values((Object) null), null)), + expectedPhysicalRow( + 4, + sharedShreddingField( + "tags", mapping(-1), values((Object) null), null))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId(), fieldToColumns(), overflowFields(), 1, 0))), + expectedSharedShreddingFile( + Arrays.asList( + sharedShreddingRow(5, "a", null), + sharedShreddingRow(6, "b", null), + sharedShreddingRow(7, "c", 7L, "d", null)), + shreddingColumns("tags", 1), + Arrays.asList( + expectedPhysicalRow( + 5, + sharedShreddingField( + "tags", mapping(0), values((Object) null), null)), + expectedPhysicalRow( + 6, + sharedShreddingField( + "tags", mapping(1), values((Object) null), null)), + expectedPhysicalRow( + 7, + sharedShreddingField( + "tags", + mapping(2), + values(7L), + overflow(3, null)))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("a", 0, "b", 1, "c", 2, "d", 3), + fieldToColumns( + 0, columns(0), + 1, columns(0), + 2, columns(0)), + overflowFields(3), + 1, + 2)))); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"parquet", "orc"}) + public void testWriteSharedShreddingMapWithOverflow(String fileFormat) throws Exception { + RowType writeType = sharedShreddingTagsWriteType(); + Options rawOptions = sharedShreddingOptions("tags", 2); + + assertSharedShreddingAppendWrite( + fileFormat, + writeType, + rawOptions, + expectedSharedShreddingFile( + Arrays.asList( + sharedShreddingRow(1, "a", 1L, "b", 2L), + sharedShreddingRow(2, "c", 3L, "a", 4L, "b", 5L), + sharedShreddingRow(3, "d", 6L, "e", 7L, "f", 8L, "a", 9L)), + shreddingColumns("tags", 2), + Arrays.asList( + expectedPhysicalRow( + 1, + sharedShreddingField( + "tags", mapping(0, 1), values(1L, 2L), null)), + expectedPhysicalRow( + 2, + sharedShreddingField( + "tags", + mapping(2, 0), + values(3L, 4L), + overflow(1, 5L))), + expectedPhysicalRow( + 3, + sharedShreddingField( + "tags", + mapping(3, 4), + values(6L, 7L), + overflow(5, 8L, 0, 9L)))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("a", 0, "b", 1, "c", 2, "d", 3, "e", 4, "f", 5), + fieldToColumns( + 0, columns(0, 1), + 1, columns(1), + 2, columns(0), + 3, columns(0), + 4, columns(1)), + overflowFields(0, 1, 5), + 2, + 4)))); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"parquet", "orc"}) + public void testSharedShreddingMapKAdaptationAcrossFiles(String fileFormat) throws Exception { + RowType writeType = sharedShreddingTagsWriteType(); + Options rawOptions = sharedShreddingOptions("tags", 10); + + assertSharedShreddingAppendWrite( + fileFormat, + writeType, + rawOptions, + expectedSharedShreddingFile( + Arrays.asList( + sharedShreddingRow(1, "a", 10L, "b", 20L), + sharedShreddingRow(2, "c", 30L, "a", 40L, "b", 50L)), + shreddingColumns("tags", 10), + Arrays.asList( + expectedPhysicalRow( + 1, + sharedShreddingField( + "tags", + mapping(0, 1, -1, -1, -1, -1, -1, -1, -1, -1), + paddedValues(10, 10L, 20L), + null)), + expectedPhysicalRow( + 2, + sharedShreddingField( + "tags", + mapping(2, 0, 1, -1, -1, -1, -1, -1, -1, -1), + paddedValues(10, 30L, 40L, 50L), + null))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("a", 0, "b", 1, "c", 2), + fieldToColumns( + 0, columns(0, 1), + 1, columns(1, 2), + 2, columns(0)), + overflowFields(), + 10, + 3))), + expectedSharedShreddingFile( + Collections.singletonList( + sharedShreddingRow( + 3, "x", 100L, "y", 200L, "z", 300L, "w", 400L, "v", 500L)), + shreddingColumns("tags", 3), + Collections.singletonList( + expectedPhysicalRow( + 3, + sharedShreddingField( + "tags", + mapping(0, 1, 2), + values(100L, 200L, 300L), + overflow(3, 400L, 4, 500L)))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("x", 0, "y", 1, "z", 2, "w", 3, "v", 4), + fieldToColumns( + 0, columns(0), + 1, columns(1), + 2, columns(2)), + overflowFields(3, 4), + 3, + 5))), + expectedSharedShreddingFile( + Collections.singletonList( + sharedShreddingRow( + 4, "p", 1000L, "q", 2000L, "r", 3000L, "s", 4000L)), + shreddingColumns("tags", 5), + Collections.singletonList( + expectedPhysicalRow( + 4, + sharedShreddingField( + "tags", + mapping(0, 1, 2, 3, -1), + paddedValues(5, 1000L, 2000L, 3000L, 4000L), + null))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("p", 0, "q", 1, "r", 2, "s", 3), + fieldToColumns( + 0, columns(0), + 1, columns(1), + 2, columns(2), + 3, columns(3)), + overflowFields(), + 5, + 4)))); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"parquet", "orc"}) + public void testMultipleSharedShreddingMapFieldsWithKAdaptation(String fileFormat) + throws Exception { + RowType writeType = sharedShreddingTagsAndAttrsWriteType(); + Options rawOptions = sharedShreddingOptions("tags", 8, "attrs", 4); + + assertSharedShreddingAppendWrite( + fileFormat, + writeType, + rawOptions, + expectedSharedShreddingFile( + Arrays.asList( + sharedShreddingLogicalRow( + 1, map("a", 10L, "b", 20L), map("x", "v1")), + sharedShreddingLogicalRow(2, map("a", 30L), map("x", "v2"))), + shreddingColumns("tags", 8, "attrs", 4), + Arrays.asList( + expectedPhysicalRow( + 1, + sharedShreddingField( + "tags", + mapping(0, 1, -1, -1, -1, -1, -1, -1), + paddedValues(8, 10L, 20L), + null), + sharedShreddingField( + "attrs", + mapping(0, -1, -1, -1), + paddedValues(4, "v1"), + null)), + expectedPhysicalRow( + 2, + sharedShreddingField( + "tags", + mapping(0, -1, -1, -1, -1, -1, -1, -1), + paddedValues(8, 30L), + null), + sharedShreddingField( + "attrs", + mapping(0, -1, -1, -1), + paddedValues(4, "v2"), + null))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("a", 0, "b", 1), + fieldToColumns(0, columns(0), 1, columns(1)), + overflowFields(), + 8, + 2), + "attrs", + sharedShreddingMeta( + nameToId("x", 0), + fieldToColumns(0, columns(0)), + overflowFields(), + 4, + 1))), + expectedSharedShreddingFile( + Collections.singletonList( + sharedShreddingLogicalRow( + 3, + map("c", 100L, "d", 200L, "e", 300L), + map("p", "a1", "q", "a2", "r", "a3"))), + shreddingColumns("tags", 2, "attrs", 1), + Collections.singletonList( + expectedPhysicalRow( + 3, + sharedShreddingField( + "tags", + mapping(0, 1), + values(100L, 200L), + overflow(2, 300L)), + sharedShreddingField( + "attrs", + mapping(0), + values("a1"), + overflow(1, "a2", 2, "a3")))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("c", 0, "d", 1, "e", 2), + fieldToColumns(0, columns(0), 1, columns(1)), + overflowFields(2), + 2, + 3), + "attrs", + sharedShreddingMeta( + nameToId("p", 0, "q", 1, "r", 2), + fieldToColumns(0, columns(0)), + overflowFields(1, 2), + 1, + 3))), + expectedSharedShreddingFile( + Collections.singletonList( + sharedShreddingLogicalRow( + 4, map("f", 400L, "g", 500L), map("s", "b1", "t", "b2"))), + shreddingColumns("tags", 3, "attrs", 3), + Collections.singletonList( + expectedPhysicalRow( + 4, + sharedShreddingField( + "tags", + mapping(0, 1, -1), + paddedValues(3, 400L, 500L), + null), + sharedShreddingField( + "attrs", + mapping(0, 1, -1), + paddedValues(3, "b1", "b2"), + null))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("f", 0, "g", 1), + fieldToColumns(0, columns(0), 1, columns(1)), + overflowFields(), + 3, + 2), + "attrs", + sharedShreddingMeta( + nameToId("s", 0, "t", 1), + fieldToColumns(0, columns(0), 1, columns(1)), + overflowFields(), + 3, + 2)))); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"parquet", "orc"}) + public void testSharedShreddingMapDataFileMetaInfo(String fileFormat) throws Exception { + RowType writeType = sharedShreddingTagsWriteType(); + Options rawOptions = sharedShreddingOptions("tags", 3); + rawOptions.setString("metadata.stats-mode", "full"); + SharedShreddingAppendContext context = + createSharedShreddingAppendContext(fileFormat, writeType, rawOptions); + AppendOnlyWriter writer = createSharedShreddingAppendWriter(writeType, context, 5L, 9L); + ExpectedSharedShreddingFile expectedFile = + expectedSharedShreddingFile( + Arrays.asList( + sharedShreddingRow(1, "a", 10L, "b", 20L), + sharedShreddingLogicalRow(2, null), + sharedShreddingRow(3, "a", 40L, "b", 50L, "c", 60L)), + shreddingColumns("tags", 3), + Arrays.asList( + expectedPhysicalRow( + 1, + sharedShreddingField( + "tags", + mapping(0, 1, -1), + values(10L, 20L, null), + null)), + expectedPhysicalRow(2, sharedShreddingNullField("tags")), + expectedPhysicalRow( + 3, + sharedShreddingField( + "tags", + mapping(0, 1, 2), + values(40L, 50L, 60L), + null))), + shreddingMetas( + "tags", + sharedShreddingMeta( + nameToId("a", 0, "b", 1, "c", 2), + fieldToColumns( + 0, columns(0), + 1, columns(1), + 2, columns(2)), + overflowFields(), + 3, + 3))); + + for (InternalRow row : expectedFile.rows) { + writer.write(row); + } + CommitIncrement increment = writer.prepareCommit(true); + writer.close(); + + DataFileMeta file = assertSingleNewFile(increment); + assertThat(file.rowCount()).isEqualTo(3L); + assertThat(file.schemaId()).isEqualTo(5L); + assertThat(file.minSequenceNumber()).isEqualTo(10L); + assertThat(file.maxSequenceNumber()).isEqualTo(12L); + assertThat(file.level()).isEqualTo(DataFileMeta.DUMMY_LEVEL); + assertThat(file.minKey()).isEqualTo(EMPTY_ROW); + assertThat(file.maxKey()).isEqualTo(EMPTY_ROW); + assertThat(file.keyStats()).isEqualTo(EMPTY_STATS); + assertThat(file.valueStats().minValues().getInt(0)).isEqualTo(1); + assertThat(file.valueStats().maxValues().getInt(0)).isEqualTo(3); + assertThat(file.valueStats().nullCounts().getLong(0)).isEqualTo(0L); + assertThat(file.valueStats().minValues().isNullAt(1)).isTrue(); + assertThat(file.valueStats().maxValues().isNullAt(1)).isTrue(); + if ("orc".equals(fileFormat)) { + assertThat(file.valueStats().nullCounts().getLong(1)).isEqualTo(1L); + } else { + assertThat(file.valueStats().nullCounts().isNullAt(1)).isTrue(); + } + assertThat(file.fileSource()).hasValue(FileSource.APPEND); + assertThat(file.fileFormat()).isEqualTo(fileFormat); + assertSharedShreddingDataFile(context, writeType, file, expectedFile); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"parquet", "orc"}) + public void testSharedShreddingMapDoesNotSupportBlob(String fileFormat) throws Exception { + RowType writeType = + DataTypes.ROW( + DataTypes.FIELD(0, "id", DataTypes.INT()), + DataTypes.FIELD( + 1, "tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())), + DataTypes.FIELD(2, "payload", new BlobType())); + Options rawOptions = sharedShreddingOptions("tags", 3); + SharedShreddingAppendContext context = + createSharedShreddingAppendContext(fileFormat, writeType, rawOptions); + BlobFileContext blobContext = + BlobFileContext.create(writeType, new CoreOptions(rawOptions)); + assertThat(blobContext).isNotNull(); + + AppendOnlyWriter writer = + createSharedShreddingAppendWriter( + writeType, context, SCHEMA_ID, -1L, new FileIndexOptions(), blobContext); + + Assertions.assertThatThrownBy( + () -> + writer.write( + sharedShreddingLogicalRow( + 1, + map("a", 10L), + new BlobData(new byte[] {1, 2, 3})))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining( + "MAP shared-shredding does not support blob or vector file writes yet."); + writer.close(); + } + + private void assertSharedShreddingAppendWrite( + String fileFormat, + RowType writeType, + Options rawOptions, + ExpectedSharedShreddingFile... expectedFiles) + throws Exception { + SharedShreddingAppendContext context = + createSharedShreddingAppendContext(fileFormat, writeType, rawOptions); + AppendOnlyWriter writer = createSharedShreddingAppendWriter(writeType, context); + for (ExpectedSharedShreddingFile expectedFile : expectedFiles) { + for (InternalRow row : expectedFile.rows) { + writer.write(row); + } + CommitIncrement increment = writer.prepareCommit(true); + DataFileMeta file = assertSingleNewFile(increment); + assertSharedShreddingDataFile(context, writeType, file, expectedFile); + } + writer.close(); + } + + private SharedShreddingAppendContext createSharedShreddingAppendContext( + String fileFormat, RowType writeType, Options rawOptions) { + CoreOptions options = new CoreOptions(rawOptions); + DataFilePathFactory pathFactory = createPathFactory(fileFormat); + LocalFileIO fileIO = LocalFileIO.create(); + return new SharedShreddingAppendContext( + options, + pathFactory, + fileIO, + FileFormat.fromIdentifier(fileFormat, new Options()), + MapSharedShreddingCoreUtils.createAndRestoreContext( + writeType, Collections.emptyList(), pathFactory, options, fileIO)); + } + + private AppendOnlyWriter createSharedShreddingAppendWriter( + RowType writeType, SharedShreddingAppendContext context) { + return createSharedShreddingAppendWriter(writeType, context, SCHEMA_ID, -1L); + } + + private AppendOnlyWriter createSharedShreddingAppendWriter( + RowType writeType, + SharedShreddingAppendContext context, + long schemaId, + long maxSequenceNumber) { + return createSharedShreddingAppendWriter( + writeType, context, schemaId, maxSequenceNumber, new FileIndexOptions()); + } + + private AppendOnlyWriter createSharedShreddingAppendWriter( + RowType writeType, + SharedShreddingAppendContext context, + long schemaId, + long maxSequenceNumber, + FileIndexOptions fileIndexOptions) { + return createSharedShreddingAppendWriter( + writeType, context, schemaId, maxSequenceNumber, fileIndexOptions, null); + } + + private AppendOnlyWriter createSharedShreddingAppendWriter( + RowType writeType, + SharedShreddingAppendContext context, + long schemaId, + long maxSequenceNumber, + FileIndexOptions fileIndexOptions, + BlobFileContext blobContext) { + return new AppendOnlyWriter( + context.fileIO, + null, + schemaId, + context.format, + null, + 1024 * 1024L, + 1024 * 1024L, + 1024 * 1024L, + writeType, + null, + maxSequenceNumber, + new NoopCompactManager(), + null, + false, + context.pathFactory, + null, + false, + false, + CoreOptions.FILE_COMPRESSION.defaultValue(), + CompressOptions.defaultOptions(), + new StatsCollectorFactories(context.options), + MemorySize.MAX_VALUE, + fileIndexOptions, + true, + false, + false, + null, + blobContext, + context.sharedShreddingContext); + } + + private DataFileMeta assertSingleNewFile(CommitIncrement increment) { + assertThat(increment.newFilesIncrement().newFiles()).hasSize(1); + return increment.newFilesIncrement().newFiles().get(0); + } + + private void assertSharedShreddingDataFile( + SharedShreddingAppendContext context, + RowType writeType, + DataFileMeta file, + ExpectedSharedShreddingFile expectedFile) + throws IOException { + Path path = context.pathFactory.toPath(file); + RowType physicalType = + MapSharedShreddingUtils.logicalToPhysicalSchema( + writeType, expectedFile.shreddingColumns); + assertSharedShreddingPhysicalRows( + context.format, + context.fileIO, + path, + file.fileSize(), + physicalType, + expectedFile.expectedRows); + assertSharedShreddingFileSchema( + context, path, file.fileSize(), physicalType, expectedFile.expectedMetas); + } + + private void assertSharedShreddingFileSchema( + SharedShreddingAppendContext context, + Path path, + long fileSize, + RowType expectedPhysicalType, + Map expectedMetas) + throws IOException { + RowType emptyRowType = new RowType(Collections.emptyList()); + try (FileRecordReader reader = + context.format + .createReaderFactory(emptyRowType, emptyRowType, Collections.emptyList()) + .createReader(new FormatReaderContext(context.fileIO, path, fileSize))) { + Map> fieldMetadata = + ((SupportsReaderFieldMetadata) reader).readFieldMetadata(); + for (Map.Entry entry : expectedMetas.entrySet()) { + String fieldName = entry.getKey(); + assertThat(expectedPhysicalType.containsField(fieldName)).isTrue(); + assertThat(readSharedShreddingFieldMeta(fieldMetadata, fieldName)) + .isEqualTo(entry.getValue()); + } + } + } + + private void assertSharedShreddingPhysicalRows( + FileFormat format, + LocalFileIO fileIO, + Path path, + long fileSize, + RowType physicalType, + List expectedRows) + throws IOException { + InternalRowSerializer serializer = new InternalRowSerializer(physicalType); + List rows = new ArrayList<>(); + try (FileRecordReader reader = + format.createReaderFactory(physicalType, physicalType, Collections.emptyList()) + .createReader(new FormatReaderContext(fileIO, path, fileSize))) { + FileRecordIterator batch; + while ((batch = reader.readBatch()) != null) { + InternalRow row; + while ((row = batch.next()) != null) { + rows.add(serializer.copy(row)); + } + batch.releaseBatch(); + } + } + + assertThat(rows).hasSize(expectedRows.size()); + for (int i = 0; i < expectedRows.size(); i++) { + assertSharedShreddingPhysicalRow(rows.get(i), physicalType, expectedRows.get(i)); + } + } + + private MapSharedShreddingFieldMeta readSharedShreddingFieldMeta( + Map> fieldMetadata, String fieldName) { + return MapSharedShreddingUtils.deserializeMetadata( + fieldMetadata.get(fieldName), MapSharedShreddingDefine.DEFAULT_DICT_COMPRESSION); + } + + private void assertSharedShreddingPhysicalRow( + InternalRow row, RowType physicalType, ExpectedPhysicalRow expectedRow) { + assertThat(row.getInt(physicalType.getFieldIndex("id"))).isEqualTo(expectedRow.id); + for (ExpectedShreddingField expectedField : expectedRow.shreddingFields) { + int fieldIndex = physicalType.getFieldIndex(expectedField.fieldName); + if (expectedField.fieldIsNull) { + assertThat(row.isNullAt(fieldIndex)).isTrue(); + continue; + } + RowType fieldType = (RowType) physicalType.getTypeAt(fieldIndex); + InternalRow field = row.getRow(fieldIndex, fieldType.getFieldCount()); + assertThat(field.getArray(0).toIntArray()).containsExactly(expectedField.fieldMapping); + for (int i = 0; i < expectedField.columnValues.length; i++) { + assertFieldValue(field, i + 1, expectedField.columnValues[i]); + } + if (expectedField.overflow == null) { + assertThat(field.isNullAt(fieldType.getFieldCount() - 1)).isTrue(); + } else { + MapType overflowType = (MapType) fieldType.getTypeAt(fieldType.getFieldCount() - 1); + assertInternalMap( + field.getMap(fieldType.getFieldCount() - 1), + expectedField.overflow, + overflowType); + } + } + } + + private void assertFieldValue(InternalRow row, int pos, Object expected) { + if (expected == null) { + assertThat(row.isNullAt(pos)).isTrue(); + } else if (expected instanceof Long) { + assertThat(row.getLong(pos)).isEqualTo(expected); + } else if (expected instanceof Integer) { + assertThat(row.getInt(pos)).isEqualTo(expected); + } else if (expected instanceof String) { + assertThat(row.getString(pos)).isEqualTo(BinaryString.fromString((String) expected)); + } else if (expected instanceof BinaryString) { + assertThat(row.getString(pos)).isEqualTo(expected); + } else { + assertThat(row.getRow(pos, ((InternalRow) expected).getFieldCount())) + .isEqualTo(expected); + } + } + + private void assertInternalMap(InternalMap actual, InternalMap expected, MapType mapType) { + assertThat(toJavaMap(actual, mapType)).isEqualTo(toJavaMap(expected, mapType)); + } + + private Map toJavaMap(InternalMap map, MapType mapType) { + InternalArray.ElementGetter keyGetter = + InternalArray.createElementGetter(mapType.getKeyType()); + InternalArray.ElementGetter valueGetter = + InternalArray.createElementGetter(mapType.getValueType()); + Map result = new LinkedHashMap<>(); + InternalArray keys = map.keyArray(); + InternalArray values = map.valueArray(); + for (int i = 0; i < map.size(); i++) { + result.put( + keyGetter.getElementOrNull(keys, i), valueGetter.getElementOrNull(values, i)); + } + return result; + } + @Test public void testNoSpillWhenMeetBlobType() throws Exception { // Create a schema with BLOB type @@ -647,6 +1499,172 @@ private InternalRow row(int id, String name, String dt) { return GenericRow.of(id, BinaryString.fromString(name), BinaryString.fromString(dt)); } + private RowType sharedShreddingTagsWriteType() { + return DataTypes.ROW( + DataTypes.FIELD(0, "id", DataTypes.INT()), + DataTypes.FIELD(1, "tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT()))); + } + + private RowType sharedShreddingTagsAndAttrsWriteType() { + return DataTypes.ROW( + DataTypes.FIELD(0, "id", DataTypes.INT()), + DataTypes.FIELD(1, "tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())), + DataTypes.FIELD(2, "attrs", DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()))); + } + + private Options sharedShreddingOptions(Object... fieldToMaxColumns) { + Options options = new Options(); + for (int i = 0; i < fieldToMaxColumns.length; i += 2) { + String fieldName = (String) fieldToMaxColumns[i]; + options.setString("fields." + fieldName + ".map.storage-layout", "shared-shredding"); + options.setString( + "fields." + fieldName + ".map.shared-shredding.max-columns", + String.valueOf(fieldToMaxColumns[i + 1])); + } + options.setString("metadata.stats-mode", "none"); + return options; + } + + private InternalRow sharedShreddingRow(int id, Object... keyValues) { + return sharedShreddingLogicalRow(id, map(keyValues)); + } + + private InternalRow sharedShreddingLogicalRow(int id, Object... fields) { + if (fields == null) { + fields = new Object[] {null}; + } + GenericRow row = new GenericRow(fields.length + 1); + row.setField(0, id); + for (int i = 0; i < fields.length; i++) { + row.setField(i + 1, fields[i]); + } + return row; + } + + private InternalMap map(Object... keyValues) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + map.put( + BinaryString.fromString((String) keyValues[i]), + internalValue(keyValues[i + 1])); + } + return new GenericMap(map); + } + + private InternalMap overflow(Object... idValues) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < idValues.length; i += 2) { + map.put((Integer) idValues[i], internalValue(idValues[i + 1])); + } + return new GenericMap(map); + } + + private Object internalValue(Object value) { + if (value instanceof String) { + return BinaryString.fromString((String) value); + } + return value; + } + + private MapSharedShreddingFieldMeta sharedShreddingMeta( + Map nameToId, + Map> fieldToColumns, + Set overflowFields, + int numColumns, + int maxRowWidth) { + return new MapSharedShreddingFieldMeta( + new TreeMap<>(nameToId), + new TreeMap<>(fieldToColumns), + new TreeSet<>(overflowFields), + numColumns, + maxRowWidth); + } + + private Map shreddingColumns(Object... pairs) { + Map result = new TreeMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + result.put((String) pairs[i], (Integer) pairs[i + 1]); + } + return result; + } + + private Map shreddingMetas(Object... pairs) { + Map result = new TreeMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + result.put((String) pairs[i], (MapSharedShreddingFieldMeta) pairs[i + 1]); + } + return result; + } + + private Map nameToId(Object... pairs) { + Map result = new TreeMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + result.put((String) pairs[i], (Integer) pairs[i + 1]); + } + return result; + } + + @SuppressWarnings("unchecked") + private Map> fieldToColumns(Object... pairs) { + Map> result = new TreeMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + result.put((Integer) pairs[i], (List) pairs[i + 1]); + } + return result; + } + + private int[] mapping(int... fieldIds) { + return fieldIds; + } + + private Object[] values(Object... values) { + return values; + } + + private Object[] paddedValues(int length, Object... values) { + Object[] result = new Object[length]; + System.arraycopy(values, 0, result, 0, values.length); + return result; + } + + private List columns(int... columns) { + List result = new ArrayList<>(); + for (int column : columns) { + result.add(column); + } + return result; + } + + private Set overflowFields(int... fieldIds) { + Set result = new TreeSet<>(); + for (int fieldId : fieldIds) { + result.add(fieldId); + } + return result; + } + + private ExpectedPhysicalRow expectedPhysicalRow( + int id, ExpectedShreddingField... shreddingFields) { + return new ExpectedPhysicalRow(id, shreddingFields); + } + + private ExpectedSharedShreddingFile expectedSharedShreddingFile( + List rows, + Map shreddingColumns, + List expectedRows, + Map expectedMetas) { + return new ExpectedSharedShreddingFile(rows, shreddingColumns, expectedRows, expectedMetas); + } + + private ExpectedShreddingField sharedShreddingNullField(String fieldName) { + return new ExpectedShreddingField(fieldName); + } + + private ExpectedShreddingField sharedShreddingField( + String fieldName, int[] fieldMapping, Object[] columnValues, InternalMap overflow) { + return new ExpectedShreddingField(fieldName, fieldMapping, columnValues, overflow); + } + private InternalRow rowWithVectors(int id, String name, float[]... vectors) { GenericRow row = new GenericRow(vectors.length + 2); row.setField(0, id); @@ -658,9 +1676,13 @@ private InternalRow rowWithVectors(int id, String name, float[]... vectors) { } private DataFilePathFactory createPathFactory() { + return createPathFactory(CoreOptions.FILE_FORMAT_AVRO); + } + + private DataFilePathFactory createPathFactory(String fileFormat) { return new DataFilePathFactory( new Path(tempDir + "/dt=" + PART + "/bucket-0"), - CoreOptions.FILE_FORMAT_AVRO, + fileFormat, CoreOptions.DATA_FILE_PREFIX.defaultValue(), CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), @@ -668,6 +1690,84 @@ private DataFilePathFactory createPathFactory() { null); } + private static class SharedShreddingAppendContext { + + private final CoreOptions options; + private final DataFilePathFactory pathFactory; + private final LocalFileIO fileIO; + private final FileFormat format; + private final MapSharedShreddingContext sharedShreddingContext; + + private SharedShreddingAppendContext( + CoreOptions options, + DataFilePathFactory pathFactory, + LocalFileIO fileIO, + FileFormat format, + MapSharedShreddingContext sharedShreddingContext) { + this.options = options; + this.pathFactory = pathFactory; + this.fileIO = fileIO; + this.format = format; + this.sharedShreddingContext = sharedShreddingContext; + } + } + + private static class ExpectedSharedShreddingFile { + + private final List rows; + private final Map shreddingColumns; + private final List expectedRows; + private final Map expectedMetas; + + private ExpectedSharedShreddingFile( + List rows, + Map shreddingColumns, + List expectedRows, + Map expectedMetas) { + this.rows = rows; + this.shreddingColumns = shreddingColumns; + this.expectedRows = expectedRows; + this.expectedMetas = expectedMetas; + } + } + + private static class ExpectedPhysicalRow { + + private final int id; + private final ExpectedShreddingField[] shreddingFields; + + private ExpectedPhysicalRow(int id, ExpectedShreddingField[] shreddingFields) { + this.id = id; + this.shreddingFields = shreddingFields; + } + } + + private static class ExpectedShreddingField { + + private final String fieldName; + private final boolean fieldIsNull; + private final int[] fieldMapping; + private final Object[] columnValues; + private final InternalMap overflow; + + private ExpectedShreddingField(String fieldName) { + this.fieldName = fieldName; + this.fieldIsNull = true; + this.fieldMapping = null; + this.columnValues = null; + this.overflow = null; + } + + private ExpectedShreddingField( + String fieldName, int[] fieldMapping, Object[] columnValues, InternalMap overflow) { + this.fieldName = fieldName; + this.fieldIsNull = false; + this.fieldMapping = fieldMapping; + this.columnValues = columnValues; + this.overflow = overflow; + } + } + private AppendOnlyWriter createEmptyWriter(long targetFileSize) { return createWriter(targetFileSize, false, false, false, Collections.emptyList()).getLeft(); } @@ -828,7 +1928,8 @@ private Pair> createWriterBase( false, options.dataEvolutionEnabled(), null, - BlobFileContext.create(writeSchema, options)); + BlobFileContext.create(writeSchema, options), + null); writer.setMemoryPool( new HeapMemorySegmentPool(options.writeBufferSize(), options.pageSize())); return Pair.of(writer, compactManager.allFiles()); diff --git a/paimon-core/src/test/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtilsTest.java new file mode 100644 index 000000000000..406f17033fcf --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtilsTest.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data.shredding; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.format.FileFormat; +import org.apache.paimon.format.FormatMetadataUtils; +import org.apache.paimon.format.FormatWriter; +import org.apache.paimon.format.FormatWriterFactory; +import org.apache.paimon.format.SupportsWriterMetadata; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataFilePathFactory; +import org.apache.paimon.options.Options; +import org.apache.paimon.stats.SimpleStats; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link MapSharedShreddingCoreUtils}. */ +public class MapSharedShreddingCoreUtilsTest { + + private static final RowType WRITE_TYPE = + DataTypes.ROW( + DataTypes.FIELD(0, "id", DataTypes.INT()), + DataTypes.FIELD( + 1, "tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())), + DataTypes.FIELD( + 2, "attrs", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))); + + @TempDir java.nio.file.Path tempDir; + + @Test + public void testNoShreddingColumnsReturnsNull() { + MapSharedShreddingContext context = + MapSharedShreddingCoreUtils.createAndRestoreContext( + WRITE_TYPE, + Collections.emptyList(), + testPathFactory(), + new CoreOptions(new Options()), + null); + + assertThat(context).isNull(); + } + + @Test + public void testNoRestoredFilesUsesConfiguredMaxColumns() { + MapSharedShreddingContext context = + MapSharedShreddingCoreUtils.createAndRestoreContext( + WRITE_TYPE, + Collections.emptyList(), + testPathFactory(), + options("tags", 128), + null); + + assertThat(context).isNotNull(); + assertThat(context.computeNextK()).containsEntry("tags", 128); + } + + @Test + public void testRestoreLatestFieldMetadata() throws IOException { + DataFilePathFactory pathFactory = testPathFactory(); + FileIO fileIO = LocalFileIO.create(); + DataFileMeta olderFile = + writeParquetFile( + pathFactory, fileIO, "older.parquet", null, schemaWithField("tags", 7)); + DataFileMeta newerFile = + writeParquetFile( + pathFactory, fileIO, "newer.parquet", null, schemaWithField("tags", 23)); + + MapSharedShreddingContext context = + MapSharedShreddingCoreUtils.createAndRestoreContext( + WRITE_TYPE, + Arrays.asList(olderFile, newerFile), + pathFactory, + options("tags", 128), + fileIO); + + assertThat(context).isNotNull(); + assertThat(context.computeNextK()).containsEntry("tags", 23); + } + + @Test + public void testRestoreMultipleFieldsIndependently() throws IOException { + DataFilePathFactory pathFactory = testPathFactory(); + FileIO fileIO = LocalFileIO.create(); + DataFileMeta tagsFile = + writeParquetFile( + pathFactory, + fileIO, + "tags.parquet", + Collections.singletonList("tags"), + schemaWithField("tags", 17)); + DataFileMeta attrsFile = + writeParquetFile( + pathFactory, + fileIO, + "attrs.parquet", + Collections.singletonList("attrs"), + schemaWithField("attrs", 9)); + + MapSharedShreddingContext context = + MapSharedShreddingCoreUtils.createAndRestoreContext( + WRITE_TYPE, + Arrays.asList(tagsFile, attrsFile), + pathFactory, + options(Arrays.asList("tags", "attrs"), Arrays.asList(128, 64)), + fileIO); + + assertThat(context).isNotNull(); + assertThat(context.computeNextK()).containsEntry("tags", 17).containsEntry("attrs", 9); + } + + @Test + public void testWriteColsFilterSkipsUnrelatedFiles() throws IOException { + DataFilePathFactory pathFactory = testPathFactory(); + FileIO fileIO = LocalFileIO.create(); + DataFileMeta tagsFile = + writeParquetFile( + pathFactory, + fileIO, + "tags.parquet", + Collections.singletonList("tags"), + schemaWithField("tags", 19)); + + MapSharedShreddingContext context = + MapSharedShreddingCoreUtils.createAndRestoreContext( + WRITE_TYPE, + Arrays.asList( + tagsFile, dataFile("id.parquet", Collections.singletonList("id"))), + pathFactory, + options("tags", 128), + fileIO); + + assertThat(context).isNotNull(); + assertThat(context.computeNextK()).containsEntry("tags", 19); + } + + @Test + public void testArrowSchemaWithoutSharedShreddingMetadataIsSkipped() throws IOException { + DataFilePathFactory pathFactory = testPathFactory(); + FileIO fileIO = LocalFileIO.create(); + byte[] schema = + FormatMetadataUtils.buildArrowSchemaMetadata( + WRITE_TYPE, + Collections.singletonMap( + "tags", Collections.singletonMap("custom.key", "value")), + FormatMetadataUtils.PARQUET_FIELD_ID_KEY); + DataFileMeta file = + writeParquetFile(pathFactory, fileIO, "plain-metadata.parquet", null, schema); + + MapSharedShreddingContext context = + MapSharedShreddingCoreUtils.createAndRestoreContext( + WRITE_TYPE, + Collections.singletonList(file), + pathFactory, + options("tags", 128), + fileIO); + + assertThat(context).isNotNull(); + assertThat(context.computeNextK()).containsEntry("tags", 128); + } + + private static CoreOptions options(String fieldName, int maxColumns) { + return options(Collections.singletonList(fieldName), Collections.singletonList(maxColumns)); + } + + private static CoreOptions options(List fieldNames, List maxColumns) { + Options options = new Options(); + for (int i = 0; i < fieldNames.size(); i++) { + String fieldName = fieldNames.get(i); + options.setString("fields." + fieldName + ".map.storage-layout", "shared-shredding"); + options.setString( + "fields." + fieldName + ".map.shared-shredding.max-columns", + String.valueOf(maxColumns.get(i))); + } + return new CoreOptions(options); + } + + private static byte[] schemaWithField(String fieldName, int maxRowWidth) { + return schemaWithField(WRITE_TYPE, fieldName, maxRowWidth); + } + + private static byte[] schemaWithField(RowType rowType, String fieldName, int maxRowWidth) { + return FormatMetadataUtils.buildArrowSchemaMetadata( + rowType, + Collections.singletonMap(fieldName, fieldMetadata(maxRowWidth)), + FormatMetadataUtils.PARQUET_FIELD_ID_KEY); + } + + private static Map fieldMetadata(int maxRowWidth) { + Map nameToId = new TreeMap<>(); + nameToId.put("k", 0); + + Map> fieldToColumns = new TreeMap<>(); + fieldToColumns.put(0, Collections.singletonList(0)); + + Set overflowFieldSet = new TreeSet<>(); + Map metadata = new LinkedHashMap<>(); + MapSharedShreddingUtils.serializeMetadata( + new MapSharedShreddingFieldMeta( + nameToId, fieldToColumns, overflowFieldSet, 128, maxRowWidth), + MapSharedShreddingDefine.DEFAULT_DICT_COMPRESSION, + metadata); + return metadata; + } + + private static DataFileMeta dataFile(String fileName, @Nullable List writeCols) { + return dataFile(fileName, writeCols, 10L, 0L); + } + + private static DataFileMeta dataFile( + String fileName, @Nullable List writeCols, long fileSize, long schemaId) { + return DataFileMeta.forAppend( + fileName, + fileSize, + 1L, + SimpleStats.EMPTY_STATS, + 0L, + 0L, + schemaId, + Collections.emptyList(), + null, + null, + null, + null, + null, + writeCols); + } + + private static DataFileMeta writeParquetFile( + DataFilePathFactory pathFactory, + FileIO fileIO, + String fileName, + @Nullable List writeCols, + byte[] arrowSchema) + throws IOException { + Path path = pathFactory.toPath(dataFile(fileName, null)); + FileFormat format = FileFormat.fromIdentifier("parquet", new Options()); + FormatWriterFactory writerFactory = format.createWriterFactory(WRITE_TYPE); + try (PositionOutputStream out = fileIO.newOutputStream(path, false); + FormatWriter writer = writerFactory.create(out, "zstd")) { + ((SupportsWriterMetadata) writer) + .addMetadata( + Collections.singletonMap( + FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY, arrowSchema)); + writer.addElement( + GenericRow.of( + 1, + new GenericMap( + Collections.singletonMap(BinaryString.fromString("k"), 1L)), + new GenericMap( + Collections.singletonMap(BinaryString.fromString("k"), 1)))); + } + return dataFile(fileName, writeCols, fileIO.getFileSize(path), 0L); + } + + private DataFilePathFactory testPathFactory() { + return new DataFilePathFactory( + new Path(tempDir.toUri()), "parquet", "data-", "changelog-", false, "none", null); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java index d295ed575e40..266badd090b7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java @@ -296,7 +296,8 @@ public void testFileSuffix(@TempDir java.nio.file.Path tempDir) throws Exception false, options.dataEvolutionEnabled(), null, - BlobFileContext.create(schema, options)); + BlobFileContext.create(schema, options), + null); appendOnlyWriter.setMemoryPool( new HeapMemorySegmentPool(options.writeBufferSize(), options.pageSize())); appendOnlyWriter.write( diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java index ce714e27078f..794cea32d01f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java @@ -573,6 +573,14 @@ public void testFileIndexNestedColumn() { .hasMessageContaining( "Column 'mi' is configured as nested column in 'file-index..columns', but its map key type is INT. Only CHAR/VARCHAR/STRING is supported."); } + + Map sharedShreddingOptions = new HashMap<>(); + sharedShreddingOptions.put("file-index.bloom-filter.columns", "m[k]"); + sharedShreddingOptions.put("fields.m.map.storage-layout", "shared-shredding"); + assertThatThrownBy(() -> validateTableSchemaWithMapField(sharedShreddingOptions)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining( + "Column 'm' is configured with map.storage-layout=shared-shredding, but MAP shared-shredding does not support nested file index."); } private void validateTableSchemaWithMapField(Map options) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java b/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java index 371d6d8593ec..e8abaae336a8 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java @@ -63,11 +63,11 @@ * Minimal Arrow IPC schema metadata encoder and decoder used by format metadata. * *

NOTE: The RowType-to-Arrow-field conversion in this class is copied from {@code - * org.apache.paimon.arrow.ArrowUtils} and must be kept in sync with that class. The IPC - * serialization code is a minimal implementation of the public Arrow IPC FlatBuffers layout used by - * {@code ARROW:schema}. It implements only the subset needed by Paimon field metadata so that - * {@code paimon-format} can stay compatible with Arrow metadata without depending on the Arrow - * runtime. + * org.apache.paimon.arrow.ArrowUtils} and must be kept in sync with that class. The Arrow IPC + * FlatBuffers layout, enum values, and defaults are adapted from Apache Arrow Java / Arrow format + * generated classes. This class implements only the subset needed by Paimon field metadata so that + * {@code paimon-format} can stay compatible with {@code ARROW:schema} without depending on the + * Arrow runtime. */ class ArrowSchemaMetadata { @@ -102,12 +102,16 @@ class ArrowSchemaMetadata { private static final short PRECISION_DOUBLE = 2; private static final short DATE_UNIT_DAY = 0; + private static final short DATE_UNIT_MILLISECOND = 1; private static final short TIME_UNIT_SECOND = 0; private static final short TIME_UNIT_MILLISECOND = 1; private static final short TIME_UNIT_MICROSECOND = 2; private static final short TIME_UNIT_NANOSECOND = 3; + private static final int TIME_BIT_WIDTH_MILLISECOND = 32; + private static final int DECIMAL_BIT_WIDTH_128 = 128; + private ArrowSchemaMetadata() {} static byte[] serialize( @@ -242,16 +246,16 @@ private static int buildType(FlatBufferBuilder builder, ArrowTypeInfo type) { builder.startTable(3); builder.addInt(0, type.precisionValue, 0); builder.addInt(1, type.scale, 0); - builder.addInt(2, type.bitWidth, 0); + builder.addInt(2, type.bitWidth, DECIMAL_BIT_WIDTH_128); return builder.endTable(); case TYPE_DATE: builder.startTable(1); - builder.addShort(0, DATE_UNIT_DAY, 0); + builder.addShort(0, DATE_UNIT_DAY, DATE_UNIT_MILLISECOND); return builder.endTable(); case TYPE_TIME: builder.startTable(2); - builder.addShort(0, type.unit, 0); - builder.addInt(1, type.bitWidth, 0); + builder.addShort(0, type.unit, TIME_UNIT_MILLISECOND); + builder.addInt(1, type.bitWidth, TIME_BIT_WIDTH_MILLISECOND); return builder.endTable(); case TYPE_TIMESTAMP: int timezone = type.timezone == null ? 0 : builder.createString(type.timezone); @@ -566,7 +570,7 @@ public ArrowTypeInfo visit(DecimalType decimalType) { ArrowTypeInfo type = ArrowTypeInfo.simple(TYPE_DECIMAL); type.precisionValue = decimalType.getPrecision(); type.scale = decimalType.getScale(); - type.bitWidth = 128; + type.bitWidth = DECIMAL_BIT_WIDTH_128; return type; } @@ -613,7 +617,7 @@ public ArrowTypeInfo visit(DateType dateType) { public ArrowTypeInfo visit(TimeType timeType) { ArrowTypeInfo type = ArrowTypeInfo.simple(TYPE_TIME); type.unit = TIME_UNIT_MILLISECOND; - type.bitWidth = 32; + type.bitWidth = TIME_BIT_WIDTH_MILLISECOND; return type; } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/FormatMetadataUtils.java b/paimon-format/src/main/java/org/apache/paimon/format/FormatMetadataUtils.java index 342f1ad442f7..5ef3d07ecfb5 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/FormatMetadataUtils.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/FormatMetadataUtils.java @@ -33,6 +33,7 @@ public class FormatMetadataUtils { public static final String ARROW_SCHEMA_METADATA_KEY = "ARROW:schema"; public static final String PARQUET_FIELD_ID_KEY = "PARQUET:field_id"; + public static final String ORC_FIELD_ID_KEY = "paimon.id"; private FormatMetadataUtils() {} diff --git a/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java b/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java index f01854f9e314..3f1c3a6cbfdb 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -106,6 +107,7 @@ public void testBuildArrowSchemaWithFieldMetadata() { 4, "scores", DataTypes.ARRAY(DataTypes.INT()))))); Map tagsMetadata = new LinkedHashMap<>(); tagsMetadata.put("paimon.test.tags", "enabled"); + tagsMetadata.put("PARQUET:field_id", "999"); Map> fieldMetadata = new LinkedHashMap<>(); fieldMetadata.put("tags", tagsMetadata); @@ -118,7 +120,7 @@ public void testBuildArrowSchemaWithFieldMetadata() { FormatMetadataUtils.readFieldMetadata(schemaBytes); assertThat(metadata).containsOnlyKeys("id", "tags", "nested"); assertThat(metadata.get("id")).containsEntry(FormatMetadataUtils.PARQUET_FIELD_ID_KEY, "0"); - assertThat(metadata.get("tags")).containsAllEntriesOf(tagsMetadata); + assertThat(metadata.get("tags")).containsEntry("paimon.test.tags", "enabled"); assertThat(metadata.get("tags")) .containsEntry(FormatMetadataUtils.PARQUET_FIELD_ID_KEY, "1"); assertThat(metadata.get("nested")).doesNotContainKey("paimon.test.tags"); @@ -145,4 +147,22 @@ public void testBuildArrowSchemaWithoutFieldIdMetadata() { assertThat(metadata.get("name")) .doesNotContainKey(FormatMetadataUtils.PARQUET_FIELD_ID_KEY); } + + @Test + public void testBuildArrowSchemaWithOrcFieldIdMetadata() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD(0, "id", DataTypes.INT()), + DataTypes.FIELD(1, "name", DataTypes.STRING())); + + byte[] schemaBytes = + FormatMetadataUtils.buildArrowSchemaMetadata( + rowType, Collections.emptyMap(), FormatMetadataUtils.ORC_FIELD_ID_KEY); + + Map> metadata = + FormatMetadataUtils.readFieldMetadata(schemaBytes); + assertThat(metadata).containsOnlyKeys("id", "name"); + assertThat(metadata.get("id")).containsEntry(FormatMetadataUtils.ORC_FIELD_ID_KEY, "0"); + assertThat(metadata.get("name")).containsEntry(FormatMetadataUtils.ORC_FIELD_ID_KEY, "1"); + } } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java index a03af8dc250a..a6084dd4e331 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java @@ -106,7 +106,7 @@ public void testWriteMetadata() throws IOException { fieldMetadataByName.put("name", fieldMetadata); byte[] arrowSchemaBytes = FormatMetadataUtils.buildArrowSchemaMetadata( - rowType, fieldMetadataByName, OrcTypeUtil.PAIMON_ORC_FIELD_ID_KEY); + rowType, fieldMetadataByName, FormatMetadataUtils.ORC_FIELD_ID_KEY); Map metadata = new HashMap<>(); metadata.put("paimon.test.key", "paimon-test-value".getBytes(StandardCharsets.UTF_8)); metadata.put(FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY, arrowSchemaBytes); @@ -141,10 +141,10 @@ public void testWriteMetadata() throws IOException { ((SupportsReaderFieldMetadata) reader).readFieldMetadata(); assertThat(readFieldMetadata).containsOnlyKeys("id", "name"); assertThat(readFieldMetadata.get("id")) - .containsEntry(OrcTypeUtil.PAIMON_ORC_FIELD_ID_KEY, "0"); + .containsEntry(FormatMetadataUtils.ORC_FIELD_ID_KEY, "0"); assertThat(readFieldMetadata.get("name")).containsAllEntriesOf(fieldMetadata); assertThat(readFieldMetadata.get("name")) - .containsEntry(OrcTypeUtil.PAIMON_ORC_FIELD_ID_KEY, "1"); + .containsEntry(FormatMetadataUtils.ORC_FIELD_ID_KEY, "1"); } } From 9fbb1007f78f7c48107642ad2d772e7a63eae58c Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Thu, 25 Jun 2026 19:04:55 +0800 Subject: [PATCH 2/6] fix close --- .../org/apache/paimon/io/SingleFileWriter.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java index 5926e042c55d..27083b8d29f9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java @@ -193,8 +193,11 @@ public void close() throws IOException { try { if (writer != null) { - beforeCloseFormatWriter(writer); - writer.close(); + try { + beforeCloseFormatWriter(writer); + } finally { + writer.close(); + } writerMetadata = writer.writerMetadata(); writer = null; } @@ -204,10 +207,13 @@ public void close() throws IOException { out.close(); out = null; } - } catch (IOException e) { + } catch (Exception e) { LOG.warn("Exception occurs when closing file {}. Cleaning up.", path, e); abort(); - throw e; + if (e instanceof IOException) { + throw (IOException) e; + } + throw new IOException(e); } finally { closed = true; } From 7f554e716859dd04fed3a1ef7e95f97530b197aa Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Fri, 26 Jun 2026 09:33:19 +0800 Subject: [PATCH 3/6] add validate and failed with rewrite --- .../paimon/append/AppendOnlyWriter.java | 2 +- .../MapSharedShreddingCoreUtils.java | 2 +- .../operation/BaseAppendFileStoreWrite.java | 11 +++ .../paimon/schema/SchemaValidation.java | 34 +++++++ .../BucketedAppendFileStoreWriteTest.java | 53 +++++++++++ .../paimon/schema/SchemaValidationTest.java | 88 +++++++++++++++++++ 6 files changed, 188 insertions(+), 2 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java index 74c1344b4f9f..aecc0260ca5b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java @@ -371,7 +371,7 @@ private RollingFileWriter createRollingRowWriter() { ? new MapSharedShreddingWritePlanFactory( writeSchema, sharedShreddingContext, - MapSharedShreddingCoreUtils.sharedShreddingFieldIdKey(fileFormat)) + MapSharedShreddingCoreUtils.fieldIdMetadataKey(fileFormat)) : null); } diff --git a/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java index 5f895999c476..d9d005f5b541 100644 --- a/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java @@ -50,7 +50,7 @@ public class MapSharedShreddingCoreUtils { private MapSharedShreddingCoreUtils() {} @Nullable - public static String sharedShreddingFieldIdKey(FileFormat fileFormat) { + public static String fieldIdMetadataKey(FileFormat fileFormat) { switch (fileFormat.getFormatIdentifier()) { case "parquet": return FormatMetadataUtils.PARQUET_FIELD_ID_KEY; diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/BaseAppendFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/BaseAppendFileStoreWrite.java index 44ce3cfb329b..587309b130f4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/BaseAppendFileStoreWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/BaseAppendFileStoreWrite.java @@ -28,6 +28,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.shredding.MapSharedShreddingContext; import org.apache.paimon.data.shredding.MapSharedShreddingCoreUtils; +import org.apache.paimon.data.shredding.MapSharedShreddingUtils; import org.apache.paimon.deletionvectors.BucketedDvMaintainer; import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.fileindex.FileIndexOptions; @@ -204,6 +205,7 @@ public List compactRewrite( if (toCompact.isEmpty()) { return Collections.emptyList(); } + checkNoSharedShreddingRewrite("Compaction rewrite"); Exception collectedExceptions = null; RowDataRollingFileWriter rewriter = createRollingFileWriter( @@ -236,6 +238,7 @@ public List compactRewrite( public List clusterRewrite( BinaryRow partition, int bucket, List toCluster) throws Exception { + checkNoSharedShreddingRewrite("Cluster rewrite"); RecordReaderIterator reader = createFilesIterator(partition, bucket, toCluster, null); @@ -272,6 +275,14 @@ public List clusterRewrite( return rewriter.result(); } + private void checkNoSharedShreddingRewrite(String rewriteName) { + if (!MapSharedShreddingUtils.detectShreddingColumns(writeType, options).isEmpty()) { + // TODO (xinyu.lxy): Support rewrite writers for MAP shared-shredding. + throw new UnsupportedOperationException( + rewriteName + " is not supported for MAP shared-shredding."); + } + } + private RowDataRollingFileWriter createRollingFileWriter( BinaryRow partition, int bucket, Supplier seqNumCounterSupplier) { return new RowDataRollingFileWriter( diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 3835db635b9f..32db6182d96d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -616,6 +616,7 @@ private static void validateMapStorageLayout(TableSchema schema, CoreOptions opt fieldMap.put(field.name(), field); } + boolean hasSharedShredding = false; for (String key : options.toMap().keySet()) { if (!key.startsWith(FIELDS_PREFIX + ".") || !key.endsWith(layoutSuffix)) { continue; @@ -644,6 +645,7 @@ private static void validateMapStorageLayout(TableSchema schema, CoreOptions opt if (layout != MapStorageLayout.SHARED_SHREDDING) { continue; } + hasSharedShredding = true; if (!MapSharedShreddingUtils.isShreddingKeyMap(fieldType)) { throw new IllegalArgumentException( @@ -653,6 +655,38 @@ private static void validateMapStorageLayout(TableSchema schema, CoreOptions opt } options.mapSharedShreddingMaxColumns(fieldName); } + if (hasSharedShredding) { + validateMapSharedShreddingFileFormats(options); + } + } + + private static void validateMapSharedShreddingFileFormats(CoreOptions options) { + validateMapSharedShreddingFileFormat( + CoreOptions.FILE_FORMAT.key(), options.fileFormatString()); + for (Map.Entry entry : options.fileFormatPerLevel().entrySet()) { + validateMapSharedShreddingFileFormat( + CoreOptions.FILE_FORMAT_PER_LEVEL.key() + "[" + entry.getKey() + "]", + entry.getValue()); + } + validateMapSharedShreddingFileFormat( + CoreOptions.CHANGELOG_FILE_FORMAT.key(), options.changelogFileFormat()); + if (options.withVectorFormat()) { + validateMapSharedShreddingFileFormat( + CoreOptions.VECTOR_FILE_FORMAT.key(), options.vectorFileFormatString()); + } + } + + private static void validateMapSharedShreddingFileFormat(String optionKey, String fileFormat) { + if (fileFormat == null + || CoreOptions.FILE_FORMAT_PARQUET.equals(fileFormat) + || CoreOptions.FILE_FORMAT_ORC.equals(fileFormat)) { + return; + } + throw new IllegalArgumentException( + String.format( + "MAP shared-shredding only supports ORC and Parquet file formats, " + + "but '%s' is configured as '%s'.", + optionKey, fileFormat)); } private static void validateFileIndex(TableSchema schema) { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/BucketedAppendFileStoreWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/BucketedAppendFileStoreWriteTest.java index 556209b2d318..fbdac50a5300 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/BucketedAppendFileStoreWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/BucketedAppendFileStoreWriteTest.java @@ -54,6 +54,7 @@ import static org.apache.paimon.CoreOptions.BUCKET_APPEND_ORDERED; import static org.apache.paimon.CoreOptions.WRITE_MAX_WRITERS_TO_SPILL; import static org.apache.paimon.CoreOptions.WRITE_ONLY; +import static org.apache.paimon.io.DataFileTestUtils.newFile; /** Tests for {@link BucketedAppendFileStoreWrite}. */ public class BucketedAppendFileStoreWriteTest { @@ -176,6 +177,58 @@ protected FileStoreTable createFileStoreTable() throws Exception { return (FileStoreTable) catalog.getTable(identifier); } + @Test + public void testSharedShreddingDoesNotSupportRewrite() throws Exception { + Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), new Path(tempDir.toString())); + catalog.createDatabase("default", false); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())) + .option("bucket", "1") + .option("bucket-key", "id") + .option("fields.tags.map.storage-layout", "shared-shredding") + .build(); + Identifier compactIdentifier = Identifier.create("default", "compact_test"); + Identifier clusterIdentifier = Identifier.create("default", "cluster_test"); + catalog.createTable(compactIdentifier, schema, false); + catalog.createTable(clusterIdentifier, schema, false); + + BaseAppendFileStoreWrite compactWrite = + (BaseAppendFileStoreWrite) + ((FileStoreTable) catalog.getTable(compactIdentifier)) + .store() + .newWrite("ss"); + + Assertions.assertThatThrownBy( + () -> + compactWrite.compactRewrite( + BinaryRow.EMPTY_ROW, + 0, + null, + Collections.singletonList( + newFile("data-0.orc", 0, 0, 1, 1)))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining( + "Compaction rewrite is not supported for MAP shared-shredding."); + + BaseAppendFileStoreWrite clusterWrite = + (BaseAppendFileStoreWrite) + ((FileStoreTable) catalog.getTable(clusterIdentifier)) + .store() + .newWrite("ss"); + + Assertions.assertThatThrownBy( + () -> + clusterWrite.clusterRewrite( + BinaryRow.EMPTY_ROW, + 0, + Collections.singletonList( + newFile("data-0.orc", 0, 0, 1, 1)))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Cluster rewrite is not supported for MAP shared-shredding."); + } + @Test public void testIgnorePreviousFilesChecksPartitionBucketNumber() throws Exception { FileStoreTable table = createFileStoreTable().copy(bucketOptions(2, false, false)); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java index 794cea32d01f..29a256abdbbf 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java @@ -311,6 +311,94 @@ public void testMapStorageLayout() { options, ""))) .hasMessageContaining("options map.shared-shredding.max-columns must > 0"); + + options.remove("fields.metrics.map.shared-shredding.max-columns"); + options.put(CoreOptions.FILE_FORMAT.key(), "orc"); + assertThatNoException() + .isThrownBy( + () -> + validateTableSchema( + new TableSchema( + 1, + fields, + 10, + emptyList(), + emptyList(), + options, + ""))); + + options.put(CoreOptions.FILE_FORMAT.key(), "avro"); + assertThatThrownBy( + () -> + validateTableSchema( + new TableSchema( + 1, + fields, + 10, + emptyList(), + emptyList(), + options, + ""))) + .hasMessageContaining( + "MAP shared-shredding only supports ORC and Parquet file formats") + .hasMessageContaining("'" + CoreOptions.FILE_FORMAT.key() + "'") + .hasMessageContaining("'avro'"); + + options.put(CoreOptions.FILE_FORMAT.key(), "parquet"); + options.put(CoreOptions.FILE_FORMAT_PER_LEVEL.key(), "0:avro"); + assertThatThrownBy( + () -> + validateTableSchema( + new TableSchema( + 1, + fields, + 10, + emptyList(), + emptyList(), + options, + ""))) + .hasMessageContaining( + "MAP shared-shredding only supports ORC and Parquet file formats") + .hasMessageContaining("'" + CoreOptions.FILE_FORMAT_PER_LEVEL.key() + "[0]'") + .hasMessageContaining("'avro'"); + + options.remove(CoreOptions.FILE_FORMAT_PER_LEVEL.key()); + options.put(CoreOptions.CHANGELOG_FILE_FORMAT.key(), "avro"); + assertThatThrownBy( + () -> + validateTableSchema( + new TableSchema( + 1, + fields, + 10, + emptyList(), + emptyList(), + options, + ""))) + .hasMessageContaining( + "MAP shared-shredding only supports ORC and Parquet file formats") + .hasMessageContaining("'" + CoreOptions.CHANGELOG_FILE_FORMAT.key() + "'") + .hasMessageContaining("'avro'"); + + options.remove(CoreOptions.CHANGELOG_FILE_FORMAT.key()); + options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true"); + options.put(DATA_EVOLUTION_ENABLED.key(), "true"); + options.put(VECTOR_FILE_FORMAT.key(), "json"); + assertThatThrownBy( + () -> + validateTableSchema( + new TableSchema( + 1, + fields, + 10, + emptyList(), + emptyList(), + options, + ""))) + .hasMessageContaining( + "MAP shared-shredding only supports ORC and Parquet file formats") + .hasMessageContaining("'" + VECTOR_FILE_FORMAT.key() + "'") + .hasMessageContaining("'json'"); } @Test From 490befe69e82f90357655fe2180d57c72f0e5dbb Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Sat, 27 Jun 2026 10:16:46 +0800 Subject: [PATCH 4/6] add read path for append shared-shredding and e2e tests --- .../shredding/MapSharedShreddingReader.java | 334 +++++++++++ .../shredding/MapSharedShreddingUtils.java | 52 ++ .../MapSharedShreddingReaderTest.java | 190 +++++++ .../MapSharedShreddingCoreUtils.java | 31 +- .../paimon/append/AppendOnlyWriterTest.java | 24 +- .../table/MapSharedShreddingTableTest.java | 528 ++++++++++++++++++ ...tadata.java => SupportsFieldMetadata.java} | 7 +- .../paimon/format/orc/OrcFileFormat.java | 18 +- .../paimon/format/orc/OrcReaderFactory.java | 146 +++-- .../format/parquet/ParquetFileFormat.java | 24 +- .../format/parquet/ParquetReaderFactory.java | 79 ++- .../reader/VectorizedParquetRecordReader.java | 25 +- .../format/orc/OrcFormatReadWriteTest.java | 24 +- .../parquet/ParquetFormatReadWriteTest.java | 24 +- 14 files changed, 1346 insertions(+), 160 deletions(-) create mode 100644 paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingReader.java create mode 100644 paimon-common/src/test/java/org/apache/paimon/data/shredding/MapSharedShreddingReaderTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/MapSharedShreddingTableTest.java rename paimon-format/src/main/java/org/apache/paimon/format/{SupportsReaderFieldMetadata.java => SupportsFieldMetadata.java} (82%) diff --git a/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingReader.java b/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingReader.java new file mode 100644 index 000000000000..258bca5d76bd --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingReader.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data.shredding; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.InternalVector; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.data.variant.Variant; +import org.apache.paimon.fs.Path; +import org.apache.paimon.reader.FileRecordIterator; +import org.apache.paimon.reader.FileRecordReader; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.RowKind; +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * A reader wrapper that rebuilds logical MAP values from shared-shredding physical ROW values. + * + *

The wrapped format reader reads the physical schema stored in a file. This reader presents the + * original logical schema to upper layers by lazily converting only shared-shredding MAP fields + * when {@link InternalRow#getMap(int)} is called. + */ +public class MapSharedShreddingReader implements FileRecordReader { + + private final FileRecordReader reader; + private final RowType logicalType; + private final Map contextByFieldIndex; + + public MapSharedShreddingReader( + FileRecordReader reader, + RowType logicalType, + Map fieldMetas) { + this.reader = reader; + this.logicalType = logicalType; + this.contextByFieldIndex = createContexts(logicalType, fieldMetas); + } + + public static Map readSharedShreddingMetas( + Map> fieldMetadata) { + Map metas = new LinkedHashMap<>(); + for (Map.Entry> entry : fieldMetadata.entrySet()) { + if (MapSharedShreddingUtils.hasShreddingMetadata(entry.getValue())) { + metas.put( + entry.getKey(), + MapSharedShreddingUtils.deserializeMetadata( + entry.getValue(), + MapSharedShreddingDefine.DEFAULT_DICT_COMPRESSION)); + } + } + return metas; + } + + private static Map createContexts( + RowType logicalType, Map fieldMetas) { + Map contexts = new LinkedHashMap<>(); + for (int i = 0; i < logicalType.getFieldCount(); i++) { + DataField field = logicalType.getFields().get(i); + MapSharedShreddingFieldMeta fieldMeta = fieldMetas.get(field.name()); + if (fieldMeta != null) { + contexts.put(i, new SharedShreddingContext(fieldMeta, field.type())); + } + } + return contexts; + } + + @Nullable + @Override + public FileRecordIterator readBatch() throws IOException { + FileRecordIterator iterator = reader.readBatch(); + if (iterator == null) { + return null; + } + return new SharedShreddingIterator(iterator); + } + + @Override + public void close() throws IOException { + reader.close(); + } + + private class SharedShreddingIterator implements FileRecordIterator { + + private final FileRecordIterator iterator; + + private SharedShreddingIterator(FileRecordIterator iterator) { + this.iterator = iterator; + } + + @Override + public long returnedPosition() { + return iterator.returnedPosition(); + } + + @Override + public Path filePath() { + return iterator.filePath(); + } + + @Nullable + @Override + public InternalRow next() throws IOException { + InternalRow row = iterator.next(); + if (row == null) { + return null; + } + return new SharedShreddingRow(row); + } + + @Override + public void releaseBatch() { + iterator.releaseBatch(); + } + } + + private class SharedShreddingRow implements InternalRow { + + private final InternalRow row; + + private SharedShreddingRow(InternalRow row) { + this.row = row; + } + + @Override + public int getFieldCount() { + return logicalType.getFieldCount(); + } + + @Override + public RowKind getRowKind() { + return row.getRowKind(); + } + + @Override + public void setRowKind(RowKind kind) { + row.setRowKind(kind); + } + + @Override + public boolean isNullAt(int pos) { + return row.isNullAt(pos); + } + + @Override + public boolean getBoolean(int pos) { + return row.getBoolean(pos); + } + + @Override + public byte getByte(int pos) { + return row.getByte(pos); + } + + @Override + public short getShort(int pos) { + return row.getShort(pos); + } + + @Override + public int getInt(int pos) { + return row.getInt(pos); + } + + @Override + public long getLong(int pos) { + return row.getLong(pos); + } + + @Override + public float getFloat(int pos) { + return row.getFloat(pos); + } + + @Override + public double getDouble(int pos) { + return row.getDouble(pos); + } + + @Override + public BinaryString getString(int pos) { + return row.getString(pos); + } + + @Override + public Decimal getDecimal(int pos, int precision, int scale) { + return row.getDecimal(pos, precision, scale); + } + + @Override + public Timestamp getTimestamp(int pos, int precision) { + return row.getTimestamp(pos, precision); + } + + @Override + public byte[] getBinary(int pos) { + return row.getBinary(pos); + } + + @Override + public Variant getVariant(int pos) { + return row.getVariant(pos); + } + + @Override + public Blob getBlob(int pos) { + return row.getBlob(pos); + } + + @Override + public InternalArray getArray(int pos) { + return row.getArray(pos); + } + + @Override + public InternalVector getVector(int pos) { + return row.getVector(pos); + } + + @Override + public InternalMap getMap(int pos) { + SharedShreddingContext context = contextByFieldIndex.get(pos); + if (context == null) { + return row.getMap(pos); + } + if (row.isNullAt(pos)) { + return null; + } + InternalRow physicalRow = row.getRow(pos, context.numPhysicalFields); + return rebuildLogicalMap(physicalRow, context); + } + + @Override + public InternalRow getRow(int pos, int numFields) { + return row.getRow(pos, numFields); + } + } + + private static InternalMap rebuildLogicalMap( + InternalRow physicalRow, SharedShreddingContext context) { + InternalArray fieldMapping = physicalRow.getArray(0); + Map result = new LinkedHashMap<>(); + int numMappedColumns = Math.min(context.numColumns, fieldMapping.size()); + for (int column = 0; column < numMappedColumns; column++) { + int fieldId = fieldMapping.isNullAt(column) ? -1 : fieldMapping.getInt(column); + if (fieldId < 0) { + continue; + } + BinaryString fieldName = context.nameById.get(fieldId); + if (fieldName == null) { + continue; + } + Object value = null; + int valuePosition = column + 1; + if (valuePosition < physicalRow.getFieldCount() + && !physicalRow.isNullAt(valuePosition)) { + value = context.valueGetters[column].getFieldOrNull(physicalRow); + } + result.put(fieldName, value); + } + if (context.overflowPosition < physicalRow.getFieldCount() + && !physicalRow.isNullAt(context.overflowPosition)) { + InternalMap overflow = physicalRow.getMap(context.overflowPosition); + InternalArray keys = overflow.keyArray(); + InternalArray values = overflow.valueArray(); + for (int i = 0; i < overflow.size(); i++) { + int fieldId = keys.getInt(i); + BinaryString fieldName = context.nameById.get(fieldId); + if (fieldName != null) { + result.put(fieldName, context.overflowValueGetter.getElementOrNull(values, i)); + } + } + } + return new GenericMap(result); + } + + private static class SharedShreddingContext { + + private final Map nameById; + private final InternalRow.FieldGetter[] valueGetters; + private final InternalArray.ElementGetter overflowValueGetter; + private final int numColumns; + private final int overflowPosition; + private final int numPhysicalFields; + + private SharedShreddingContext(MapSharedShreddingFieldMeta fieldMeta, DataType fieldType) { + MapType mapType = (MapType) fieldType; + this.nameById = new LinkedHashMap<>(); + for (Map.Entry entry : fieldMeta.nameToId().entrySet()) { + this.nameById.put(entry.getValue(), BinaryString.fromString(entry.getKey())); + } + this.valueGetters = new InternalRow.FieldGetter[fieldMeta.numColumns()]; + for (int i = 0; i < fieldMeta.numColumns(); i++) { + this.valueGetters[i] = + InternalRow.createFieldGetter(mapType.getValueType(), i + 1); + } + this.overflowValueGetter = InternalArray.createElementGetter(mapType.getValueType()); + this.numColumns = fieldMeta.numColumns(); + this.overflowPosition = fieldMeta.numColumns() + 1; + this.numPhysicalFields = + fieldMeta.overflowFieldSet().isEmpty() + ? fieldMeta.numColumns() + 1 + : fieldMeta.numColumns() + 2; + } + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingUtils.java b/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingUtils.java index 6f0cd879c52a..e2ecaecc9814 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingUtils.java @@ -188,6 +188,58 @@ public static boolean hasShreddingMetadata(@Nullable Map metadat metadata.get(MapShreddingDefine.STORAGE_LAYOUT)); } + public static RowType buildSpecificPhysicalStructType( + DataType valueType, Set physicalColumnIds, boolean includeOverflow) { + RowType.Builder builder = RowType.builder(); + builder.field(MapSharedShreddingDefine.FIELD_MAPPING, new ArrayType(new IntType())); + for (Integer columnId : new TreeSet<>(physicalColumnIds)) { + builder.field(MapSharedShreddingDefine.physicalColumnName(columnId), valueType); + } + if (includeOverflow) { + builder.field(MapSharedShreddingDefine.OVERFLOW, new MapType(new IntType(), valueType)); + } + return builder.build(); + } + + public static RowType buildPhysicalReadType( + RowType logicalReadType, + Map sharedShreddingFieldMetas) { + if (sharedShreddingFieldMetas.isEmpty()) { + return logicalReadType; + } + + List physicalReadFields = new ArrayList<>(); + boolean converted = false; + for (DataField logicalReadField : logicalReadType.getFields()) { + MapSharedShreddingFieldMeta fieldMeta = + sharedShreddingFieldMetas.get(logicalReadField.name()); + if (fieldMeta == null) { + physicalReadFields.add(logicalReadField); + continue; + } + + if (!(logicalReadField.type() instanceof MapType)) { + physicalReadFields.add(logicalReadField); + continue; + } + + MapType mapType = (MapType) logicalReadField.type(); + Set physicalColumnIds = new TreeSet<>(); + for (int columnId = 0; columnId < fieldMeta.numColumns(); columnId++) { + physicalColumnIds.add(columnId); + } + DataType physicalType = + buildSpecificPhysicalStructType( + mapType.getValueType(), + physicalColumnIds, + !fieldMeta.overflowFieldSet().isEmpty()) + .copy(logicalReadField.type().isNullable()); + physicalReadFields.add(logicalReadField.newType(physicalType)); + converted = true; + } + return converted ? new RowType(logicalReadType.isNullable(), physicalReadFields) : logicalReadType; + } + private static RowType buildPhysicalStructType(DataType valueType, int numColumns) { RowType.Builder builder = RowType.builder(); builder.field(MapSharedShreddingDefine.FIELD_MAPPING, new ArrayType(new IntType())); diff --git a/paimon-common/src/test/java/org/apache/paimon/data/shredding/MapSharedShreddingReaderTest.java b/paimon-common/src/test/java/org/apache/paimon/data/shredding/MapSharedShreddingReaderTest.java new file mode 100644 index 000000000000..43327461b4a4 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/data/shredding/MapSharedShreddingReaderTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.data.shredding; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.Path; +import org.apache.paimon.reader.FileRecordIterator; +import org.apache.paimon.reader.FileRecordReader; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link MapSharedShreddingReader}. */ +class MapSharedShreddingReaderTest { + + @Test + void testReadProjectedPhysicalRowWithoutOverflowColumn() throws IOException { + RowType logicalType = + DataTypes.ROW( + DataTypes.FIELD( + 0, + "metrics", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT()))); + MapSharedShreddingFieldMeta fieldMeta = + new MapSharedShreddingFieldMeta( + nameToId("a", 0, "b", 1), + Collections.emptyMap(), + new TreeSet(), + 3, + 2); + + GenericRow physicalMap = new GenericRow(4); + physicalMap.setField(0, new GenericArray(new int[] {0, -1, 1})); + physicalMap.setField(1, 10L); + physicalMap.setField(2, null); + physicalMap.setField(3, 20L); + + InternalMap restored = readMap(logicalType, fieldMeta, physicalMap); + + assertThat(restored.size()).isEqualTo(2); + assertThat(restored.keyArray().getString(0)).isEqualTo(BinaryString.fromString("a")); + assertThat(restored.keyArray().getString(1)).isEqualTo(BinaryString.fromString("b")); + assertThat(restored.valueArray().getLong(0)).isEqualTo(10L); + assertThat(restored.valueArray().getLong(1)).isEqualTo(20L); + } + + @Test + void testReadOverflowOnlyWhenOverflowColumnExists() throws IOException { + RowType logicalType = + DataTypes.ROW( + DataTypes.FIELD( + 0, + "metrics", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT()))); + MapSharedShreddingFieldMeta fieldMeta = + new MapSharedShreddingFieldMeta( + nameToId("a", 0, "overflowed", 1), + Collections.emptyMap(), + new TreeSet(Collections.singletonList(1)), + 1, + 1); + + Map overflow = new LinkedHashMap<>(); + overflow.put(1, 30L); + GenericRow physicalMap = new GenericRow(3); + physicalMap.setField(0, new GenericArray(new int[] {-1})); + physicalMap.setField(1, null); + physicalMap.setField(2, new GenericMap(overflow)); + + InternalMap restored = readMap(logicalType, fieldMeta, physicalMap); + + assertThat(restored.size()).isEqualTo(1); + assertThat(restored.keyArray().getString(0)) + .isEqualTo(BinaryString.fromString("overflowed")); + assertThat(restored.valueArray().getLong(0)).isEqualTo(30L); + } + + private static InternalMap readMap( + RowType logicalType, MapSharedShreddingFieldMeta fieldMeta, InternalRow physicalMap) + throws IOException { + GenericRow physicalRow = GenericRow.of(physicalMap); + Map fieldMetas = new LinkedHashMap<>(); + fieldMetas.put("metrics", fieldMeta); + MapSharedShreddingReader reader = + new MapSharedShreddingReader( + new SingleRowRecordReader(physicalRow), logicalType, fieldMetas); + FileRecordIterator iterator = reader.readBatch(); + InternalRow row = iterator.next(); + iterator.releaseBatch(); + reader.close(); + return row.getMap(0); + } + + private static Map nameToId(Object... pairs) { + Map result = new TreeMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + result.put((String) pairs[i], (Integer) pairs[i + 1]); + } + return result; + } + + private static class SingleRowRecordReader implements FileRecordReader { + + private final InternalRow row; + private boolean consumed; + + private SingleRowRecordReader(InternalRow row) { + this.row = row; + } + + @Nullable + @Override + public FileRecordIterator readBatch() { + if (consumed) { + return null; + } + consumed = true; + return new SingleRowFileRecordIterator(row); + } + + @Override + public void close() {} + } + + private static class SingleRowFileRecordIterator implements FileRecordIterator { + + private final InternalRow row; + private boolean consumed; + + private SingleRowFileRecordIterator(InternalRow row) { + this.row = row; + } + + @Override + public long returnedPosition() { + return consumed ? 0 : -1; + } + + @Override + public Path filePath() { + return new Path("/tmp/shared-shredding-reader-test"); + } + + @Nullable + @Override + public InternalRow next() { + if (consumed) { + return null; + } + consumed = true; + return row; + } + + @Override + public void releaseBatch() {} + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java index d9d005f5b541..91738cd7481e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java @@ -23,12 +23,10 @@ import org.apache.paimon.format.FileFormatDiscover; import org.apache.paimon.format.FormatMetadataUtils; import org.apache.paimon.format.FormatReaderContext; -import org.apache.paimon.format.FormatReaderFactory; -import org.apache.paimon.format.SupportsReaderFieldMetadata; +import org.apache.paimon.format.SupportsFieldMetadata; import org.apache.paimon.fs.FileIO; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFilePathFactory; -import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.types.RowType; import javax.annotation.Nullable; @@ -45,8 +43,6 @@ /** Core utilities for shared-shredding MAP write and restore flows. */ public class MapSharedShreddingCoreUtils { - private static final RowType METADATA_READER_ROW_TYPE = new RowType(Collections.emptyList()); - private MapSharedShreddingCoreUtils() {} @Nullable @@ -108,21 +104,18 @@ private static void restoreRecentFileStats( continue; } - FormatReaderFactory readerFactory = - fileFormatDiscover - .discover(file.fileFormat()) - .createReaderFactory( - METADATA_READER_ROW_TYPE, METADATA_READER_ROW_TYPE, null); - try (FileRecordReader reader = - readerFactory.createReader( - new FormatReaderContext( - fileIO, pathFactory.toPath(file), file.fileSize()))) { - if (!(reader instanceof SupportsReaderFieldMetadata)) { - continue; - } - + FileFormat fileFormat = fileFormatDiscover.discover(file.fileFormat()); + if (!(fileFormat instanceof SupportsFieldMetadata)) { + continue; + } + try { Map> fieldMetadata = - ((SupportsReaderFieldMetadata) reader).readFieldMetadata(); + ((SupportsFieldMetadata) fileFormat) + .readFieldMetadata( + new FormatReaderContext( + fileIO, + pathFactory.toPath(file), + file.fileSize())); for (String fieldName : candidateFields) { Map metadata = fieldMetadata.get(fieldName); if (!MapSharedShreddingUtils.hasShreddingMetadata(metadata)) { diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java index 34b11498c28e..757567c5cfc6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java @@ -45,7 +45,7 @@ import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FormatReaderContext; import org.apache.paimon.format.SimpleColStats; -import org.apache.paimon.format.SupportsReaderFieldMetadata; +import org.apache.paimon.format.SupportsFieldMetadata; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; @@ -1145,19 +1145,15 @@ private void assertSharedShreddingFileSchema( RowType expectedPhysicalType, Map expectedMetas) throws IOException { - RowType emptyRowType = new RowType(Collections.emptyList()); - try (FileRecordReader reader = - context.format - .createReaderFactory(emptyRowType, emptyRowType, Collections.emptyList()) - .createReader(new FormatReaderContext(context.fileIO, path, fileSize))) { - Map> fieldMetadata = - ((SupportsReaderFieldMetadata) reader).readFieldMetadata(); - for (Map.Entry entry : expectedMetas.entrySet()) { - String fieldName = entry.getKey(); - assertThat(expectedPhysicalType.containsField(fieldName)).isTrue(); - assertThat(readSharedShreddingFieldMeta(fieldMetadata, fieldName)) - .isEqualTo(entry.getValue()); - } + Map> fieldMetadata = + ((SupportsFieldMetadata) context.format) + .readFieldMetadata( + new FormatReaderContext(context.fileIO, path, fileSize)); + for (Map.Entry entry : expectedMetas.entrySet()) { + String fieldName = entry.getKey(); + assertThat(expectedPhysicalType.containsField(fieldName)).isTrue(); + assertThat(readSharedShreddingFieldMeta(fieldMetadata, fieldName)) + .isEqualTo(entry.getValue()); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/MapSharedShreddingTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/MapSharedShreddingTableTest.java new file mode 100644 index 000000000000..0576db38681e --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/MapSharedShreddingTableTest.java @@ -0,0 +1,528 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.shredding.MapSharedShreddingDefine; +import org.apache.paimon.data.shredding.MapSharedShreddingFieldMeta; +import org.apache.paimon.data.shredding.MapSharedShreddingUtils; +import org.apache.paimon.format.FileFormat; +import org.apache.paimon.format.FileFormatDiscover; +import org.apache.paimon.format.FormatReaderContext; +import org.apache.paimon.format.SupportsFieldMetadata; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataFilePathFactory; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Range; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Table-level tests for MAP shared-shredding. */ +public class MapSharedShreddingTableTest extends TableTestBase { + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet"}) + public void testAppendOnlyTableReadWrite(String format) throws Exception { + Table table = createTable(format, "metrics"); + + write( + table, + GenericRow.of(1, mapOf("a", 11L, "b", 12L, "c", 13L)), + GenericRow.of(2, mapOf()), + GenericRow.of(3, null), + GenericRow.of(4, mapOf("a", null, "b", 42L, "c", null))); + + Map> actual = new LinkedHashMap<>(); + for (InternalRow row : read(table)) { + actual.put(row.getInt(0), row.isNullAt(1) ? null : toJavaMap(row.getMap(1))); + } + + assertThat(actual) + .containsEntry(1, javaMapOf("a", 11L, "b", 12L, "c", 13L)) + .containsEntry(2, javaMapOf()) + .containsEntry(3, null) + .containsEntry(4, javaMapOf("a", null, "b", 42L, "c", null)); + } + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet"}) + public void testAppendOnlyTableReadWriteWithTwoMapFields(String format) throws Exception { + Table table = createTable(format, "metrics", "labels"); + + write( + table, + GenericRow.of( + 1, + mapOf("a", 11L, "b", 12L, "c", 13L), + mapOf("x", 21L, "y", 22L, "z", 23L)), + GenericRow.of(2, mapOf("a", 31L), mapOf()), + GenericRow.of(3, null, mapOf("x", 41L))); + + Map>> actual = new LinkedHashMap<>(); + for (InternalRow row : read(table)) { + actual.put( + row.getInt(0), + Arrays.asList( + row.isNullAt(1) ? null : toJavaMap(row.getMap(1)), + row.isNullAt(2) ? null : toJavaMap(row.getMap(2)))); + } + + assertThat(actual) + .containsEntry( + 1, + Arrays.asList( + javaMapOf("a", 11L, "b", 12L, "c", 13L), + javaMapOf("x", 21L, "y", 22L, "z", 23L))) + .containsEntry(2, Arrays.asList(javaMapOf("a", 31L), javaMapOf())) + .containsEntry(3, Arrays.asList(null, javaMapOf("x", 41L))); + + Map> projected = new LinkedHashMap<>(); + for (InternalRow row : read(table, new int[] {0, 2})) { + projected.put(row.getInt(0), row.isNullAt(1) ? null : toJavaMap(row.getMap(1))); + } + + assertThat(projected) + .containsEntry(1, javaMapOf("x", 21L, "y", 22L, "z", 23L)) + .containsEntry(2, javaMapOf()) + .containsEntry(3, javaMapOf("x", 41L)); + } + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet"}) + public void testRestoreAdaptiveColumnCountFromFileMetadata(String format) throws Exception { + Table table = createTableWithBucket(format, 8, "1", "metrics"); + + write(table, GenericRow.of(1, mapOf("a", 11L))); + write(table, GenericRow.of(2, mapOf("b", 22L))); + + FileStoreTable fileStoreTable = (FileStoreTable) table; + List files = currentDataFiles(fileStoreTable); + files.sort(Comparator.comparingLong(file -> file.dataFile.minSequenceNumber())); + assertThat(files).hasSize(2); + + MapSharedShreddingFieldMeta firstFileMeta = + readSharedShreddingFieldMeta(fileStoreTable, files.get(0), "metrics"); + assertThat(firstFileMeta.numColumns()).isEqualTo(8); + assertThat(firstFileMeta.maxRowWidth()).isEqualTo(1); + + MapSharedShreddingFieldMeta secondFileMeta = + readSharedShreddingFieldMeta(fileStoreTable, files.get(1), "metrics"); + assertThat(secondFileMeta.numColumns()).isEqualTo(1); + assertThat(secondFileMeta.maxRowWidth()).isEqualTo(1); + + Map> actual = new LinkedHashMap<>(); + for (InternalRow row : read(table)) { + actual.put(row.getInt(0), row.isNullAt(1) ? null : toJavaMap(row.getMap(1))); + } + + assertThat(actual) + .containsEntry(1, javaMapOf("a", 11L)) + .containsEntry(2, javaMapOf("b", 22L)); + } + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet"}) + public void testSwitchMapLayoutAndUseMaxColumnsWithoutMetadata(String format) throws Exception { + Table table = + createTableWithBucket( + format, + 4, + "1", + Arrays.asList("metrics", "labels"), + Arrays.asList("labels")); + + write( + table, + GenericRow.of(1, mapOf("a", 11L, "b", 12L), mapOf("x", 21L))); + + catalog.alterTable( + identifier(format), + Arrays.asList( + SchemaChange.setOption( + "fields.metrics.map.storage-layout", "shared-shredding"), + SchemaChange.setOption( + "fields.metrics.map.shared-shredding.max-columns", "3"), + SchemaChange.setOption("fields.labels.map.storage-layout", "default")), + false); + table = catalog.getTable(identifier(format)); + + write( + table, + GenericRow.of(2, mapOf("c", 31L), mapOf("y", 41L, "z", 42L))); + + FileStoreTable fileStoreTable = (FileStoreTable) table; + List files = currentDataFiles(fileStoreTable); + files.sort(Comparator.comparingLong(file -> file.dataFile.minSequenceNumber())); + assertThat(files).hasSize(2); + + MapSharedShreddingFieldMeta metricsMeta = + readSharedShreddingFieldMeta(fileStoreTable, files.get(1), "metrics"); + assertThat(metricsMeta.numColumns()).isEqualTo(3); + assertThat(metricsMeta.maxRowWidth()).isEqualTo(1); + + Map>> actual = new LinkedHashMap<>(); + for (InternalRow row : read(table)) { + actual.put( + row.getInt(0), + Arrays.asList( + row.isNullAt(1) ? null : toJavaMap(row.getMap(1)), + row.isNullAt(2) ? null : toJavaMap(row.getMap(2)))); + } + + assertThat(actual) + .containsEntry( + 1, + Arrays.asList( + javaMapOf("a", 11L, "b", 12L), javaMapOf("x", 21L))) + .containsEntry( + 2, + Arrays.asList( + javaMapOf("c", 31L), javaMapOf("y", 41L, "z", 42L))); + } + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet"}) + public void testReadSharedShreddingMapAfterRenameColumn(String format) throws Exception { + Table table = createTable(format, "metrics"); + + write( + table, + GenericRow.of(1, mapOf("a", 11L, "b", 12L)), + GenericRow.of(2, mapOf("c", 21L))); + + catalog.alterTable( + identifier(format), + Arrays.asList( + SchemaChange.renameColumn("metrics", "renamed_metrics"), + SchemaChange.removeOption("fields.metrics.map.storage-layout"), + SchemaChange.removeOption( + "fields.metrics.map.shared-shredding.max-columns"), + SchemaChange.setOption( + "fields.renamed_metrics.map.storage-layout", "shared-shredding"), + SchemaChange.setOption( + "fields.renamed_metrics.map.shared-shredding.max-columns", "2")), + false); + table = catalog.getTable(identifier(format)); + + assertThat(table.rowType().getFieldNames()).containsExactly("id", "renamed_metrics"); + + Map> actual = new LinkedHashMap<>(); + for (InternalRow row : read(table)) { + actual.put(row.getInt(0), row.isNullAt(1) ? null : toJavaMap(row.getMap(1))); + } + + assertThat(actual) + .containsEntry(1, javaMapOf("a", 11L, "b", 12L)) + .containsEntry(2, javaMapOf("c", 21L)); + } + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet"}) + public void testDataEvolutionMergeWithOverwrittenSharedShreddingMaps(String format) + throws Exception { + Table table = createDataEvolutionTable(format, "metrics", "labels"); + RowType rowType = table.rowType(); + + writeWithWriteType( + table, + rowType.project(Arrays.asList("id", "metrics")), + GenericRow.of(1, mapOf("old-a", 11L)), + GenericRow.of(2, mapOf("old-b", 21L))); + writeWithWriteType( + table, + rowType.project(Arrays.asList("id", "labels")), + 0L, + GenericRow.of(101, mapOf("label-a", 101L)), + GenericRow.of(102, mapOf("label-b", 102L))); + writeWithWriteType( + table, + rowType.project(Collections.singletonList("metrics")), + 0L, + GenericRow.of(mapOf("new-a", 111L)), + GenericRow.of(mapOf("new-b", 222L))); + + Map>> actual = new LinkedHashMap<>(); + for (InternalRow row : read(table)) { + actual.put( + row.getInt(0), + Arrays.asList(toJavaMap(row.getMap(1)), toJavaMap(row.getMap(2)))); + } + + assertThat(actual) + .containsEntry( + 101, + Arrays.asList(javaMapOf("new-a", 111L), javaMapOf("label-a", 101L))) + .containsEntry( + 102, + Arrays.asList(javaMapOf("new-b", 222L), javaMapOf("label-b", 102L))); + + Map>> partialActual = new LinkedHashMap<>(); + ReadBuilder readBuilder = + table.newReadBuilder().withRowRanges(Collections.singletonList(new Range(1L, 1L))); + readBuilder + .newRead() + .createReader(readBuilder.newScan().plan()) + .forEachRemaining( + row -> + partialActual.put( + row.getInt(0), + Arrays.asList( + toJavaMap(row.getMap(1)), + toJavaMap(row.getMap(2))))); + + assertThat(partialActual) + .containsOnlyKeys(102) + .containsEntry( + 102, + Arrays.asList(javaMapOf("new-b", 222L), javaMapOf("label-b", 102L))); + } + + private Table createTable(String format, String... sharedShreddingFields) throws Exception { + return createTable(format, 2, sharedShreddingFields); + } + + private Table createTable(String format, int maxColumns, String... sharedShreddingFields) + throws Exception { + return createTableWithBucket(format, maxColumns, "-1", sharedShreddingFields); + } + + private Table createTableWithBucket( + String format, int maxColumns, String bucket, String... sharedShreddingFields) + throws Exception { + return createTableWithBucket( + format, + maxColumns, + bucket, + Arrays.asList(sharedShreddingFields), + Arrays.asList(sharedShreddingFields)); + } + + private Table createTableWithBucket( + String format, + int maxColumns, + String bucket, + List mapFields, + List sharedShreddingFields) + throws Exception { + catalog.createTable( + identifier(format), + schemaWithBucket(format, maxColumns, bucket, mapFields, sharedShreddingFields), + true); + return catalog.getTable(identifier(format)); + } + + private Table createDataEvolutionTable(String format, String... sharedShreddingFields) + throws Exception { + Schema.Builder builder = Schema.newBuilder().column("id", DataTypes.INT()); + for (String field : sharedShreddingFields) { + builder.column(field, DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())); + } + builder.option("bucket", "-1") + .option("file.format", format) + .option(CoreOptions.WRITE_ONLY.key(), "true") + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true"); + for (String field : sharedShreddingFields) { + builder.option("fields." + field + ".map.storage-layout", "shared-shredding") + .option("fields." + field + ".map.shared-shredding.max-columns", "2"); + } + catalog.createTable(identifier(format), builder.build(), true); + return catalog.getTable(identifier(format)); + } + + private void writeWithWriteType(Table table, RowType writeType, InternalRow... rows) + throws Exception { + writeWithWriteType(table, writeType, null, rows); + } + + private void writeWithWriteType( + Table table, RowType writeType, Long firstRowId, InternalRow... rows) throws Exception { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite().withWriteType(writeType); + BatchTableCommit commit = builder.newCommit()) { + for (InternalRow row : rows) { + write.write(row); + } + List commitMessages = write.prepareCommit(); + if (firstRowId != null) { + setFirstRowId(commitMessages, firstRowId); + } + commit.commit(commitMessages); + } + } + + private void setFirstRowId(List commitMessages, long firstRowId) { + for (CommitMessage message : commitMessages) { + CommitMessageImpl commitMessage = (CommitMessageImpl) message; + List newFiles = + new ArrayList<>(commitMessage.newFilesIncrement().newFiles()); + commitMessage.newFilesIncrement().newFiles().clear(); + for (DataFileMeta newFile : newFiles) { + commitMessage + .newFilesIncrement() + .newFiles() + .add(newFile.assignFirstRowId(firstRowId)); + } + } + } + + private Schema schema(String format, String... sharedShreddingFields) { + return schema(format, 2, sharedShreddingFields); + } + + private Schema schema(String format, int maxColumns, String... sharedShreddingFields) { + return schemaWithBucket(format, maxColumns, "-1", sharedShreddingFields); + } + + private Schema schemaWithBucket( + String format, int maxColumns, String bucket, String... sharedShreddingFields) { + return schemaWithBucket( + format, + maxColumns, + bucket, + Arrays.asList(sharedShreddingFields), + Arrays.asList(sharedShreddingFields)); + } + + private Schema schemaWithBucket( + String format, + int maxColumns, + String bucket, + List mapFields, + List sharedShreddingFields) { + Schema.Builder builder = Schema.newBuilder().column("id", DataTypes.INT()); + for (String field : mapFields) { + builder.column(field, DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())); + } + builder.option("bucket", bucket) + .option("file.format", format) + .option(CoreOptions.WRITE_ONLY.key(), "true"); + if (!"-1".equals(bucket)) { + builder.option("bucket-key", "id"); + } + for (String field : sharedShreddingFields) { + builder.option("fields." + field + ".map.storage-layout", "shared-shredding") + .option( + "fields." + field + ".map.shared-shredding.max-columns", + String.valueOf(maxColumns)); + } + return builder.build(); + } + + private List currentDataFiles(FileStoreTable table) throws Exception { + List files = new ArrayList<>(); + for (DataSplit split : table.newSnapshotReader().read().dataSplits()) { + for (DataFileMeta dataFile : split.dataFiles()) { + files.add(new DataFileWithSplit(split.partition(), split.bucket(), dataFile)); + } + } + return files; + } + + private MapSharedShreddingFieldMeta readSharedShreddingFieldMeta( + FileStoreTable table, DataFileWithSplit file, String fieldName) throws Exception { + DataFilePathFactory pathFactory = + table.store() + .pathFactory() + .createDataFilePathFactory(file.partition, file.bucket); + FileFormat fileFormat = + FileFormatDiscover.of(new CoreOptions(table.options())) + .discover(file.dataFile.fileFormat()); + Map> fieldMetadata = + ((SupportsFieldMetadata) fileFormat) + .readFieldMetadata( + new FormatReaderContext( + table.fileIO(), + pathFactory.toPath(file.dataFile), + file.dataFile.fileSize())); + return MapSharedShreddingUtils.deserializeMetadata( + fieldMetadata.get(fieldName), MapSharedShreddingDefine.DEFAULT_DICT_COMPRESSION); + } + + private GenericMap mapOf(Object... entries) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < entries.length; i += 2) { + map.put(BinaryString.fromString((String) entries[i]), (Long) entries[i + 1]); + } + return new GenericMap(map); + } + + private Map toJavaMap(InternalMap map) { + Map result = new LinkedHashMap<>(); + for (int i = 0; i < map.size(); i++) { + result.put( + map.keyArray().getString(i).toString(), + map.valueArray().isNullAt(i) ? null : map.valueArray().getLong(i)); + } + return result; + } + + private Map javaMapOf(Object... entries) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < entries.length; i += 2) { + map.put((String) entries[i], (Long) entries[i + 1]); + } + return map; + } + + @Override + protected Schema schemaDefault() { + return schema("parquet", "metrics"); + } + + private static class DataFileWithSplit { + + private final BinaryRow partition; + private final int bucket; + private final DataFileMeta dataFile; + + private DataFileWithSplit(BinaryRow partition, int bucket, DataFileMeta dataFile) { + this.partition = partition; + this.bucket = bucket; + this.dataFile = dataFile; + } + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/SupportsReaderFieldMetadata.java b/paimon-format/src/main/java/org/apache/paimon/format/SupportsFieldMetadata.java similarity index 82% rename from paimon-format/src/main/java/org/apache/paimon/format/SupportsReaderFieldMetadata.java rename to paimon-format/src/main/java/org/apache/paimon/format/SupportsFieldMetadata.java index 0dc9f191fb21..ca0e487bc015 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/SupportsReaderFieldMetadata.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/SupportsFieldMetadata.java @@ -21,8 +21,8 @@ import java.io.IOException; import java.util.Map; -/** Reader capability for formats that can recover top-level field metadata. */ -public interface SupportsReaderFieldMetadata { +/** Format capability for recovering top-level field metadata from a file footer or schema. */ +public interface SupportsFieldMetadata { /** * Reads metadata from top-level file fields. @@ -31,5 +31,6 @@ public interface SupportsReaderFieldMetadata { * attached to that field. Implementations return an empty map when the file does not contain * field metadata. */ - Map> readFieldMetadata() throws IOException; + Map> readFieldMetadata(FormatReaderFactory.Context context) + throws IOException; } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java index 01c2fa95173d..73f2318c5058 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java @@ -25,6 +25,7 @@ import org.apache.paimon.format.FormatReaderFactory; import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.format.SimpleStatsExtractor; +import org.apache.paimon.format.SupportsFieldMetadata; import org.apache.paimon.format.orc.filter.OrcFilters; import org.apache.paimon.format.orc.filter.OrcPredicateFunctionVisitor; import org.apache.paimon.format.orc.filter.OrcSimpleStatsExtractor; @@ -49,8 +50,10 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; +import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.stream.Collectors; @@ -60,7 +63,7 @@ /** Orc {@link FileFormat}. */ @ThreadSafe -public class OrcFileFormat extends FileFormat { +public class OrcFileFormat extends FileFormat implements SupportsFieldMetadata { public static final String IDENTIFIER = "orc"; @@ -129,6 +132,19 @@ public FormatReaderFactory createReaderFactory( legacyTimestampLtzType); } + @Override + public Map> readFieldMetadata(FormatReaderFactory.Context context) + throws IOException { + org.apache.orc.Reader reader = + OrcReaderFactory.createReader( + readerConf, context.fileIO(), context.filePath(), context.selection()); + try { + return OrcReaderFactory.readFieldMetadata(reader); + } finally { + reader.close(); + } + } + @Override public void validateDataFields(RowType rowType) { DataType refinedType = refineDataType(rowType); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java index 63f07ad152f1..5000794c9cbf 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java @@ -24,10 +24,12 @@ import org.apache.paimon.data.columnar.ColumnarRowIterator; import org.apache.paimon.data.columnar.VectorizedColumnBatch; import org.apache.paimon.data.columnar.VectorizedRowIterator; +import org.apache.paimon.data.shredding.MapSharedShreddingFieldMeta; +import org.apache.paimon.data.shredding.MapSharedShreddingReader; +import org.apache.paimon.data.shredding.MapSharedShreddingUtils; import org.apache.paimon.format.FormatMetadataUtils; import org.apache.paimon.format.FormatReaderFactory; import org.apache.paimon.format.OrcFormatReaderContext; -import org.apache.paimon.format.SupportsReaderFieldMetadata; import org.apache.paimon.format.fs.HadoopReadOnlyFileSystem; import org.apache.paimon.format.orc.filter.OrcFilters; import org.apache.paimon.fs.FileIO; @@ -100,27 +102,58 @@ public OrcReaderFactory( // ------------------------------------------------------------------------ @Override - public OrcVectorizedReader createReader(FormatReaderFactory.Context context) + public FileRecordReader createReader(FormatReaderFactory.Context context) throws IOException { int poolSize = context instanceof OrcFormatReaderContext ? ((OrcFormatReaderContext) context).poolSize() : 1; - Pool poolOfBatches = - createPoolOfBatches(context.filePath(), poolSize, context.fileIO()); - - OrcRecordReader orcReader = - createRecordReader( - hadoopConfig, - schema, - conjunctPredicates, - context.fileIO(), - context.filePath(), - 0, - context.fileSize(), - context.selection(), - deletionVectorsEnabled); - return new OrcVectorizedReader(orcReader, poolOfBatches); + + org.apache.orc.Reader fileReader = + createReader( + hadoopConfig, context.fileIO(), context.filePath(), context.selection()); + try { + Map sharedShreddingFieldMetas = + MapSharedShreddingReader.readSharedShreddingMetas( + readFieldMetadata(fileReader)); + RowType readType = tableType; + TypeDescription readSchema = schema; + RowType physicalReadType = + MapSharedShreddingUtils.buildPhysicalReadType( + tableType, sharedShreddingFieldMetas); + if (physicalReadType != tableType) { + readType = physicalReadType; + readSchema = convertToOrcSchema(readType); + } + Pool poolOfBatches = + createPoolOfBatches( + context.filePath(), + poolSize, + context.fileIO(), + readSchema, + readType); + + OrcRecordReader orcReader = + createRecordReader( + hadoopConfig, + fileReader, + readSchema, + conjunctPredicates, + 0, + context.fileSize(), + context.selection(), + deletionVectorsEnabled); + OrcVectorizedReader orcVectorizedReader = + new OrcVectorizedReader(orcReader, poolOfBatches); + if (physicalReadType == tableType) { + return orcVectorizedReader; + } + return new MapSharedShreddingReader( + orcVectorizedReader, tableType, sharedShreddingFieldMetas); + } catch (IOException | RuntimeException e) { + IOUtils.closeQuietly(fileReader); + throw e; + } } /** @@ -133,11 +166,20 @@ public OrcReaderBatch createReaderBatch( VectorizedRowBatch orcBatch, Pool.Recycler recycler, FileIO fileIO) { - List tableFieldNames = tableType.getFieldNames(); - List tableFieldTypes = tableType.getFieldTypes(); + return createReaderBatch(filePath, orcBatch, recycler, fileIO, tableType); + } + + private OrcReaderBatch createReaderBatch( + Path filePath, + VectorizedRowBatch orcBatch, + Pool.Recycler recycler, + FileIO fileIO, + RowType readType) { + List tableFieldNames = readType.getFieldNames(); + List tableFieldTypes = readType.getFieldTypes(); // create and initialize the row batch - ColumnVector[] vectors = new ColumnVector[tableType.getFieldCount()]; + ColumnVector[] vectors = new ColumnVector[readType.getFieldCount()]; for (int i = 0; i < vectors.length; i++) { String name = tableFieldNames.get(i); DataType type = tableFieldTypes.get(i); @@ -154,13 +196,19 @@ public OrcReaderBatch createReaderBatch( // ------------------------------------------------------------------------ - private Pool createPoolOfBatches(Path filePath, int numBatches, FileIO fileIO) { + private Pool createPoolOfBatches( + Path filePath, + int numBatches, + FileIO fileIO, + TypeDescription readSchema, + RowType readType) { final Pool pool = new Pool<>(numBatches); for (int i = 0; i < numBatches; i++) { - final VectorizedRowBatch orcBatch = createBatchWrapper(schema, batchSize / numBatches); + final VectorizedRowBatch orcBatch = + createBatchWrapper(readSchema, batchSize / numBatches); final OrcReaderBatch batch = - createReaderBatch(filePath, orcBatch, pool.recycler(), fileIO); + createReaderBatch(filePath, orcBatch, pool.recycler(), fileIO, readType); pool.add(batch); } @@ -229,8 +277,7 @@ private ColumnarRowIterator convertAndGetIterator( * batch is addressed by the starting row number of the batch, plus the number of records to be * skipped before. */ - private static final class OrcVectorizedReader - implements FileRecordReader, SupportsReaderFieldMetadata { + private static final class OrcVectorizedReader implements FileRecordReader { private final OrcRecordReader orcReader; private final Pool pool; @@ -256,30 +303,6 @@ public ColumnarRowIterator readBatch() throws IOException { return batch.convertAndGetIterator(orcVectorBatch, rowNumber); } - @Override - public Map> readFieldMetadata() { - org.apache.orc.Reader fileReader = orcReader.fileReader; - if (!fileReader - .getMetadataKeys() - .contains(FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY)) { - return Collections.emptyMap(); - } - String encodedSchema = - StandardCharsets.UTF_8 - .decode( - fileReader - .getMetadataValue( - FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY) - .duplicate()) - .toString(); - return FormatMetadataUtils.readFieldMetadata( - FormatMetadataUtils.decodeMetadata( - Collections.singletonMap( - FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY, - encodedSchema)) - .get(FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY)); - } - @Override public void close() throws IOException { try { @@ -312,16 +335,14 @@ private OrcRecordReader(org.apache.orc.Reader fileReader, RecordReader recordRea private static OrcRecordReader createRecordReader( org.apache.hadoop.conf.Configuration conf, + org.apache.orc.Reader orcReader, TypeDescription schema, List conjunctPredicates, - FileIO fileIO, - org.apache.paimon.fs.Path path, long splitStart, long splitLength, @Nullable RoaringBitmap32 selection, boolean deletionVectorsEnabled) throws IOException { - org.apache.orc.Reader orcReader = createReader(conf, fileIO, path, selection); try { // get offset and length for the stripes that start in the split Pair offsetAndLength = @@ -368,6 +389,27 @@ private static OrcRecordReader createRecordReader( } } + public static Map> readFieldMetadata( + org.apache.orc.Reader fileReader) { + if (!fileReader.getMetadataKeys().contains(FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY)) { + return Collections.emptyMap(); + } + String encodedSchema = + StandardCharsets.UTF_8 + .decode( + fileReader + .getMetadataValue( + FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY) + .duplicate()) + .toString(); + return FormatMetadataUtils.readFieldMetadata( + FormatMetadataUtils.decodeMetadata( + Collections.singletonMap( + FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY, + encodedSchema)) + .get(FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY)); + } + private static VectorizedRowBatch createBatchWrapper(TypeDescription schema, int batchSize) { return schema.createRowBatch(batchSize); } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormat.java index 66ab9565e70e..d663704d392b 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormat.java @@ -24,6 +24,7 @@ import org.apache.paimon.format.FormatReaderFactory; import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.format.SimpleStatsExtractor; +import org.apache.paimon.format.SupportsFieldMetadata; import org.apache.paimon.format.parquet.writer.RowDataParquetBuilder; import org.apache.paimon.format.variant.VariantInferenceConfig; import org.apache.paimon.format.variant.VariantInferenceWriterFactory; @@ -34,18 +35,22 @@ import org.apache.paimon.statistics.SimpleColStatsCollector; import org.apache.paimon.types.RowType; +import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.filter2.predicate.ParquetFilters; +import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.ParquetOutputFormat; import javax.annotation.Nullable; +import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.apache.paimon.format.parquet.ParquetFileFormatFactory.IDENTIFIER; /** Parquet {@link FileFormat}. */ -public class ParquetFileFormat extends FileFormat { +public class ParquetFileFormat extends FileFormat implements SupportsFieldMetadata { private final FormatContext formatContext; private final Options options; @@ -73,6 +78,23 @@ public FormatReaderFactory createReaderFactory( options, projectedRowType, readBatchSize, ParquetFilters.convert(filters)); } + @Override + public Map> readFieldMetadata(FormatReaderFactory.Context context) + throws IOException { + ParquetReadOptions readOptions = ParquetUtil.getParquetReadOptionsBuilder(options).build(); + ParquetFileReader reader = + new ParquetFileReader( + ParquetInputFile.fromPath( + context.fileIO(), context.filePath(), context.fileSize()), + readOptions, + null); + try { + return ParquetReaderFactory.readFieldMetadata(reader); + } finally { + reader.close(); + } + } + @Override public FormatWriterFactory createWriterFactory(RowType type) { ParquetWriterFactory baseFactory = diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java index 43c516a48ce9..202e904dd6e3 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java @@ -22,9 +22,13 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.columnar.VectorizedColumnBatch; import org.apache.paimon.data.columnar.writable.WritableColumnVector; +import org.apache.paimon.data.shredding.MapSharedShreddingFieldMeta; +import org.apache.paimon.data.shredding.MapSharedShreddingReader; +import org.apache.paimon.data.shredding.MapSharedShreddingUtils; import org.apache.paimon.data.variant.PaimonShreddingUtils; import org.apache.paimon.data.variant.VariantMetadataUtils; import org.apache.paimon.data.variant.VariantPathSegment; +import org.apache.paimon.format.FormatMetadataUtils; import org.apache.paimon.format.FormatReaderFactory; import org.apache.paimon.format.parquet.reader.VectorizedParquetRecordReader; import org.apache.paimon.format.parquet.type.ParquetField; @@ -58,6 +62,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -81,7 +86,7 @@ public class ParquetReaderFactory implements FormatReaderFactory { private static final Logger LOG = LoggerFactory.getLogger(ParquetReaderFactory.class); private final Options conf; - private final DataField[] readFields; + private final RowType readType; private final int batchSize; private final boolean caseSensitive; @Nullable private final FilterCompat.Filter filter; @@ -102,7 +107,7 @@ public class ParquetReaderFactory implements FormatReaderFactory { public ParquetReaderFactory( Options conf, RowType readType, int batchSize, @Nullable FilterCompat.Filter filter) { this.conf = conf; - this.readFields = readType.getFields().toArray(new DataField[0]); + this.readType = readType; this.batchSize = batchSize; this.caseSensitive = conf.getOptional(CatalogOptions.CASE_SENSITIVE).orElse(true); this.filter = filter; @@ -128,7 +133,18 @@ public FileRecordReader createReader(FormatReaderFactory.Context co builder.build(), context.selection()); MessageType fileSchema = reader.getFileMetaData().getSchema(); - RequestedSchema requestedSchema = getOrCreateRequestedSchema(fileSchema); + Map sharedShreddingFieldMetas = + MapSharedShreddingReader.readSharedShreddingMetas(readFieldMetadata(reader)); + RowType physicalReadType = + MapSharedShreddingUtils.buildPhysicalReadType(readType, sharedShreddingFieldMetas); + DataField[] physicalReadFields = readFields(readType); + RequestedSchema requestedSchema; + if (physicalReadType != readType) { + physicalReadFields = readFields(physicalReadType); + requestedSchema = createRequestedSchema(fileSchema, physicalReadFields); + } else { + requestedSchema = getOrCreateRequestedSchema(fileSchema); + } if (LOG.isDebugEnabled()) { LOG.debug( @@ -144,16 +160,22 @@ public FileRecordReader createReader(FormatReaderFactory.Context co "Parquet read batch size should be positive: %s", actualBatchSize); reader.setRequestedSchema(requestedSchema.messageType); - WritableColumnVector[] writableVectors = createWritableVectors(actualBatchSize); - - return new VectorizedParquetRecordReader( - context.filePath(), - reader, - fileSchema, - requestedSchema.fields, - writableVectors, - actualBatchSize, - context.fileIO()); + WritableColumnVector[] writableVectors = + createWritableVectors(actualBatchSize, physicalReadFields); + + VectorizedParquetRecordReader parquetReader = + new VectorizedParquetRecordReader( + context.filePath(), + reader, + fileSchema, + requestedSchema.fields, + writableVectors, + actualBatchSize, + context.fileIO()); + if (physicalReadType == readType) { + return parquetReader; + } + return new MapSharedShreddingReader(parquetReader, readType, sharedShreddingFieldMetas); } private RequestedSchema getOrCreateRequestedSchema(MessageType fileSchema) { @@ -167,14 +189,22 @@ private RequestedSchema getOrCreateRequestedSchema(MessageType fileSchema) { } private RequestedSchema createRequestedSchema(MessageType fileSchema) { - MessageType rs = clipParquetSchema(fileSchema); + return createRequestedSchema(fileSchema, readFields(readType)); + } + + private static DataField[] readFields(RowType readType) { + return readType.getFields().toArray(new DataField[0]); + } + + private RequestedSchema createRequestedSchema(MessageType fileSchema, DataField[] readFields) { + MessageType rs = clipParquetSchema(fileSchema, readFields); MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(rs); List f = buildFieldsList(readFields, columnIO, rs); return new RequestedSchema(rs, f); } /** Clips `parquetSchema` according to `fieldNames`. */ - private MessageType clipParquetSchema(GroupType parquetSchema) { + private MessageType clipParquetSchema(GroupType parquetSchema, DataField[] readFields) { Type[] types = new Type[readFields.length]; for (int i = 0; i < readFields.length; ++i) { String fieldName = readFields[i].name(); @@ -357,7 +387,7 @@ protected int computeBatchSize(ParquetFileReader reader, MessageType requestedSc return batchSize; } - private WritableColumnVector[] createWritableVectors(int batchSize) { + private WritableColumnVector[] createWritableVectors(int batchSize, DataField[] readFields) { WritableColumnVector[] columns = new WritableColumnVector[readFields.length]; for (int i = 0; i < readFields.length; i++) { columns[i] = createWritableColumnVector(batchSize, readFields[i].type()); @@ -365,6 +395,23 @@ private WritableColumnVector[] createWritableVectors(int batchSize) { return columns; } + public static Map> readFieldMetadata(ParquetFileReader reader) { + String encodedSchema = + reader.getFooter() + .getFileMetaData() + .getKeyValueMetaData() + .get(FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY); + byte[] schemaMetadata = + encodedSchema == null + ? null + : FormatMetadataUtils.decodeMetadata( + Collections.singletonMap( + FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY, + encodedSchema)) + .get(FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY); + return FormatMetadataUtils.readFieldMetadata(schemaMetadata); + } + private static class RequestedSchema { private final MessageType messageType; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/VectorizedParquetRecordReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/VectorizedParquetRecordReader.java index ebc38e31a729..af63a7d9f96b 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/VectorizedParquetRecordReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/VectorizedParquetRecordReader.java @@ -20,8 +20,6 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.columnar.writable.WritableColumnVector; -import org.apache.paimon.format.FormatMetadataUtils; -import org.apache.paimon.format.SupportsReaderFieldMetadata; import org.apache.paimon.format.parquet.type.ParquetField; import org.apache.paimon.format.parquet.type.ParquetPrimitiveField; import org.apache.paimon.fs.FileIO; @@ -41,10 +39,8 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -52,8 +48,7 @@ import static org.apache.paimon.format.parquet.reader.ParquetReaderUtil.createReadableColumnVectors; /** Record reader for parquet. */ -public class VectorizedParquetRecordReader - implements FileRecordReader, SupportsReaderFieldMetadata { +public class VectorizedParquetRecordReader implements FileRecordReader { private ParquetFileReader reader; @@ -274,24 +269,6 @@ private void initColumnReader(PageReadStore pages, ParquetColumnVector cv) throw } } - @Override - public Map> readFieldMetadata() throws IOException { - String encodedSchema = - reader.getFooter() - .getFileMetaData() - .getKeyValueMetaData() - .get(FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY); - if (encodedSchema == null) { - return FormatMetadataUtils.readFieldMetadata(null); - } - return FormatMetadataUtils.readFieldMetadata( - FormatMetadataUtils.decodeMetadata( - Collections.singletonMap( - FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY, - encodedSchema)) - .get(FormatMetadataUtils.ARROW_SCHEMA_METADATA_KEY)); - } - @Override public void close() throws IOException { if (reader != null) { diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java index a6084dd4e331..310514f720f9 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java @@ -29,7 +29,7 @@ import org.apache.paimon.format.FormatReaderContext; import org.apache.paimon.format.FormatWriter; import org.apache.paimon.format.OrcOptions; -import org.apache.paimon.format.SupportsReaderFieldMetadata; +import org.apache.paimon.format.SupportsFieldMetadata; import org.apache.paimon.format.SupportsWriterMetadata; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.Options; @@ -132,20 +132,14 @@ public void testWriteMetadata() throws IOException { FormatReaderContext context = new FormatReaderContext(fileIO, file, fileIO.getFileSize(file)); - RowType emptyRowType = new RowType(Collections.emptyList()); - try (FileRecordReader reader = - newFormat - .createReaderFactory(emptyRowType, emptyRowType, Collections.emptyList()) - .createReader(context)) { - Map> readFieldMetadata = - ((SupportsReaderFieldMetadata) reader).readFieldMetadata(); - assertThat(readFieldMetadata).containsOnlyKeys("id", "name"); - assertThat(readFieldMetadata.get("id")) - .containsEntry(FormatMetadataUtils.ORC_FIELD_ID_KEY, "0"); - assertThat(readFieldMetadata.get("name")).containsAllEntriesOf(fieldMetadata); - assertThat(readFieldMetadata.get("name")) - .containsEntry(FormatMetadataUtils.ORC_FIELD_ID_KEY, "1"); - } + Map> readFieldMetadata = + ((SupportsFieldMetadata) newFormat).readFieldMetadata(context); + assertThat(readFieldMetadata).containsOnlyKeys("id", "name"); + assertThat(readFieldMetadata.get("id")) + .containsEntry(FormatMetadataUtils.ORC_FIELD_ID_KEY, "0"); + assertThat(readFieldMetadata.get("name")).containsAllEntriesOf(fieldMetadata); + assertThat(readFieldMetadata.get("name")) + .containsEntry(FormatMetadataUtils.ORC_FIELD_ID_KEY, "1"); } @Test diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java index e4d03b8afc02..75beb963c5fa 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java @@ -27,7 +27,7 @@ import org.apache.paimon.format.FormatReadWriteTest; import org.apache.paimon.format.FormatReaderContext; import org.apache.paimon.format.FormatWriter; -import org.apache.paimon.format.SupportsReaderFieldMetadata; +import org.apache.paimon.format.SupportsFieldMetadata; import org.apache.paimon.format.SupportsWriterMetadata; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.Options; @@ -110,20 +110,14 @@ public void testWriteMetadata() throws Exception { FormatReaderContext context = new FormatReaderContext(fileIO, file, fileIO.getFileSize(file)); - RowType emptyRowType = new RowType(Collections.emptyList()); - try (FileRecordReader reader = - format.createReaderFactory(emptyRowType, emptyRowType, Collections.emptyList()) - .createReader(context)) { - Map> readFieldMetadata = - ((SupportsReaderFieldMetadata) reader).readFieldMetadata(); - Assertions.assertThat(readFieldMetadata).containsKey("id").containsKey("name"); - Assertions.assertThat(readFieldMetadata.get("id")) - .containsEntry(FormatMetadataUtils.PARQUET_FIELD_ID_KEY, "0"); - Assertions.assertThat(readFieldMetadata.get("name")) - .containsAllEntriesOf(fieldMetadata); - Assertions.assertThat(readFieldMetadata.get("name")) - .containsEntry(FormatMetadataUtils.PARQUET_FIELD_ID_KEY, "1"); - } + Map> readFieldMetadata = + ((SupportsFieldMetadata) format).readFieldMetadata(context); + Assertions.assertThat(readFieldMetadata).containsKey("id").containsKey("name"); + Assertions.assertThat(readFieldMetadata.get("id")) + .containsEntry(FormatMetadataUtils.PARQUET_FIELD_ID_KEY, "0"); + Assertions.assertThat(readFieldMetadata.get("name")).containsAllEntriesOf(fieldMetadata); + Assertions.assertThat(readFieldMetadata.get("name")) + .containsEntry(FormatMetadataUtils.PARQUET_FIELD_ID_KEY, "1"); } @ParameterizedTest From dc94d681e11d4bb524bc6319d34a7af09f89179a Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Sat, 27 Jun 2026 10:35:15 +0800 Subject: [PATCH 5/6] fix code style --- .../shredding/MapSharedShreddingReader.java | 3 +- .../shredding/MapSharedShreddingUtils.java | 4 ++- .../MapSharedShreddingCoreUtils.java | 5 +--- .../paimon/append/AppendOnlyWriterTest.java | 3 +- .../table/MapSharedShreddingTableTest.java | 30 +++++-------------- .../paimon/format/orc/OrcReaderFactory.java | 6 +--- .../format/orc/OrcFormatReadWriteTest.java | 1 - .../parquet/ParquetFormatReadWriteTest.java | 3 -- 8 files changed, 15 insertions(+), 40 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingReader.java b/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingReader.java index 258bca5d76bd..990c131e1005 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingReader.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingReader.java @@ -319,8 +319,7 @@ private SharedShreddingContext(MapSharedShreddingFieldMeta fieldMeta, DataType f } this.valueGetters = new InternalRow.FieldGetter[fieldMeta.numColumns()]; for (int i = 0; i < fieldMeta.numColumns(); i++) { - this.valueGetters[i] = - InternalRow.createFieldGetter(mapType.getValueType(), i + 1); + this.valueGetters[i] = InternalRow.createFieldGetter(mapType.getValueType(), i + 1); } this.overflowValueGetter = InternalArray.createElementGetter(mapType.getValueType()); this.numColumns = fieldMeta.numColumns(); diff --git a/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingUtils.java b/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingUtils.java index e2ecaecc9814..e6e1dbca0264 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingUtils.java @@ -237,7 +237,9 @@ public static RowType buildPhysicalReadType( physicalReadFields.add(logicalReadField.newType(physicalType)); converted = true; } - return converted ? new RowType(logicalReadType.isNullable(), physicalReadFields) : logicalReadType; + return converted + ? new RowType(logicalReadType.isNullable(), physicalReadFields) + : logicalReadType; } private static RowType buildPhysicalStructType(DataType valueType, int numColumns) { diff --git a/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java index 91738cd7481e..10687521ea20 100644 --- a/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/data/shredding/MapSharedShreddingCoreUtils.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -113,9 +112,7 @@ private static void restoreRecentFileStats( ((SupportsFieldMetadata) fileFormat) .readFieldMetadata( new FormatReaderContext( - fileIO, - pathFactory.toPath(file), - file.fileSize())); + fileIO, pathFactory.toPath(file), file.fileSize())); for (String fieldName : candidateFields) { Map metadata = fieldMetadata.get(fieldName); if (!MapSharedShreddingUtils.hasShreddingMetadata(metadata)) { diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java index 757567c5cfc6..6ddd5d1c866c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java @@ -1147,8 +1147,7 @@ private void assertSharedShreddingFileSchema( throws IOException { Map> fieldMetadata = ((SupportsFieldMetadata) context.format) - .readFieldMetadata( - new FormatReaderContext(context.fileIO, path, fileSize)); + .readFieldMetadata(new FormatReaderContext(context.fileIO, path, fileSize)); for (Map.Entry entry : expectedMetas.entrySet()) { String fieldName = entry.getKey(); assertThat(expectedPhysicalType.containsField(fieldName)).isTrue(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/MapSharedShreddingTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/MapSharedShreddingTableTest.java index 0576db38681e..58a5fc993e6c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/MapSharedShreddingTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/MapSharedShreddingTableTest.java @@ -174,9 +174,7 @@ public void testSwitchMapLayoutAndUseMaxColumnsWithoutMetadata(String format) th Arrays.asList("metrics", "labels"), Arrays.asList("labels")); - write( - table, - GenericRow.of(1, mapOf("a", 11L, "b", 12L), mapOf("x", 21L))); + write(table, GenericRow.of(1, mapOf("a", 11L, "b", 12L), mapOf("x", 21L))); catalog.alterTable( identifier(format), @@ -189,9 +187,7 @@ public void testSwitchMapLayoutAndUseMaxColumnsWithoutMetadata(String format) th false); table = catalog.getTable(identifier(format)); - write( - table, - GenericRow.of(2, mapOf("c", 31L), mapOf("y", 41L, "z", 42L))); + write(table, GenericRow.of(2, mapOf("c", 31L), mapOf("y", 41L, "z", 42L))); FileStoreTable fileStoreTable = (FileStoreTable) table; List files = currentDataFiles(fileStoreTable); @@ -213,14 +209,9 @@ public void testSwitchMapLayoutAndUseMaxColumnsWithoutMetadata(String format) th } assertThat(actual) + .containsEntry(1, Arrays.asList(javaMapOf("a", 11L, "b", 12L), javaMapOf("x", 21L))) .containsEntry( - 1, - Arrays.asList( - javaMapOf("a", 11L, "b", 12L), javaMapOf("x", 21L))) - .containsEntry( - 2, - Arrays.asList( - javaMapOf("c", 31L), javaMapOf("y", 41L, "z", 42L))); + 2, Arrays.asList(javaMapOf("c", 31L), javaMapOf("y", 41L, "z", 42L))); } @ParameterizedTest @@ -293,11 +284,9 @@ public void testDataEvolutionMergeWithOverwrittenSharedShreddingMaps(String form assertThat(actual) .containsEntry( - 101, - Arrays.asList(javaMapOf("new-a", 111L), javaMapOf("label-a", 101L))) + 101, Arrays.asList(javaMapOf("new-a", 111L), javaMapOf("label-a", 101L))) .containsEntry( - 102, - Arrays.asList(javaMapOf("new-b", 222L), javaMapOf("label-b", 102L))); + 102, Arrays.asList(javaMapOf("new-b", 222L), javaMapOf("label-b", 102L))); Map>> partialActual = new LinkedHashMap<>(); ReadBuilder readBuilder = @@ -316,8 +305,7 @@ public void testDataEvolutionMergeWithOverwrittenSharedShreddingMaps(String form assertThat(partialActual) .containsOnlyKeys(102) .containsEntry( - 102, - Arrays.asList(javaMapOf("new-b", 222L), javaMapOf("label-b", 102L))); + 102, Arrays.asList(javaMapOf("new-b", 222L), javaMapOf("label-b", 102L))); } private Table createTable(String format, String... sharedShreddingFields) throws Exception { @@ -465,9 +453,7 @@ private List currentDataFiles(FileStoreTable table) throws Ex private MapSharedShreddingFieldMeta readSharedShreddingFieldMeta( FileStoreTable table, DataFileWithSplit file, String fieldName) throws Exception { DataFilePathFactory pathFactory = - table.store() - .pathFactory() - .createDataFilePathFactory(file.partition, file.bucket); + table.store().pathFactory().createDataFilePathFactory(file.partition, file.bucket); FileFormat fileFormat = FileFormatDiscover.of(new CoreOptions(table.options())) .discover(file.dataFile.fileFormat()); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java index 5000794c9cbf..687ecbd1d136 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java @@ -127,11 +127,7 @@ public FileRecordReader createReader(FormatReaderFactory.Context co } Pool poolOfBatches = createPoolOfBatches( - context.filePath(), - poolSize, - context.fileIO(), - readSchema, - readType); + context.filePath(), poolSize, context.fileIO(), readSchema, readType); OrcRecordReader orcReader = createRecordReader( diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java index 310514f720f9..7c91b51ab8d4 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java @@ -33,7 +33,6 @@ import org.apache.paimon.format.SupportsWriterMetadata; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.Options; -import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java index 75beb963c5fa..fd1552189f03 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java @@ -20,7 +20,6 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; -import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FileFormatFactory; import org.apache.paimon.format.FormatMetadataUtils; @@ -31,7 +30,6 @@ import org.apache.paimon.format.SupportsWriterMetadata; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.Options; -import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -47,7 +45,6 @@ import org.junit.jupiter.params.provider.ValueSource; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; From 636ab7fe109042a2a2b8c689f997b73df8449bdf Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Mon, 29 Jun 2026 14:40:21 +0800 Subject: [PATCH 6/6] add pk and write-only validate --- .../paimon/schema/SchemaValidation.java | 16 +++++++ .../BucketedAppendFileStoreWriteTest.java | 1 + .../paimon/schema/SchemaValidationTest.java | 46 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 32db6182d96d..aba4b88793cb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -656,10 +656,26 @@ private static void validateMapStorageLayout(TableSchema schema, CoreOptions opt options.mapSharedShreddingMaxColumns(fieldName); } if (hasSharedShredding) { + validateMapSharedShreddingTableMode(schema, options); validateMapSharedShreddingFileFormats(options); } } + private static void validateMapSharedShreddingTableMode( + TableSchema schema, CoreOptions options) { + if (!schema.primaryKeys().isEmpty()) { + throw new IllegalArgumentException( + "MAP shared-shredding is only supported for append-only tables currently."); + } + + if (options.bucket() != -1 && !options.writeOnly()) { + throw new IllegalArgumentException( + String.format( + "MAP shared-shredding is only supported for append-only tables with '%s' = -1 or '%s' = true currently.", + CoreOptions.BUCKET.key(), CoreOptions.WRITE_ONLY.key())); + } + } + private static void validateMapSharedShreddingFileFormats(CoreOptions options) { validateMapSharedShreddingFileFormat( CoreOptions.FILE_FORMAT.key(), options.fileFormatString()); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/BucketedAppendFileStoreWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/BucketedAppendFileStoreWriteTest.java index fbdac50a5300..b38bddca5d51 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/BucketedAppendFileStoreWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/BucketedAppendFileStoreWriteTest.java @@ -187,6 +187,7 @@ public void testSharedShreddingDoesNotSupportRewrite() throws Exception { .column("tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())) .option("bucket", "1") .option("bucket-key", "id") + .option(WRITE_ONLY.key(), "true") .option("fields.tags.map.storage-layout", "shared-shredding") .build(); Identifier compactIdentifier = Identifier.create("default", "compact_test"); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java index 29a256abdbbf..6d1ae9450356 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java @@ -236,6 +236,52 @@ public void testMapStorageLayout() { options, ""))); + Map primaryKeyOptions = new HashMap<>(options); + assertThatThrownBy( + () -> + validateTableSchema( + new TableSchema( + 1, + fields, + 10, + emptyList(), + singletonList("id"), + primaryKeyOptions, + ""))) + .hasMessageContaining( + "MAP shared-shredding is only supported for append-only tables currently."); + + Map fixedBucketOptions = new HashMap<>(options); + fixedBucketOptions.put(BUCKET.key(), "1"); + fixedBucketOptions.put(CoreOptions.BUCKET_KEY.key(), "id"); + assertThatThrownBy( + () -> + validateTableSchema( + new TableSchema( + 1, + fields, + 10, + emptyList(), + emptyList(), + fixedBucketOptions, + ""))) + .hasMessageContaining( + "MAP shared-shredding is only supported for append-only tables with 'bucket' = -1 or 'write-only' = true currently."); + + fixedBucketOptions.put(CoreOptions.WRITE_ONLY.key(), "true"); + assertThatNoException() + .isThrownBy( + () -> + validateTableSchema( + new TableSchema( + 1, + fields, + 10, + emptyList(), + emptyList(), + fixedBucketOptions, + ""))); + options.put("fields.metrics.map.storage-layout", "default"); assertThatNoException() .isThrownBy(