From 7a3691a639752e1c23bb79fab151fe615cde2a0b Mon Sep 17 00:00:00 2001 From: Raghav Aggarwal Date: Fri, 3 Jul 2026 22:12:52 +0530 Subject: [PATCH 1/2] HIVE-29695: ClassCastException in MapJoin when Parquet String is mapped to Date --- .../ql/io/parquet/convert/ETypeConverter.java | 14 ++++ .../parquet/convert/TestETypeConverter.java | 30 +++++++ .../parquet_string_to_date_mapjoin.q | 25 ++++++ .../llap/parquet_string_to_date_mapjoin.q.out | 81 +++++++++++++++++++ 4 files changed, 150 insertions(+) create mode 100644 ql/src/test/queries/clientpositive/parquet_string_to_date_mapjoin.q create mode 100644 ql/src/test/results/clientpositive/llap/parquet_string_to_date_mapjoin.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java index c1bbd35ddeaa..8660d373d766 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java @@ -16,6 +16,7 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.time.ZoneOffset; import java.util.ArrayList; @@ -24,6 +25,7 @@ import java.util.TimeZone; import com.google.common.base.MoreObjects; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.conf.HiveConf; @@ -525,6 +527,18 @@ protected HiveVarcharWritable convert(Binary binary) { return new HiveVarcharWritable(binary.getBytes(), ((VarcharTypeInfo) hiveTypeInfo).getLength()); } }; + case DATE: + return new BinaryConverter(type, parent, index) { + @Override + protected DateWritableV2 convert(Binary binary) { + String s = new String(binary.getBytes(), StandardCharsets.UTF_8); + try { + return new DateWritableV2(Date.valueOf(s)); + } catch (IllegalArgumentException e) { + return null; + } + } + }; } } // STRING type diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java index 3173d2db9007..ff0b31a3982c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.stringTypeInfo; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.nio.ByteBuffer; @@ -33,6 +34,7 @@ import org.apache.hadoop.hive.ql.io.parquet.convert.ETypeConverter.BinaryConverter; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; @@ -341,6 +343,34 @@ public void testGetTextConverterNoHiveTypeInfo() { assertEquals(value, textWritable.toString()); } + @Test + public void testGetTextConverterForDate() { + PrimitiveType primitiveType = + Types.optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("value"); + String value = "2026-01-01"; + DateWritableV2 dateWritable = + (DateWritableV2) + getWritableFromBinaryConverter( + TypeInfoFactory.dateTypeInfo, primitiveType, Binary.fromString(value)); + assertEquals(org.apache.hadoop.hive.common.type.Date.valueOf(value), dateWritable.get()); + } + + @Test + public void testGetTextConverterForDateInvalid() { + PrimitiveType primitiveType = + Types.optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("value"); + String value = "invalid_date"; + DateWritableV2 dateWritable = + (DateWritableV2) + getWritableFromBinaryConverter( + TypeInfoFactory.dateTypeInfo, primitiveType, Binary.fromString(value)); + assertNull(dateWritable); + } + @Test public void testGetIntConverterForTinyInt() throws Exception { PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.INT32) diff --git a/ql/src/test/queries/clientpositive/parquet_string_to_date_mapjoin.q b/ql/src/test/queries/clientpositive/parquet_string_to_date_mapjoin.q new file mode 100644 index 000000000000..6a6c8fe3f321 --- /dev/null +++ b/ql/src/test/queries/clientpositive/parquet_string_to_date_mapjoin.q @@ -0,0 +1,25 @@ +CREATE TABLE repro_parquet_string ( + id INT, + col1 STRING +) STORED AS PARQUET; + +INSERT INTO repro_parquet_string VALUES (1, '2026-01-01'); + +CREATE EXTERNAL TABLE repro_parquet_date ( + id INT, + col1 DATE +) STORED AS PARQUET +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/repro_parquet_string'; + +CREATE TABLE small_table ( + id INT, + date_col DATE +); +INSERT INTO small_table VALUES (1, '2026-01-01'); + +SET hive.auto.convert.join=true; +SET hive.vectorized.execution.enabled=false; + +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id; diff --git a/ql/src/test/results/clientpositive/llap/parquet_string_to_date_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/parquet_string_to_date_mapjoin.q.out new file mode 100644 index 000000000000..5e260d1f7f86 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/parquet_string_to_date_mapjoin.q.out @@ -0,0 +1,81 @@ +PREHOOK: query: CREATE TABLE repro_parquet_string ( + id INT, + col1 STRING +) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@repro_parquet_string +POSTHOOK: query: CREATE TABLE repro_parquet_string ( + id INT, + col1 STRING +) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repro_parquet_string +PREHOOK: query: INSERT INTO repro_parquet_string VALUES (1, '2026-01-01') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@repro_parquet_string +POSTHOOK: query: INSERT INTO repro_parquet_string VALUES (1, '2026-01-01') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@repro_parquet_string +POSTHOOK: Lineage: repro_parquet_string.col1 SCRIPT [] +POSTHOOK: Lineage: repro_parquet_string.id SCRIPT [] +PREHOOK: query: CREATE EXTERNAL TABLE repro_parquet_date ( + id INT, + col1 DATE +) STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@repro_parquet_date +POSTHOOK: query: CREATE EXTERNAL TABLE repro_parquet_date ( + id INT, + col1 DATE +) STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repro_parquet_date +PREHOOK: query: CREATE TABLE small_table ( + id INT, + date_col DATE +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@small_table +POSTHOOK: query: CREATE TABLE small_table ( + id INT, + date_col DATE +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_table +PREHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@small_table +POSTHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@small_table +POSTHOOK: Lineage: small_table.date_col SCRIPT [] +POSTHOOK: Lineage: small_table.id SCRIPT [] +PREHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@repro_parquet_date +PREHOOK: Input: default@small_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@repro_parquet_date +POSTHOOK: Input: default@small_table +#### A masked pattern was here #### +2026-01-01 From dea8c33729283c201d73ab41d4d5f277c4750155 Mon Sep 17 00:00:00 2001 From: Raghav Aggarwal Date: Wed, 8 Jul 2026 15:42:52 +0530 Subject: [PATCH 2/2] Add support for Parquet Sting-to-Date in vectorization flow --- .../ParquetDataColumnReaderFactory.java | 138 ++++++++- ...=> parquet_string_to_date_vectorization.q} | 11 + .../llap/parquet_string_to_date_mapjoin.q.out | 81 ------ ...parquet_string_to_date_vectorization.q.out | 266 ++++++++++++++++++ 4 files changed, 402 insertions(+), 94 deletions(-) rename ql/src/test/queries/clientpositive/{parquet_string_to_date_mapjoin.q => parquet_string_to_date_vectorization.q} (67%) delete mode 100644 ql/src/test/results/clientpositive/llap/parquet_string_to_date_mapjoin.q.out create mode 100644 ql/src/test/results/clientpositive/llap/parquet_string_to_date_vectorization.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java index 472deeca41f8..c272d78d1a8a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.io.parquet.vector; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.HiveBaseChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.Timestamp; @@ -1934,6 +1935,66 @@ private byte[] truncateIfNecesssary(byte[] bytes) { } } + /** + * Reads Parquet string bytes for Date column ("2024-07-09") and converts them to long (days since + * epoch) + */ + public static class TypesFromStringToDatePageReader extends TypesFromStringPageReader { + /** + * A cache to pre-calculate string-to-date conversions for dictionary encoded pages. + * + *

'parquet.dictionary.page.size' defaults to 1MB. If a column has too many unique values, + * Parquet automatically disables the dictionary (isDict = false) and falls back to the + * ValuesReader flow. Therefore, a 1MB dictionary holds at most ~100,000 date strings (10 bytes + * each). This guarantees our long[] cache will never exceed ~800 KB, and the boolean[] ~100 KB! + */ + private long[] dictDateCache; + private boolean[] dictDateCacheValid; + + public TypesFromStringToDatePageReader(ValuesReader realReader, int length) { + super(realReader, length); + } + + public TypesFromStringToDatePageReader(Dictionary dict, int length) { + super(dict, length); + int maxId = dict.getMaxId(); + // IDs are zero-indexed + dictDateCache = new long[maxId + 1]; + dictDateCacheValid = new boolean[maxId + 1]; + for (int i = 0; i <= maxId; i++) { + dictDateCache[i] = parseDateString(dict.decodeToBinary(i).getBytesUnsafe()); + dictDateCacheValid[i] = isValid; + } + } + + @Override + public long readLong() { + return parseDateString(valuesReader.readBytes().getBytesUnsafe()); + } + + @Override + public long readLong(int id) { + isValid = dictDateCacheValid[id]; + return dictDateCache[id]; + } + + private long parseDateString(byte[] bytes) { + if (bytes == null) { + isValid = false; + return 0; + } + try { + String s = new String(bytes, StandardCharsets.UTF_8); + long epochDay = Date.valueOf(s).toEpochDay(); + isValid = true; + return epochDay; + } catch (Exception e) { + isValid = false; + return 0; + } + } + } + private static ParquetDataColumnReader getDataColumnReaderByTypeHelper(boolean isDictionary, PrimitiveType parquetType, TypeInfo hiveType, @@ -2021,13 +2082,26 @@ private static ParquetDataColumnReader getDataColumnReaderByTypeHelper(boolean i } } + /** + * Finds the right reader for Parquet string and binary data (BINARY and FIXED_LEN_BYTE_ARRAY). + * + *

This method handles cases where the Parquet data type doesn't perfectly match the + * Hive data type (like reading a Parquet String as a Hive Date). This is incredibly important + * during Vectorized MapJoins, because if we don't convert the data into the exact format Hive + * expects, it can crash the query with a ClassCastException. + * + * @param isDict True if the Parquet page uses dictionary encoding + * @param parquetType The primitive type from the Parquet schema + * @param hiveType The expected Hive TypeInfo + * @param valuesReader The fallback values reader if dictionary is not used + * @param dictionary The dictionary reader + * @return A configured ParquetDataColumnReader + */ private static ParquetDataColumnReader getConvertorFromBinary(boolean isDict, PrimitiveType parquetType, TypeInfo hiveType, ValuesReader valuesReader, Dictionary dictionary) { - LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); - // max length for varchar and char cases int length = getVarcharLength(hiveType); TypeInfo realHiveType = (hiveType instanceof ListTypeInfo) ? @@ -2042,12 +2116,20 @@ private static ParquetDataColumnReader getConvertorFromBinary(boolean isDict, int hiveScale = (typeName.equalsIgnoreCase(serdeConstants.DECIMAL_TYPE_NAME)) ? ((DecimalTypeInfo) realHiveType).getScale() : 0; + LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); + if (logicalType == null) { - return isDict ? new DefaultParquetDataColumnReader(dictionary, length) : new - DefaultParquetDataColumnReader(valuesReader, length); + // Check if we need to convert the string into a specific Hive type (like DATE) + ParquetDataColumnReader reader = + getReaderForString(realHiveType, isDict, dictionary, valuesReader, length); + return reader != null + ? reader + : (isDict + ? new DefaultParquetDataColumnReader(dictionary, length) + : new DefaultParquetDataColumnReader(valuesReader, length)); } - Optional reader = parquetType.getLogicalTypeAnnotation() + Optional reader = logicalType .accept(new LogicalTypeAnnotationVisitor<>() { @Override public Optional visit( DecimalLogicalTypeAnnotation logicalTypeAnnotation) { @@ -2062,18 +2144,48 @@ private static ParquetDataColumnReader getConvertorFromBinary(boolean isDict, @Override public Optional visit( StringLogicalTypeAnnotation logicalTypeAnnotation) { - return isDict ? Optional - .of(new TypesFromStringPageReader(dictionary, length)) : Optional - .of(new TypesFromStringPageReader(valuesReader, length)); + // Only try to find a specialized reader (like for DATE) right when we actually need it. + // If we don't find one, we just fall back to reading it as a normal string. + ParquetDataColumnReader reader = getReaderForString( + realHiveType, isDict, dictionary, valuesReader, length); + return Optional.of(reader != null ? reader : + (isDict + ? new TypesFromStringPageReader(dictionary, length) + : new TypesFromStringPageReader(valuesReader, length))); } }); - if (reader.isPresent()) { - return reader.get(); - } + // If the visitor above didn't find a reader (e.g. for unsupported logical types), + // default to a standard binary reader. + return reader.orElseGet( + () -> + isDict + ? new DefaultParquetDataColumnReader(dictionary, length) + : new DefaultParquetDataColumnReader(valuesReader, length)); + } - return isDict ? new DefaultParquetDataColumnReader(dictionary, length) : new - DefaultParquetDataColumnReader(valuesReader, length); + /** + * Returns a specialized reader if we need to convert Parquet string bytes into a specific Hive + * data type (like DATE). + * + *

If the type doesn't need special string conversion, it simply returns null. This makes it + * easy to add support for other data types in the future if they also need to be converted from + * strings during vectorized reads. + */ + private static ParquetDataColumnReader getReaderForString(TypeInfo hiveType, + boolean isDict, Dictionary dictionary, ValuesReader valuesReader, int length) { + if (!(hiveType instanceof PrimitiveTypeInfo)) { + return null; + } + return switch (((PrimitiveTypeInfo) hiveType).getPrimitiveCategory()) { + case DATE -> isDict + ? new TypesFromStringToDatePageReader(dictionary, length) + : new TypesFromStringToDatePageReader(valuesReader, length); + case STRING, VARCHAR, CHAR -> isDict + ? new TypesFromStringPageReader(dictionary, length) + : new TypesFromStringPageReader(valuesReader, length); + default -> null; + }; } public static ParquetDataColumnReader getDataColumnReaderByTypeOnDictionary( diff --git a/ql/src/test/queries/clientpositive/parquet_string_to_date_mapjoin.q b/ql/src/test/queries/clientpositive/parquet_string_to_date_vectorization.q similarity index 67% rename from ql/src/test/queries/clientpositive/parquet_string_to_date_mapjoin.q rename to ql/src/test/queries/clientpositive/parquet_string_to_date_vectorization.q index 6a6c8fe3f321..500ff10dddc1 100644 --- a/ql/src/test/queries/clientpositive/parquet_string_to_date_mapjoin.q +++ b/ql/src/test/queries/clientpositive/parquet_string_to_date_vectorization.q @@ -20,6 +20,17 @@ INSERT INTO small_table VALUES (1, '2026-01-01'); SET hive.auto.convert.join=true; SET hive.vectorized.execution.enabled=false; +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id; + +SET hive.vectorized.execution.enabled=true; + +EXPLAIN VECTORIZATION DETAIL SELECT /*+ MAPJOIN(small_table) */ a.col1 FROM repro_parquet_date a JOIN small_table b ON a.id = b.id; + +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id; diff --git a/ql/src/test/results/clientpositive/llap/parquet_string_to_date_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/parquet_string_to_date_mapjoin.q.out deleted file mode 100644 index 5e260d1f7f86..000000000000 --- a/ql/src/test/results/clientpositive/llap/parquet_string_to_date_mapjoin.q.out +++ /dev/null @@ -1,81 +0,0 @@ -PREHOOK: query: CREATE TABLE repro_parquet_string ( - id INT, - col1 STRING -) STORED AS PARQUET -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@repro_parquet_string -POSTHOOK: query: CREATE TABLE repro_parquet_string ( - id INT, - col1 STRING -) STORED AS PARQUET -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@repro_parquet_string -PREHOOK: query: INSERT INTO repro_parquet_string VALUES (1, '2026-01-01') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@repro_parquet_string -POSTHOOK: query: INSERT INTO repro_parquet_string VALUES (1, '2026-01-01') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@repro_parquet_string -POSTHOOK: Lineage: repro_parquet_string.col1 SCRIPT [] -POSTHOOK: Lineage: repro_parquet_string.id SCRIPT [] -PREHOOK: query: CREATE EXTERNAL TABLE repro_parquet_date ( - id INT, - col1 DATE -) STORED AS PARQUET -#### A masked pattern was here #### -PREHOOK: type: CREATETABLE -#### A masked pattern was here #### -PREHOOK: Output: database:default -PREHOOK: Output: default@repro_parquet_date -POSTHOOK: query: CREATE EXTERNAL TABLE repro_parquet_date ( - id INT, - col1 DATE -) STORED AS PARQUET -#### A masked pattern was here #### -POSTHOOK: type: CREATETABLE -#### A masked pattern was here #### -POSTHOOK: Output: database:default -POSTHOOK: Output: default@repro_parquet_date -PREHOOK: query: CREATE TABLE small_table ( - id INT, - date_col DATE -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@small_table -POSTHOOK: query: CREATE TABLE small_table ( - id INT, - date_col DATE -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@small_table -PREHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@small_table -POSTHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@small_table -POSTHOOK: Lineage: small_table.date_col SCRIPT [] -POSTHOOK: Lineage: small_table.id SCRIPT [] -PREHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 -FROM repro_parquet_date a -JOIN small_table b ON a.id = b.id -PREHOOK: type: QUERY -PREHOOK: Input: default@repro_parquet_date -PREHOOK: Input: default@small_table -#### A masked pattern was here #### -POSTHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 -FROM repro_parquet_date a -JOIN small_table b ON a.id = b.id -POSTHOOK: type: QUERY -POSTHOOK: Input: default@repro_parquet_date -POSTHOOK: Input: default@small_table -#### A masked pattern was here #### -2026-01-01 diff --git a/ql/src/test/results/clientpositive/llap/parquet_string_to_date_vectorization.q.out b/ql/src/test/results/clientpositive/llap/parquet_string_to_date_vectorization.q.out new file mode 100644 index 000000000000..5515bc90f3e9 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/parquet_string_to_date_vectorization.q.out @@ -0,0 +1,266 @@ +PREHOOK: query: CREATE TABLE repro_parquet_string ( + id INT, + col1 STRING +) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@repro_parquet_string +POSTHOOK: query: CREATE TABLE repro_parquet_string ( + id INT, + col1 STRING +) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repro_parquet_string +PREHOOK: query: INSERT INTO repro_parquet_string VALUES (1, '2026-01-01') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@repro_parquet_string +POSTHOOK: query: INSERT INTO repro_parquet_string VALUES (1, '2026-01-01') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@repro_parquet_string +POSTHOOK: Lineage: repro_parquet_string.col1 SCRIPT [] +POSTHOOK: Lineage: repro_parquet_string.id SCRIPT [] +PREHOOK: query: CREATE EXTERNAL TABLE repro_parquet_date ( + id INT, + col1 DATE +) STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@repro_parquet_date +POSTHOOK: query: CREATE EXTERNAL TABLE repro_parquet_date ( + id INT, + col1 DATE +) STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repro_parquet_date +PREHOOK: query: CREATE TABLE small_table ( + id INT, + date_col DATE +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@small_table +POSTHOOK: query: CREATE TABLE small_table ( + id INT, + date_col DATE +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@small_table +PREHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@small_table +POSTHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@small_table +POSTHOOK: Lineage: small_table.date_col SCRIPT [] +POSTHOOK: Lineage: small_table.id SCRIPT [] +PREHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@repro_parquet_date +PREHOOK: Input: default@small_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@repro_parquet_date +POSTHOOK: Input: default@small_table +#### A masked pattern was here #### +2026-01-01 +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@repro_parquet_date +PREHOOK: Input: default@small_table +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@repro_parquet_date +POSTHOOK: Input: default@small_table +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: id is not null (type: boolean) + Statistics: Num rows: 25 Data size: 1260 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:id:int, 1:col1:date, 2:ROW__ID:struct, 3:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:int) + predicate: id is not null (type: boolean) + Statistics: Num rows: 20 Data size: 1008 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: int), col1 (type: date) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 20 Data size: 1008 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Map Join Vectorization: + bigTableKeyColumns: 0:int + bigTableRetainColumnNums: [1] + bigTableValueColumns: 1:date + className: VectorMapJoinInnerBigOnlyLongOperator + native: true + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true + nonOuterSmallTableKeyMapping: [] + projectedOutput: 1:date + hashTableImplementationType: OPTIMIZED + outputColumnNames: _col1 + input vertices: + 1 Map 2 + Statistics: Num rows: 22 Data size: 1108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: date) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1] + Statistics: Num rows: 22 Data size: 1108 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 22 Data size: 1108 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + includeColumns: [0, 1] + dataColumns: id:int, col1:date + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Map 2 + Map Operator Tree: + TableScan + alias: b + filterExpr: id is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:id:int, 1:date_col:date, 2:ROW__ID:struct, 3:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:int) + predicate: id is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 0:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + includeColumns: [0] + dataColumns: id:int, date_col:date + partitionColumnCount: 0 + scratchColumnTypeNames: [] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@repro_parquet_date +PREHOOK: Input: default@small_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1 +FROM repro_parquet_date a +JOIN small_table b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@repro_parquet_date +POSTHOOK: Input: default@small_table +#### A masked pattern was here #### +2026-01-01