From 403da8195e41a0a907652881b913f3645f32d50d Mon Sep 17 00:00:00 2001 From: Stamatis Zampetakis Date: Thu, 2 Jul 2026 19:34:48 +0200 Subject: [PATCH] HIVE-29694: Reuse DB column descriptors when adding partitions in HMS --- .../partition_descriptor_reuse.q | 16 ++ .../llap/partition_descriptor_reuse.q.out | 70 ++++++++ .../metastore/ColumnDescriptorSupplier.java | 38 +++++ .../hadoop/hive/metastore/ObjectStore.java | 99 +++++++++-- .../directsql/MetaStoreDirectSql.java | 59 +++++++ .../metastore/impl/TableStoreImpl.java | 5 +- .../TestHMSColumnDescriptorReuse.java | 160 ++++++++++++++++++ 7 files changed, 427 insertions(+), 20 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/partition_descriptor_reuse.q create mode 100644 ql/src/test/results/clientpositive/llap/partition_descriptor_reuse.q.out create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ColumnDescriptorSupplier.java create mode 100644 standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java diff --git a/ql/src/test/queries/clientpositive/partition_descriptor_reuse.q b/ql/src/test/queries/clientpositive/partition_descriptor_reuse.q new file mode 100644 index 000000000000..a3302747e99a --- /dev/null +++ b/ql/src/test/queries/clientpositive/partition_descriptor_reuse.q @@ -0,0 +1,16 @@ +-- Debug instructions: +-- 1. Run the test in debug mode: mvn test -Dtest=TestMiniLlapLocalCliDriver -Dqfile=partition_descriptor_reuse.q -Dtest.output.overwrite -Dtest.metastore.db=postgres -Dmaven.surefire.debug +-- 2. Put a breakpoint in CliAdapter.java:120 CliAdapter.this.tearDown() to prevent DB from being dropped. +-- 3. Connect to the dockerized database and check the state of the metadata +-- docker exec -it CONTAINER_NAME psql -U test -d hivedb +-- TODO: Probably not gonna need this file at the end so drop it +create table person(id string, fname string) partitioned by (birthyear string); +alter table person add partition (birthyear='1987'); +alter table person add partition (birthyear='1988'); +alter table person add partition (birthyear='1989'); +alter table person add partition (birthyear='1990'); +alter table person add partition (birthyear='1991'); + +alter table person partition (birthyear='1989') add columns (lname string); +alter table person partition (birthyear='1990') add columns (lname string); +alter table person partition (birthyear='1991') change column fname fullname string; diff --git a/ql/src/test/results/clientpositive/llap/partition_descriptor_reuse.q.out b/ql/src/test/results/clientpositive/llap/partition_descriptor_reuse.q.out new file mode 100644 index 000000000000..f6c6a3d71b08 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/partition_descriptor_reuse.q.out @@ -0,0 +1,70 @@ +PREHOOK: query: create table person(id string, fname string) partitioned by (birthyear string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@person +POSTHOOK: query: create table person(id string, fname string) partitioned by (birthyear string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@person +PREHOOK: query: alter table person add partition (birthyear='1987') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@person +POSTHOOK: query: alter table person add partition (birthyear='1987') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@person +POSTHOOK: Output: default@person@birthyear=1987 +PREHOOK: query: alter table person add partition (birthyear='1988') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@person +POSTHOOK: query: alter table person add partition (birthyear='1988') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@person +POSTHOOK: Output: default@person@birthyear=1988 +PREHOOK: query: alter table person add partition (birthyear='1989') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@person +POSTHOOK: query: alter table person add partition (birthyear='1989') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@person +POSTHOOK: Output: default@person@birthyear=1989 +PREHOOK: query: alter table person add partition (birthyear='1990') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@person +POSTHOOK: query: alter table person add partition (birthyear='1990') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@person +POSTHOOK: Output: default@person@birthyear=1990 +PREHOOK: query: alter table person add partition (birthyear='1991') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@person +POSTHOOK: query: alter table person add partition (birthyear='1991') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@person +POSTHOOK: Output: default@person@birthyear=1991 +PREHOOK: query: alter table person partition (birthyear='1989') add columns (lname string) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@person +PREHOOK: Output: default@person@birthyear=1989 +POSTHOOK: query: alter table person partition (birthyear='1989') add columns (lname string) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@person +POSTHOOK: Input: default@person@birthyear=1989 +POSTHOOK: Output: default@person@birthyear=1989 +PREHOOK: query: alter table person partition (birthyear='1990') add columns (lname string) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@person +PREHOOK: Output: default@person@birthyear=1990 +POSTHOOK: query: alter table person partition (birthyear='1990') add columns (lname string) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@person +POSTHOOK: Input: default@person@birthyear=1990 +POSTHOOK: Output: default@person@birthyear=1990 +PREHOOK: query: alter table person partition (birthyear='1991') change column fname fullname string +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@person +PREHOOK: Output: default@person@birthyear=1991 +POSTHOOK: query: alter table person partition (birthyear='1991') change column fname fullname string +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@person +POSTHOOK: Input: default@person@birthyear=1991 +POSTHOOK: Output: default@person@birthyear=1991 diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ColumnDescriptorSupplier.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ColumnDescriptorSupplier.java new file mode 100644 index 000000000000..e73ae0ca40e5 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ColumnDescriptorSupplier.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; +import org.apache.hadoop.hive.metastore.model.MTable; + +import java.util.List; + +public interface ColumnDescriptorSupplier { + /** + * Gets a column descriptor for the provided table and list of columns. If no matching column descriptor exists, + * returns null. The column descriptor is considered a match if it has the same number of columns, and each column has + * the same name, type, and comment in the same order. + * + * @param cols the columns to match + * @param mt the table to search + * @return an existing MColumnDescriptor if found, null otherwise + */ + MColumnDescriptor getColumnDescriptor(List cols, MTable mt) throws MetaException; +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 17ad48698ffb..9f4ead621e24 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -258,7 +258,7 @@ * to be made into a interface that can read both from a database and a * filestore. */ -public class ObjectStore implements RawStore, Configurable { +public class ObjectStore implements RawStore, ColumnDescriptorSupplier, Configurable { protected int batchSize = NO_BATCHING; private static final DateTimeFormatter YMDHMS_FORMAT = DateTimeFormatter.ofPattern( @@ -1821,11 +1821,11 @@ private static MSerDeInfo convertToMSerDeInfo(SerDeInfo ms) throws MetaException * @param cols the columns the column descriptor contains * @return a new column descriptor db-backed object */ - private static MColumnDescriptor createNewMColumnDescriptor(List cols) { + private static MColumnDescriptor createNewMColumnDescriptor(List cols) { if (cols == null) { return null; } - return new MColumnDescriptor(cols); + return new MColumnDescriptor(convertToMFieldSchemas(cols)); } private static StorageDescriptor convertToStorageDescriptor( @@ -1929,7 +1929,28 @@ private static MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor if (sd == null) { return null; } - MColumnDescriptor mcd = createNewMColumnDescriptor(convertToMFieldSchemas(sd.getCols())); + MColumnDescriptor mcd = createNewMColumnDescriptor(sd.getCols()); + return convertToMStorageDescriptor(sd, mcd); + } + + /** + * Converts a storage descriptor to a db-backed storage descriptor. Creates a + * new db-backed column descriptor object for this SD, unless a matching one already + * exists for the given table. + * @param sd the storage descriptor to wrap in a db-backed object + * @param mt the table to search for existing column descriptors, may be null + * @return the storage descriptor db-backed object + */ + private static MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd, MTable mt, ColumnDescriptorSupplier cds) + throws MetaException { + if (sd == null) { + return null; + } + + MColumnDescriptor mcd = (mt != null) ? cds.getColumnDescriptor(sd.getCols(), mt) : null; + if (mcd == null) { + mcd = createNewMColumnDescriptor(sd.getCols()); + } return convertToMStorageDescriptor(sd, mcd); } @@ -1959,6 +1980,55 @@ private static MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories()); } + @Override + public MColumnDescriptor getColumnDescriptor(List cols, MTable mt) + throws MetaException { + if (cols == null || cols.isEmpty()) { + return null; + } + + // First check to see if partition and tables column descriptor match + // that's the easy and relatively fast-check since does not require + // round-tripe to the database + List tableSchema = mt.getSd() != null && mt.getSd().getCD() != null && mt.getSd() + .getCD() + .getCols() != null ? convertToFieldSchemas(mt.getSd() + .getCD() + .getCols()) : null; + if (cols.equals(tableSchema)) { + return mt.getSd().getCD(); + } + + String catName = mt.getDatabase().getCatalogName(); + String dbName = mt.getDatabase().getName(); + String tblName = mt.getTableName(); + long tblId = mt.getId(); + try { + return new GetHelper(catName, dbName, tblName, true, true) { + @Override + protected String describeResult() { + return "Matching column descriptor"; + } + + @Override + protected MColumnDescriptor getSqlResult(GetHelper ctx) + throws MetaException { + // TODO: Need to make this configurable via property cause it may be too expensive for general use + return directSql.getColumnDescriptor(cols, tblId); + } + + @Override + protected MColumnDescriptor getJdoResult(GetHelper ctx) { + // Return null basically means allocate a new column descriptor + return null; + } + }.run(false); + } catch (NoSuchObjectException e) { + return null; + } + } + + public static MCreationMetadata convertToMCreationMetadata(CreationMetadata m, RawStore base) throws MetaException { if (m == null) { @@ -2066,7 +2136,7 @@ private static SourceTable convertToSourceTable(MMVSource mmvSource, String cata * @param mt the parent table object * @return the model partition object, and null if the input partition is null. */ - public static MPartition convertToMPart(Partition part, MTable mt) + public static MPartition convertToMPart(Partition part, MTable mt, RawStore base) throws InvalidObjectException, MetaException { // NOTE: we don't set writeId in this method. Write ID is only set after validating the // existing write ID against the caller's valid list. @@ -2078,20 +2148,13 @@ public static MPartition convertToMPart(Partition part, MTable mt) "Partition doesn't have a valid table or database name"); } - // If this partition's set of columns is the same as the parent table's, - // use the parent table's, so we do not create a duplicate column descriptor, - // thereby saving space - MStorageDescriptor msd; - if (mt.getSd() != null && mt.getSd().getCD() != null && - mt.getSd().getCD().getCols() != null && - part.getSd() != null && - convertToFieldSchemas(mt.getSd().getCD().getCols()). - equals(part.getSd().getCols())) { - msd = convertToMStorageDescriptor(part.getSd(), mt.getSd().getCD()); - } else { - msd = convertToMStorageDescriptor(part.getSd()); + ColumnDescriptorSupplier cds = (s, t) -> null; + if (base instanceof ColumnDescriptorSupplier) { + cds = (ColumnDescriptorSupplier) base; } + MStorageDescriptor msd = convertToMStorageDescriptor(part.getSd(), mt, cds); + return new MPartition(Warehouse.makePartName(convertToFieldSchemas(mt .getPartitionKeys()), part.getValues()), mt, part.getValues(), part .getCreateTime(), part.getLastAccessTime(), @@ -6239,7 +6302,7 @@ private MSchemaVersion convertToMSchemaVersion(SchemaVersion schemaVersion) thro normalizeIdentifier(schemaVersion.getSchema().getSchemaName())), schemaVersion.getVersion(), schemaVersion.getCreatedAt(), - createNewMColumnDescriptor(convertToMFieldSchemas(schemaVersion.getCols())), + createNewMColumnDescriptor(schemaVersion.getCols()), schemaVersion.isSetState() ? schemaVersion.getState().getValue() : 0, schemaVersion.isSetDescription() ? schemaVersion.getDescription() : null, schemaVersion.isSetSchemaText() ? schemaVersion.getSchemaText() : null, diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/MetaStoreDirectSql.java index 713f3ff4e338..2ae7f3ed1dbc 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/MetaStoreDirectSql.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/MetaStoreDirectSql.java @@ -107,6 +107,7 @@ import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; import org.apache.hadoop.hive.metastore.model.MConstraint; import org.apache.hadoop.hive.metastore.model.MCreationMetadata; import org.apache.hadoop.hive.metastore.model.MDatabase; @@ -534,6 +535,64 @@ public void addPartitions(List parts, List directSqlInsertPart.addPartitions(parts, partPrivilegesList, partColPrivilegesList); } + public MColumnDescriptor getColumnDescriptor(List cols, long tblId) + throws MetaException { + if (cols == null || cols.isEmpty()) { + return null; + } + Query query = null; + try { + // TODO: Is there a more efficient query? Should we add a limit? + String findDesciptorsSql = "SELECT DISTINCT \"SDS\".\"CD_ID\" FROM \"SDS\" " + + "INNER JOIN \"COLUMNS_V2\" ON \"SDS\".\"CD_ID\" = \"COLUMNS_V2\".\"CD_ID\" " + + "WHERE \"SDS\".\"SD_ID\" IN (" + + "SELECT \"SD_ID\" FROM \"PARTITIONS\" WHERE \"TBL_ID\" = ?" + + ") GROUP BY \"SDS\".\"CD_ID\", \"SDS\".\"SD_ID\" HAVING COUNT(*) = ?"; + query = pm.newQuery("javax.jdo.query.SQL", findDesciptorsSql); + List candidateIds = executeWithArray(query, new Object[]{tblId, cols.size()}, findDesciptorsSql); + if (candidateIds == null || candidateIds.isEmpty()) { + return null; + } + + for (Long cdId : candidateIds) { + if (descriptorHasColumns(cdId, cols)) { + return pm.getObjectById(MColumnDescriptor.class, cdId); + } + } + } finally { + if (query != null) { + query.closeAll(); + } + } + return null; + } + + private boolean descriptorHasColumns(Long cdId, List cols) throws MetaException { + String findColumnSql = "SELECT \"COLUMN_NAME\", \"TYPE_NAME\", \"COMMENT\" FROM \"COLUMNS_V2\" " + + "WHERE \"CD_ID\" = ? ORDER BY \"INTEGER_IDX\""; + Query query = null; + try { + query = pm.newQuery("javax.jdo.query.SQL", findColumnSql); + List rows = executeWithArray(query, new Object[] {cdId}, findColumnSql); + if (rows != null && rows.size() == cols.size()) { + boolean match = true; + for (int i = 0; i < cols.size(); i++) { + Object[] row = rows.get(i); + FieldSchema col = new FieldSchema(String.valueOf(row[0]), String.valueOf(row[1]), String.valueOf(row[2])); + if (!cols.get(i).equals(col)) { + match = false; + break; + } + } + return match; + } + return false; + } finally { + if (query != null) { + query.closeAll(); + } + } + } /** * Alter partitions in batch using direct SQL * @param table the target table diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java index c03e1b8e0016..c3c25af84250 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java @@ -721,7 +721,7 @@ public boolean addPartitions(TableName tableName, List parts) throws throw new MetaException("Partition does not belong to target table " + dbName + "." + tblName + ": " + part); } - MPartition mpart = convertToMPart(part, table); + MPartition mpart = convertToMPart(part, table, baseStore); mParts.add(mpart); int now = (int) (System.currentTimeMillis() / 1000); List mPartPrivileges = new ArrayList<>(); @@ -1901,7 +1901,8 @@ private Partition alterPartitionNoTxn(String catName, String dbname, catName = normalizeIdentifier(catName); name = normalizeIdentifier(name); dbname = normalizeIdentifier(dbname); - MPartition newp = convertToMPart(newPart, table); + // TODO: Should we perform descriptor reuse in this path? + MPartition newp = convertToMPart(newPart, table, baseStore); MColumnDescriptor oldCD = null; MStorageDescriptor oldSD = oldp.getSd(); if (oldSD != null) { diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java new file mode 100644 index 000000000000..5d1365f9985e --- /dev/null +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidInputException; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.dbinstall.rules.DatabaseRule; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Postgres; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; +import static org.junit.Assert.assertEquals; + +@Category(MetastoreUnitTest.class) +public class TestHMSColumnDescriptorReuse { + private ObjectStore objectStore = null; + // Use Postgres instead of Derby for facilitate debugging and looking into the database + // In the final version we should rather use Derby + // TODO: Beore merging we should test with all supported DBMS + private static final DatabaseRule DB = new Postgres(); + + @Before + public void setUp() throws Exception { + DB.before(); + DB.install(); + Configuration conf = MetastoreConf.newMetastoreConf(); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY, DB.getJdbcUrl()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECTION_DRIVER, DB.getJdbcDriver()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECTION_USER_NAME, DB.getHiveUser()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.PWD, DB.getHivePassword()); + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.AUTO_CREATE_ALL, false); + + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true); + + MetaStoreTestUtils.setConfForStandloneMode(conf); + + objectStore = new ObjectStore(); + objectStore.setConf(conf); + HMSHandler.createDefaultCatalog(objectStore, new Warehouse(conf)); + Database db = new DatabaseBuilder() + .setName("default") + .setDescription("description") + .setLocation("locationurl") + .build(conf); + objectStore.createDatabase(db); + } + + @After + public void tearDown() throws Exception { + DB.after(); + } + + @Test + public void testAddPartitionAlterAddPartition() throws MetaException, InvalidObjectException, InvalidInputException, NoSuchObjectException { + FieldSchema id = new FieldSchema("id", ColumnType.STRING_TYPE_NAME, ""); + FieldSchema fname = new FieldSchema("fname", ColumnType.STRING_TYPE_NAME, ""); + FieldSchema country = new FieldSchema("country", ColumnType.STRING_TYPE_NAME, ""); + + Table tbl1 = newTable("person", Arrays.asList(id, fname), Collections.singletonList(country)); + objectStore.createTable(tbl1); + objectStore.addPartition(newPart(tbl1, "US")); + objectStore.addPartition(newPart(tbl1, "Greece")); + FieldSchema lname = new FieldSchema("lname", ColumnType.STRING_TYPE_NAME, ""); + Table tbl2 = newTable("person", Arrays.asList(id, fname, lname), Collections.singletonList(country)); + objectStore.alterTable(DEFAULT_CATALOG_NAME, tbl1.getDbName(), tbl1.getTableName(), tbl2, null); + objectStore.addPartition(newPart(tbl2, "Italy")); + // Mimics replication scenario where we are adding partitions to the "same" table but with a different schema. + // The tbl1 is using the old storage descriptor so "Germany" and "Belgium" partitions will have the old schema + // And will lead to "duplicate" entries in "CDS" and "COLUMNS_V2" tables. + objectStore.addPartition(newPart(tbl1, "Germany")); + objectStore.addPartition(newPart(tbl1, "Belgium")); + // On the other hand the addition of a partition to the table with the new schema is successfully using the + // existing storage/column descriptors + objectStore.addPartition(newPart(tbl2, "England")); + // This assertion could be more elaborate and not just a simple count + assertEquals(2, countColumnDescriptors()); + } + + private int countColumnDescriptors() { + try(Connection c = DriverManager.getConnection(DB.getJdbcUrl(), DB.getHiveUser(), DB.getHivePassword())){ + try(ResultSet rs = c.prepareStatement("SELECT COUNT(*) FROM \"CDS\"").executeQuery()) { + rs.next(); + return rs.getInt(1); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + private static Table newTable(String name, List columns, List partCols ) { + int timeSec = (int) System.currentTimeMillis() / 1000; + StorageDescriptor sd = new StorageDescriptor(columns, + "/fake/location/person", + "org.apache.hadoop.mapred.TextInputFormat", + "org.apache.hadoop.mapred.MapFileOutputFormat", + false, + 0, + new SerDeInfo("SerDeName", "serializationLib", null), + null, + null, + null); + HashMap tableParams = new HashMap<>(); + tableParams.put("EXTERNAL", "false"); + return + new Table(name, "default", "owner", timeSec, timeSec, 3, sd, partCols, + tableParams, null, null, "MANAGED_TABLE"); + } + + private static Partition newPart(Table tbl, String value) { + int timeSec = (int) System.currentTimeMillis() / 1000; + HashMap partitionParams = new HashMap<>(); + partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true"); + StorageDescriptor psd = tbl.getSd().deepCopy(); + psd.setLocation(psd.getLocation() + "/" + value); + Partition p = new Partition(Collections.singletonList(value), tbl.getDbName(), tbl.getTableName(), timeSec, timeSec, psd, partitionParams); + p.setCatName(DEFAULT_CATALOG_NAME); + return p; + } + +} +