Skip to content

HIVE-26441: Expand GenericJdbcDatabaseAccessor unit test coverage #6587

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hemanthumashankar0511 wants to merge 1 commit into
base: master
Choose a base branch
from
+273 −0
Open

HIVE-26441: Expand GenericJdbcDatabaseAccessor unit test coverage #6587

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
266 changes: 266 additions & 0 deletions ...ndler/src/test/java/org/apache/hive/storage/jdbc/dao/TestGenericJdbcDatabaseAccessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,25 +14,36 @@
*/
package org.apache.hive.storage.jdbc.dao;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hive.storage.jdbc.DBRecordWritable;
import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig;
import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException;

import org.junit.Assert;
import org.junit.Test;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.equalToIgnoringCase;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.notNullValue;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class TestGenericJdbcDatabaseAccessor {

Expand Down Expand Up @@ -212,6 +223,29 @@ public void testGetRecordIterator_offsets() throws HiveJdbcDatabaseAccessExcepti
}


@Test
public void testGetRecordIterator_limitOffsetLastPartition() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
JdbcRecordIterator iterator = accessor.getRecordIterator(conf, null, null, null, 2, 4);

assertThat(iterator, is(notNullValue()));

int count = 0;
while (iterator.hasNext()) {
Map<String, Object> record = iterator.next();
count++;

assertThat(record, is(notNullValue()));
assertThat(record.size(), is(equalTo(7)));
assertThat(record.get("strategy_id"), is(equalTo(5)));
}

assertThat(count, is(equalTo(1)));
iterator.close();
}


@Test
public void testGetRecordIterator_emptyResultSet() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
Expand All @@ -225,6 +259,91 @@ public void testGetRecordIterator_emptyResultSet() throws HiveJdbcDatabaseAccess
}


@Test
public void testGetRecordIterator_partitionFirst() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
JdbcRecordIterator iterator =
accessor.getRecordIterator(conf, getStrategyIdColumnName(conf, accessor), null, "3", -1, 0);

assertThat(iterator, is(notNullValue()));

int count = 0;
while (iterator.hasNext()) {
Map<String, Object> record = iterator.next();
count++;

assertThat(record, is(notNullValue()));
assertThat(record.size(), is(equalTo(7)));
assertThat(record.get("strategy_id"), is(equalTo(count)));
}

assertThat(count, is(equalTo(2)));
iterator.close();
}


@Test
public void testGetRecordIterator_partitionMiddle() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
JdbcRecordIterator iterator =
accessor.getRecordIterator(conf, getStrategyIdColumnName(conf, accessor), "3", "5", -1, 0);

assertThat(iterator, is(notNullValue()));

int count = 0;
while (iterator.hasNext()) {
Map<String, Object> record = iterator.next();
count++;

assertThat(record, is(notNullValue()));
assertThat(record.size(), is(equalTo(7)));
assertThat(record.get("strategy_id"), is(equalTo(count + 2)));
}

assertThat(count, is(equalTo(2)));
iterator.close();
}


@Test
public void testGetRecordIterator_partitionLast() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
JdbcRecordIterator iterator =
accessor.getRecordIterator(conf, getStrategyIdColumnName(conf, accessor), "5", null, -1, 0);

assertThat(iterator, is(notNullValue()));

int count = 0;
while (iterator.hasNext()) {
Map<String, Object> record = iterator.next();
count++;

assertThat(record, is(notNullValue()));
assertThat(record.size(), is(equalTo(7)));
assertThat(record.get("strategy_id"), is(equalTo(5)));
}

assertThat(count, is(equalTo(1)));
iterator.close();
}


@Test
public void testGetRecordIterator_partitionEmptyResultSet() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
JdbcRecordIterator iterator =
accessor.getRecordIterator(conf, getStrategyIdColumnName(conf, accessor), "6", "7", -1, 0);

assertThat(iterator, is(notNullValue()));
assertThat(iterator.hasNext(), is(false));
iterator.close();
}


@Test
public void testGetRecordIterator_largeOffset() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
Expand All @@ -246,6 +365,153 @@ public void testGetRecordIterator_invalidQuery() throws HiveJdbcDatabaseAccessEx
JdbcRecordIterator iterator = accessor.getRecordIterator(conf, null, null, null, 0, 2);
}

@Test
public void testGetBounds_minAndMax() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
Pair<String, String> bounds = accessor.getBounds(conf, getStrategyIdColumnName(conf, accessor), true, true);

assertThat(bounds.getLeft(), is(equalTo("1")));
assertThat(bounds.getRight(), is(equalTo("5")));
}

@Test
public void testGetBounds_minOnly() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
Pair<String, String> bounds = accessor.getBounds(conf, getStrategyIdColumnName(conf, accessor), true, false);

assertThat(bounds.getLeft(), is(equalTo("1")));
assertThat(bounds.getRight(), is(nullValue()));
}

@Test
public void testGetBounds_maxOnly() throws HiveJdbcDatabaseAccessException {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
Pair<String, String> bounds = accessor.getBounds(conf, getStrategyIdColumnName(conf, accessor), false, true);

assertThat(bounds.getLeft(), is(nullValue()));
assertThat(bounds.getRight(), is(equalTo("5")));
}

@Test
public void testNeedColumnQuote_genericAccessor() {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);

assertThat(accessor.needColumnQuote(), is(true));
}

@Test
public void testNeedColumnQuote_mysqlAccessor() {
Configuration conf = buildConfiguration();
conf.set(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), "MYSQL");
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);

assertThat(accessor.needColumnQuote(), is(false));
}

@Test
public void testClose_afterInitializedAccessor() throws Exception {
Configuration conf = buildConfiguration();
DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);

assertThat(accessor.getColumnNames(conf), is(notNullValue()));
accessor.close();
accessor.close();
}

@Test
public void testGetRecordWriter_writesRows() throws Exception {
Configuration conf = buildConfiguration();
conf.set(JdbcStorageConfig.JDBC_URL.getPropertyName(),
conf.get(JdbcStorageConfig.JDBC_URL.getPropertyName())
.replace("jdbc:h2:mem:test;MODE=MySQL;INIT",
"jdbc:h2:mem:test_writer;MODE=MySQL;DB_CLOSE_DELAY=-1;INIT"));
conf.set(JdbcStorageConfig.TABLE.getPropertyName(), "test_writer");
conf.set(serdeConstants.LIST_COLUMNS, "id,name");
conf.set(serdeConstants.LIST_COLUMN_TYPES, "int,string");

TaskAttemptContext context = mock(TaskAttemptContext.class);
when(context.getConfiguration()).thenReturn(conf);

DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
@SuppressWarnings("rawtypes")
RecordWriter writer = accessor.getRecordWriter(context);
DBRecordWritable record = new DBRecordWritable(2);
record.set(0, 10);
record.set(1, "written");

writer.write(record, null);
writer.close(context);

String readBackUrl = conf.get(JdbcStorageConfig.JDBC_URL.getPropertyName()).replaceAll(";INIT=.*", "");
try (Connection conn = DriverManager.getConnection(readBackUrl);
Statement statement = conn.createStatement()) {
try (ResultSet rs = statement.executeQuery("select id, name from test_writer")) {
assertThat(rs.next(), is(true));
assertThat(rs.getInt("id"), is(equalTo(10)));
assertThat(rs.getString("name"), is(equalTo("written")));
assertThat(rs.next(), is(false));
}
statement.execute("SHUTDOWN");
}
}

@Test
public void testGetRecordWriter_writesMultipleRows() throws Exception {
Configuration conf = buildConfiguration();
conf.set(JdbcStorageConfig.JDBC_URL.getPropertyName(),
conf.get(JdbcStorageConfig.JDBC_URL.getPropertyName())
.replace("jdbc:h2:mem:test;MODE=MySQL;INIT",
"jdbc:h2:mem:test_writer_multi;MODE=MySQL;DB_CLOSE_DELAY=-1;INIT"));
conf.set(JdbcStorageConfig.TABLE.getPropertyName(), "test_writer");
conf.set(serdeConstants.LIST_COLUMNS, "id,name");
conf.set(serdeConstants.LIST_COLUMN_TYPES, "int,string");

TaskAttemptContext context = mock(TaskAttemptContext.class);
when(context.getConfiguration()).thenReturn(conf);

DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf);
@SuppressWarnings("rawtypes")
RecordWriter writer = accessor.getRecordWriter(context);

DBRecordWritable firstRecord = new DBRecordWritable(2);
firstRecord.set(0, 11);
firstRecord.set(1, "first");
writer.write(firstRecord, null);

DBRecordWritable secondRecord = new DBRecordWritable(2);
secondRecord.set(0, 12);
secondRecord.set(1, "second");
writer.write(secondRecord, null);

writer.close(context);

String readBackUrl = conf.get(JdbcStorageConfig.JDBC_URL.getPropertyName()).replaceAll(";INIT=.*", "");
try (Connection conn = DriverManager.getConnection(readBackUrl);
Statement statement = conn.createStatement()) {
try (ResultSet rs = statement.executeQuery("select id, name from test_writer order by id")) {
assertThat(rs.next(), is(true));
assertThat(rs.getInt("id"), is(equalTo(11)));
assertThat(rs.getString("name"), is(equalTo("first")));

assertThat(rs.next(), is(true));
assertThat(rs.getInt("id"), is(equalTo(12)));
assertThat(rs.getString("name"), is(equalTo("second")));

assertThat(rs.next(), is(false));
}
statement.execute("SHUTDOWN");
}
}

private String getStrategyIdColumnName(Configuration conf, DatabaseAccessor accessor)
throws HiveJdbcDatabaseAccessException {
return accessor.getColumnNames(conf).get(0);
}


private Configuration buildConfiguration() {
String scriptPath =
Expand Down
7 changes: 7 additions & 0 deletions jdbc-handler/src/test/resources/test_script.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
DROP TABLE IF EXISTS test_strategy;
DROP TABLE IF EXISTS test_writer;

CREATE TABLE IF NOT EXISTS test_strategy (
strategy_id int(11) NOT NULL,
Expand All @@ -18,6 +19,12 @@ INSERT INTO test_strategy (strategy_id, name, referrer, landing, priority, imple
INSERT INTO test_strategy (strategy_id, name, referrer, landing, priority, implementation, last_modified) VALUES (4,'S4','ddd','jkl',980,NULL,'2012-05-08 15:01:15');
INSERT INTO test_strategy (strategy_id, name, referrer, landing, priority, implementation, last_modified) VALUES (5,'S5','eee',NULL,NULL,NULL,'2012-05-08 15:01:15');

CREATE TABLE IF NOT EXISTS test_writer (
id int(11) NOT NULL,
name varchar(50) DEFAULT NULL,
PRIMARY KEY (id)
);

CREATE TABLE IF NOT EXISTS all_types_table (
char_col CHAR,
char_with_len_col CHAR(20),
Expand Down
Loading