From c2fc62806c88c0b8250b27532201f2c0652782c5 Mon Sep 17 00:00:00 2001 From: Priyanshu Soni Date: Fri, 15 May 2026 19:36:37 +0530 Subject: [PATCH] adding support for unauthenticated access from pinot to hadoop --- .../pinot/plugin/filesystem/HadoopPinotFS.java | 17 ++++++++++++++++- .../plugin/filesystem/HadoopPinotFSTest.java | 16 ++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/pinot-plugins/pinot-file-system/pinot-hdfs/src/main/java/org/apache/pinot/plugin/filesystem/HadoopPinotFS.java b/pinot-plugins/pinot-file-system/pinot-hdfs/src/main/java/org/apache/pinot/plugin/filesystem/HadoopPinotFS.java index be2aa28ec618..07a1e5542222 100644 --- a/pinot-plugins/pinot-file-system/pinot-hdfs/src/main/java/org/apache/pinot/plugin/filesystem/HadoopPinotFS.java +++ b/pinot-plugins/pinot-file-system/pinot-hdfs/src/main/java/org/apache/pinot/plugin/filesystem/HadoopPinotFS.java @@ -52,6 +52,8 @@ public class HadoopPinotFS extends BasePinotFS { private static final String KEYTAB = "hadoop.kerberos.keytab"; private static final String HADOOP_CONF_PATH = "hadoop.conf.path"; private static final String WRITE_CHECKSUM = "hadoop.write.checksum"; + private static final String GLOBAL_HADOOP_USER = "hadoop.user.name"; + private static final String ALLOW_INSECURE = "hadoop.allow.insecure"; private org.apache.hadoop.fs.FileSystem _hadoopFS = null; private org.apache.hadoop.conf.Configuration _hadoopConf; @@ -63,7 +65,20 @@ public HadoopPinotFS() { public void init(PinotConfiguration config) { try { _hadoopConf = getConf(config.getProperty(HADOOP_CONF_PATH)); - authenticate(_hadoopConf, config); + boolean allowInsecure = Boolean.parseBoolean(config.getProperty(ALLOW_INSECURE, "false")); + + if (!allowInsecure) { + authenticate(_hadoopConf, config); + } else { + String globalHadoopUser = config.getProperty(GLOBAL_HADOOP_USER); + if (Strings.isNullOrEmpty(globalHadoopUser)) { + throw new RuntimeException("HADOOP_USER must be provided when ALLOW_INSECURE is true"); + } + + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(globalHadoopUser); + UserGroupInformation.setLoginUser(ugi); + LOGGER.info("Setting HDFS login user to: {}", globalHadoopUser); + } _hadoopFS = org.apache.hadoop.fs.FileSystem.get(_hadoopConf); _hadoopFS.setWriteChecksum((config.getProperty(WRITE_CHECKSUM, false))); LOGGER.info("successfully initialized HadoopPinotFS"); diff --git a/pinot-plugins/pinot-file-system/pinot-hdfs/src/test/java/org/apache/pinot/plugin/filesystem/HadoopPinotFSTest.java b/pinot-plugins/pinot-file-system/pinot-hdfs/src/test/java/org/apache/pinot/plugin/filesystem/HadoopPinotFSTest.java index 9c5f7d9d576e..927e796bdce2 100644 --- a/pinot-plugins/pinot-file-system/pinot-hdfs/src/test/java/org/apache/pinot/plugin/filesystem/HadoopPinotFSTest.java +++ b/pinot-plugins/pinot-file-system/pinot-hdfs/src/test/java/org/apache/pinot/plugin/filesystem/HadoopPinotFSTest.java @@ -452,4 +452,20 @@ public void testDeleteBatchPerformanceWithManyFiles() hadoopFS.delete(baseURI, true); } } + + @Test + public void testInitWithUnauthenticatedUser() + throws IOException { + String testUser = "pinot-test-user-" + System.currentTimeMillis(); + + PinotConfiguration config = new PinotConfiguration(); + config.setProperty("hadoop.allow.insecure", "true"); + config.setProperty("hadoop.user.name", testUser); + + try (HadoopPinotFS hadoopFS = new HadoopPinotFS()) { + hadoopFS.init(config); + String currentUser = org.apache.hadoop.security.UserGroupInformation.getLoginUser().getUserName(); + Assert.assertEquals(currentUser, testUser, "The Hadoop login user was not set correctly!"); + } + } }