Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,10 @@ public void setup() throws Exception {
if (!Files.isDirectory(folder2)) {
Files.createDirectories(folder2);
}
Path propsFile2 = folder2.resolve("accumulo2.properties");
Path propsFile2 = folder2.resolve("org/apache/accumulo/core/accumulo2.properties");
FileUtils.copyURLToFile(
Objects.requireNonNull(this.getClass().getResource("/accumulo2.properties")),
Objects.requireNonNull(
this.getClass().getResource("/org/apache/accumulo/core/accumulo2.properties")),
propsFile2.toFile());
uri2 = propsFile2.toUri().toURL();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ public class CredentialProviderTokenTest {
justification = "keystoreUrl location isn't provided by user input")
@BeforeAll
public static void setup() throws Exception {
URL keystoreUrl = CredentialProviderTokenTest.class.getResource("/passwords.jceks");
URL keystoreUrl =
CredentialProviderTokenTest.class.getResource("/org/apache/accumulo/core/passwords.jceks");
assertNotNull(keystoreUrl);
keystorePath = "jceks://file/" + Path.of(keystoreUrl.toURI()).toAbsolutePath();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@

public class ClientContextTest {

private static final String keystoreName = "/site-cfg.jceks";
private static final String keystoreName = "/org/apache/accumulo/core/site-cfg.jceks";

// site-cfg.jceks={'ignored.property'=>'ignored', 'instance.secret'=>'mysecret',
// 'general.rpc.timeout'=>'timeout'}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ public class HadoopCredentialProviderTest {

private static final Configuration hadoopConf = new Configuration();

private static final String populatedKeyStoreName = "/accumulo.jceks";
private static final String emptyKeyStoreName = "/empty.jceks";
private static final String populatedKeyStoreName = "/org/apache/accumulo/core/accumulo.jceks";
private static final String emptyKeyStoreName = "/org/apache/accumulo/core/empty.jceks";
private static File emptyKeyStore;
private static File populatedKeyStore;

Expand Down Expand Up @@ -180,13 +180,13 @@ public void extractFromHdfs() throws Exception {

// One namenode, One configuration
Configuration dfsConfiguration = dfsCluster.getConfiguration(0);
Path destPath = new Path("/accumulo.jceks");
Path destPath = new Path("/org/apache/accumulo/core/accumulo.jceks");
FileSystem dfs = dfsCluster.getFileSystem();
// Put the populated keystore in hdfs
dfs.copyFromLocalFile(new Path(populatedKeyStore.toURI()), destPath);

Configuration cpConf = dfsConfiguration;
HadoopCredentialProvider.setPath(cpConf, "jceks://hdfs/accumulo.jceks");
HadoopCredentialProvider.setPath(cpConf, "jceks://hdfs/" + destPath);

// The values in the keystore
Map<String,String> expectations = new HashMap<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ public class SiteConfigurationTest {
@Test
public void testOnlySensitivePropertiesExtractedFromCredentialProvider()
throws SecurityException {
URL keystore = SiteConfigurationTest.class.getResource("/site-cfg.jceks");
URL keystore =
SiteConfigurationTest.class.getResource("/org/apache/accumulo/core/site-cfg.jceks");
assertNotNull(keystore);
String credProvPath = "jceks://file" + Path.of(keystore.getFile()).toAbsolutePath();

Expand Down Expand Up @@ -68,7 +69,8 @@ public void testDefault() {
@Test
public void testFile() {
System.setProperty("DIR", "/tmp/test/dir");
URL propsUrl = getClass().getClassLoader().getResource("accumulo2.properties");
URL propsUrl =
getClass().getClassLoader().getResource("org/apache/accumulo/core/accumulo2.properties");
var conf = new SiteConfiguration.Builder().fromUrl(propsUrl).build();
assertEquals("myhost123:2181", conf.get(Property.INSTANCE_ZK_HOST));
assertEquals("mysecret", conf.get(Property.INSTANCE_SECRET));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,8 @@ public static void setupMiniCluster() throws Exception {
Files.createDirectories(testDir);

jarFile = tempDir.resolve("iterator.jar").toFile();
FileUtils.copyURLToFile(
requireNonNull(MiniAccumuloClusterClasspathTest.class.getResource("/FooFilter.jar")),
jarFile);
FileUtils.copyURLToFile(requireNonNull(MiniAccumuloClusterClasspathTest.class
.getResource("/org/apache/accumulo/minicluster/FooFilter.jar")), jarFile);

MiniAccumuloConfig config =
new MiniAccumuloConfig(testDir.toFile(), ROOT_PASSWORD).setJDWPEnabled(true);
Expand Down
8 changes: 4 additions & 4 deletions server/tserver/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,8 @@
<licenseSet>
<inlineHeader>${accumulo.build.license.header}</inlineHeader>
<excludes>
<exclude>src/test/resources/*.walog</exclude>
<exclude>src/test/resources/walog-from-14/*</exclude>
<exclude>src/test/resources/org/apache/accumulo/tserver/*.walog</exclude>
<exclude>src/test/resources/org/apache/accumulo/tserver/walog-from-14/*</exclude>
</excludes>
</licenseSet>
</licenseSets>
Expand All @@ -153,8 +153,8 @@
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>src/test/resources/*.walog</exclude>
<exclude>src/test/resources/walog-from-14/*</exclude>
<exclude>src/test/resources/org/apache/accumulo/tserver/*.walog</exclude>
<exclude>src/test/resources/org/apache/accumulo/tserver/walog-from-14/*</exclude>
</excludes>
</configuration>
</plugin>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.Objects;
import java.util.concurrent.ScheduledThreadPoolExecutor;

import org.apache.accumulo.core.conf.DefaultConfiguration;
Expand All @@ -54,11 +55,11 @@
public class TestUpgradePathForWALogs extends WithTestNames {

// older logs no longer compatible
private static final String WALOG_FROM_15 = "walog-from-15.walog";
private static final String WALOG_FROM_15 = "org/apache/accumulo/tserver/walog-from-15.walog";
// logs from versions 1.6 through 1.10 should be the same
private static final String WALOG_FROM_16 = "walog-from-16.walog";
private static final String WALOG_FROM_16 = "org/apache/accumulo/tserver/walog-from-16.walog";
// logs from 2.0 were changed for improved crypto
private static final String WALOG_FROM_20 = "walog-from-20.walog";
private static final String WALOG_FROM_20 = "org/apache/accumulo/tserver/walog-from-20.walog";

private static final ScheduledThreadPoolExecutor EXECUTOR = new ScheduledThreadPoolExecutor(1);

Expand Down Expand Up @@ -108,72 +109,61 @@ public static void shutdown() {
*/
@Test
public void testUpgradeOf15WALog() throws IOException {
String walogToTest = WALOG_FROM_15;
String testPath = perTestTempSubDir.toAbsolutePath().toString();
java.nio.file.Path testPath = perTestTempSubDir.toAbsolutePath();
copyWalogToTestDir(WALOG_FROM_15);

try (InputStream walogStream = getClass().getResourceAsStream("/" + walogToTest);
OutputStream walogInHDFStream =
Files.newOutputStream(perTestTempSubDir.resolve(walogToTest))) {
walogStream.transferTo(walogInHDFStream);
walogInHDFStream.flush();
walogInHDFStream.close();

LogSorter logSorter = new LogSorter(server);
LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor();
LogSorter logSorter = new LogSorter(server);
LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor();

assertThrows(IllegalArgumentException.class,
() -> logProcessor.sort(context.getVolumeManager(), "/" + WALOG_FROM_15,
new Path("file://" + testPath + "/" + WALOG_FROM_15),
"file://" + testPath + "/manyMaps"));
}
assertThrows(IllegalArgumentException.class,
() -> logProcessor.sort(context.getVolumeManager(), "/" + WALOG_FROM_15,
new Path(testPath.toUri() + WALOG_FROM_15), testPath.toUri() + "manyMaps"));
}

@Test
public void testBasic16WALogRead() throws IOException {
String walogToTest = WALOG_FROM_16;
String testPath = perTestTempSubDir.toAbsolutePath().toString();
String destPath = "file://" + testPath + "/manyMaps";

try (InputStream walogStream = getClass().getResourceAsStream("/" + walogToTest);
OutputStream walogInHDFStream =
Files.newOutputStream(java.nio.file.Path.of(testPath).resolve(walogToTest))) {
walogStream.transferTo(walogInHDFStream);
walogInHDFStream.flush();
walogInHDFStream.close();
java.nio.file.Path testPath = perTestTempSubDir.toAbsolutePath();
String destPath = testPath.toUri() + "manyMaps";
copyWalogToTestDir(walogToTest);

assertFalse(context.getVolumeManager().exists(getFinishedMarkerPath(destPath)));
assertFalse(context.getVolumeManager().exists(getFinishedMarkerPath(destPath)));

LogSorter logSorter = new LogSorter(server);
LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor();
LogSorter logSorter = new LogSorter(server);
LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor();

logProcessor.sort(context.getVolumeManager(), "/" + walogToTest,
new Path("file://" + testPath + "/" + walogToTest), destPath);
logProcessor.sort(context.getVolumeManager(), "/" + walogToTest,
new Path(testPath.toUri() + walogToTest), destPath);

assertTrue(context.getVolumeManager().exists(getFinishedMarkerPath(destPath)));
}
assertTrue(context.getVolumeManager().exists(getFinishedMarkerPath(destPath)));
}

@Test
public void testBasic20WALogRead() throws IOException {
String walogToTest = WALOG_FROM_20;
String testPath = perTestTempSubDir.toAbsolutePath().toString();
String destPath = "file://" + testPath + "/manyMaps";
java.nio.file.Path testPath = perTestTempSubDir.toAbsolutePath();
String destPath = testPath.toUri() + "manyMaps";
copyWalogToTestDir(walogToTest);

try (InputStream walogStream = getClass().getResourceAsStream("/" + walogToTest);
OutputStream walogInHDFStream =
Files.newOutputStream(java.nio.file.Path.of(testPath).resolve(walogToTest))) {
walogStream.transferTo(walogInHDFStream);
walogInHDFStream.flush();
walogInHDFStream.close();
assertFalse(context.getVolumeManager().exists(getFinishedMarkerPath(destPath)));

assertFalse(context.getVolumeManager().exists(getFinishedMarkerPath(destPath)));
LogSorter logSorter = new LogSorter(server);
LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor();
logProcessor.sort(context.getVolumeManager(), "/" + walogToTest,
new Path(testPath.toUri() + walogToTest), destPath);

LogSorter logSorter = new LogSorter(server);
LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor();
logProcessor.sort(context.getVolumeManager(), "/" + walogToTest,
new Path("file://" + testPath + "/" + walogToTest), destPath);
assertTrue(context.getVolumeManager().exists(getFinishedMarkerPath(destPath)));
}

assertTrue(context.getVolumeManager().exists(getFinishedMarkerPath(destPath)));
private void copyWalogToTestDir(String walogToTest) throws IOException {
var walogPath = perTestTempSubDir.resolve(walogToTest);
Files.createDirectories(Objects.requireNonNull(walogPath.getParent()));

try (
InputStream walogStream =
Objects.requireNonNull(getClass().getResourceAsStream("/" + walogToTest));
OutputStream walogInHDFStream = Files.newOutputStream(walogPath)) {
walogStream.transferTo(walogInHDFStream);
}
}

Expand Down
19 changes: 0 additions & 19 deletions src/build/ci/check-module-package-conventions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,25 +39,6 @@ ALLOWED=(
server/native/src/main/resources/Makefile
server/native/src/main/resources/NOTICE

# TODO: these test resources should be moved into the correct package for the module
core/src/test/resources/accumulo.jceks
core/src/test/resources/empty.jceks
core/src/test/resources/site-cfg.jceks
core/src/test/resources/accumulo2.properties
core/src/test/resources/passwords.jceks
minicluster/src/test/resources/FooFilter.jar
server/tserver/src/test/resources/walog-from-15.walog
server/tserver/src/test/resources/walog-from-16.walog
server/tserver/src/test/resources/walog-from-14/550e8400-e29b-41d4-a716-446655440000
server/tserver/src/test/resources/walog-from-20.walog
test/src/main/resources/v2_import_test/README.md
test/src/main/resources/v2_import_test/data/A0000008.rf
test/src/main/resources/v2_import_test/data/A0000009.rf
test/src/main/resources/v2_import_test/data/A000000a.rf
test/src/main/resources/v2_import_test/data/A000000b.rf
test/src/main/resources/v2_import_test/data/distcp.txt
test/src/main/resources/v2_import_test/data/exportMetadata.zip

# TODO: these minicluster classes should be moved into the correct package for the module
minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
Expand Down
4 changes: 2 additions & 2 deletions test/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@
<licenseSet>
<inlineHeader>${accumulo.build.license.header}</inlineHeader>
<excludes>
<exclude>src/main/resources/v2_import_test/data/distcp.txt</exclude>
<exclude>src/main/resources/org/apache/accumulo/test/v2_import_test/data/distcp.txt</exclude>
</excludes>
</licenseSet>
</licenseSets>
Expand All @@ -288,7 +288,7 @@
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>src/main/resources/v2_import_test/data/distcp.txt</exclude>
<exclude>src/main/resources/org/apache/accumulo/test/v2_import_test/data/distcp.txt</exclude>
</excludes>
</configuration>
</plugin>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ private boolean verifyMappingsFile(String destTableId) throws IOException {
*/
@Test
public void importV2data() throws Exception {
final String dataRoot = "./target/classes/v2_import_test";
final String dataRoot = "./target/classes/org/apache/accumulo/test/v2_import_test";
final String dataSrc = dataRoot + "/data";
final String importDir = dataRoot + "/import";

Expand Down