Skip to content

Commit 9db913d

Browse files
HHMagnusHHMagnus
committed
Added a masking functionality to clone configurations
1 parent 45d81b8 commit 9db913d

14 files changed

Lines changed: 305 additions & 94 deletions

‎core/build.gradle‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ plugins {
33
}
44

55
dependencies {
6-
api 'org.postgresql:postgresql:42.7.9'
6+
implementation 'org.postgresql:postgresql:42.7.9'
7+
implementation 'org.apache.commons:commons-csv:1.14.1'
78

89
testImplementation 'org.testcontainers:postgresql:1.21.4'
910
testImplementation 'org.testcontainers:junit-jupiter:1.21.4'

‎core/src/main/java/dev/mhh/cloner/api/CloneConfiguration.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,16 @@
55
import dev.mhh.cloner.impl.builder.CloneConfigurationBuilderImpl;
66

77
import java.io.Serializable;
8-
import java.util.List;
8+
import java.util.Map;
9+
import java.util.Optional;
910
import java.util.function.Function;
1011

1112
public record CloneConfiguration<T extends Serializable>(
1213
TableConfig<T> rootTable,
1314
String idColumnName,
1415
String schema,
15-
Function<T, String> idToDatabaseStringFunction
16+
Function<T, String> idToDatabaseStringFunction,
17+
Map<String, TableMasker> tableMaskers
1618
) implements Serializable {
1719
public static <T extends Serializable> CloneConfigurationBuilder<T> builder(
1820
final String tableName,
@@ -21,4 +23,8 @@ public static <T extends Serializable> CloneConfigurationBuilder<T> builder(
2123
) {
2224
return new CloneConfigurationBuilderImpl<>(tableName, columnName, columnClass);
2325
}
26+
27+
public Optional<TableMasker> tableMasker(final String tableName) {
28+
return Optional.ofNullable(tableMaskers.get(tableName));
29+
}
2430
}

‎core/src/main/java/dev/mhh/cloner/api/CloneError.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,20 @@ public enum CloneError {
1515
PG_CONNECTION_UNWRAP_FAILED("Failed to unwrap the PGConnection. Did you pass a PGConnection?"),
1616
COPY_API_FAILED("Failed to get the copy API from the PGConnection"),
1717
ZIP_ENTRY_CREATION_FAILED("Failed to create a ZIP entry"),
18-
COPY_OUT_FAILED("Failed to copy data out from the database"),
18+
COPY_OUT_DIRECT_FAILED("Failed to copy data out from the database without masking"),
19+
COPY_OUT_MASKED_FAILED("Failed to copy data out from the database with masking"),
20+
COPY_OUT_READ_HEADER_FAILED("Failed to read the header row from the COPY output"),
21+
COPY_OUT_HEADER_EMPTY("The header row from the COPY output was empty"),
22+
COPY_OUT_WRITE_HEADER_FAILED("Failed to write the header row to the ZIP output"),
23+
COPY_OUT_HEADER_PARSER_FAILED("Failed to parse the header row from the COPY output"),
24+
COPY_OUT_HEADER_PARSER_CLOSE_FAILED("Failed to close the header parser after parsing the COPY output"),
25+
COPY_OUT_READ_ROW_FAILED("Failed to read a row from the COPY output"),
26+
COPY_OUT_ROW_PARSER_FAILED("Failed to parse a row from the COPY output"),
27+
COPY_OUT_ROW_PARSER_CLOSE_FAILED("Failed to close the row parser after parsing a COPY output row"),
28+
COPY_OUT_PRINTER_CREATION_FAILED("Failed to create a printer for the COPY output"),
29+
COPY_OUT_PRINT_ROW_FAILED("Failed to print a masked row to a String"),
30+
COPY_OUT_PRINTER_CLOSE_FAILED("Failed to close the printer after printing a masked row"),
31+
COPY_OUT_WRITE_ROW_FAILED("Failed to write a row to the ZIP output"),
1932
ZIP_ENTRY_CLOSE_FAILED("Failed to close a ZIP entry"),
2033
ZIP_FINISH_FAILED("Failed to finish the ZIP output stream"),
2134
ZIP_ENTRY_NEXT_FAILED("Failed to get the next ZIP entry"),

‎core/src/main/java/dev/mhh/cloner/api/TableMasker.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package dev.mhh.cloner.api;
2+
3+
import java.util.List;
4+
5+
public interface TableMasker {
6+
record Row(List<String> headers, List<String> values) {
7+
public boolean mask(final String header, final String value) {
8+
final var index = headers.indexOf(header);
9+
if (index == -1) {
10+
return false;
11+
}
12+
values.set(index, value);
13+
return true;
14+
}
15+
}
16+
Row mask(Row row);
17+
}

‎core/src/main/java/dev/mhh/cloner/api/builder/CloneConfigurationBuilder.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dev.mhh.cloner.api.builder;
22

33
import dev.mhh.cloner.api.CloneConfiguration;
4+
import dev.mhh.cloner.api.TableMasker;
45

56
import java.io.Serializable;
67
import java.util.List;
@@ -39,27 +40,35 @@ public interface CloneConfigurationBuilder<T extends Serializable> extends Clone
3940
*/
4041
CloneConfigurationBuilder<T> idsToDatabaseStringFunction(final Function<T, String> idsToDatabaseStringFunction);
4142

43+
/**
44+
* Defines a masker used to mask sensitive data in the clone.
45+
* @param tableName the table name to apply the masker to
46+
* @param tableMasker the masker to use
47+
* @return this builder for fluent chaining
48+
*/
49+
CloneConfigurationBuilder<T> masker(String tableName, TableMasker tableMasker);
50+
4251
@Override
4352
CloneConfigurationBuilder<T> joinByJoinTableForeignKey(String tableName, String foreignKey, Consumer<CloneConfigurationSubBuilder<T>> subTables);
4453

4554
@Override
46-
default CloneConfigurationBuilder<T> joinByJoinTableForeignKey(String tableName, String foreignKey) {
55+
default CloneConfigurationBuilder<T> joinByJoinTableForeignKey(final String tableName, final String foreignKey) {
4756
return joinByJoinTableForeignKey(tableName, foreignKey, _ -> {});
4857
}
4958

5059
@Override
5160
CloneConfigurationBuilder<T> joinByMainTableForeignKey(String tableName, String foreignKey, Consumer<CloneConfigurationSubBuilder<T>> subTables);
5261

5362
@Override
54-
default CloneConfigurationBuilder<T> joinByMainTableForeignKey(String tableName, String foreignKey) {
63+
default CloneConfigurationBuilder<T> joinByMainTableForeignKey(final String tableName, final String foreignKey) {
5564
return joinByMainTableForeignKey(tableName, foreignKey, _ -> {});
5665
}
5766

5867
@Override
5968
CloneConfigurationBuilder<T> joinByMatchingColumns(String tableName, String mainTableColumn, String joinedTableColumn, Consumer<CloneConfigurationSubBuilder<T>> subTables);
6069

6170
@Override
62-
default CloneConfigurationBuilder<T> joinByMatchingColumns(String tableName, String mainTableColumn, String joinedTableColumn) {
71+
default CloneConfigurationBuilder<T> joinByMatchingColumns(final String tableName, final String mainTableColumn, final String joinedTableColumn) {
6372
return joinByMatchingColumns(tableName, mainTableColumn, joinedTableColumn, _ -> {});
6473
}
6574

‎core/src/main/java/dev/mhh/cloner/impl/ClonerImpl.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import dev.mhh.cloner.api.CloneException;
66
import dev.mhh.cloner.api.Cloner;
77
import dev.mhh.cloner.api.verification.ConfigurationVerifier;
8+
import dev.mhh.cloner.impl.mask.CloneMasker;
89
import dev.mhh.cloner.impl.select.CloneConfigurationTransformer;
910
import dev.mhh.cloner.impl.select.DeletePreviousTransformer;
1011
import org.postgresql.PGConnection;
@@ -60,11 +61,8 @@ public void exportClone(
6061
CloneError.ZIP_ENTRY_CREATION_FAILED
6162
);
6263

63-
final var copyOut = "copy (" + select.sqlSelect() + ") to stdout with csv header";
64-
CloneException.wrapKnownExceptions(
65-
() -> copyManager.copyOut(copyOut, zos),
66-
CloneError.COPY_OUT_FAILED
67-
);
64+
final var copyOutSql = "copy (" + select.sqlSelect() + ") to stdout with csv header";
65+
CloneMasker.copyOut(copyManager, zos, select.tableName(), copyOutSql, configuration);
6866

6967
CloneException.wrapKnownExceptions(
7068
zos::closeEntry,

‎core/src/main/java/dev/mhh/cloner/impl/builder/CloneConfigurationBuilderImpl.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dev.mhh.cloner.impl.builder;
22

33
import dev.mhh.cloner.api.CloneConfiguration;
4+
import dev.mhh.cloner.api.TableMasker;
45
import dev.mhh.cloner.api.builder.CloneConfigurationBuilder;
56
import dev.mhh.cloner.api.builder.CloneConfigurationSubBuilder;
67
import dev.mhh.cloner.api.table.*;
@@ -21,6 +22,7 @@ public final class CloneConfigurationBuilderImpl<T extends Serializable>
2122
private String idColumnName = "id";
2223
private String schema = "public";
2324
private Function<T, String> idsToDatabaseStringFunction;
25+
private final Map<String, TableMasker> tableMaskers = new TreeMap<>();
2426

2527
public CloneConfigurationBuilderImpl(
2628
final String tableName,
@@ -51,10 +53,20 @@ public CloneConfigurationBuilder<T> schema(final String schema) {
5153
}
5254

5355
@Override
54-
public CloneConfigurationBuilder<T> idsToDatabaseStringFunction(Function<T, String> idsToDatabaseStringFunction) {
56+
public CloneConfigurationBuilder<T> idsToDatabaseStringFunction(final Function<T, String> idsToDatabaseStringFunction) {
57+
Objects.requireNonNull(idsToDatabaseStringFunction, "idsToDatabaseStringFunction must not be null");
5558
this.idsToDatabaseStringFunction = idsToDatabaseStringFunction;
5659
return this;
5760
}
61+
62+
@Override
63+
public CloneConfigurationBuilder<T> masker(final String tableName, final TableMasker tableMasker) {
64+
Objects.requireNonNull(tableName, "tableName must not be null");
65+
Objects.requireNonNull(tableMasker, "tableMasker must not be null");
66+
this.tableMaskers.put(tableName, tableMasker);
67+
return this;
68+
}
69+
5870
@Override
5971
public CloneConfigurationBuilder<T> joinByJoinTableForeignKey(
6072
final String tableName,
@@ -123,6 +135,6 @@ public CloneConfiguration<T> build() {
123135
joins
124136
);
125137
final var idToDatabaseString = idToDatabaseString();
126-
return new CloneConfiguration<T>(rootTable, idColumnName, schema, idToDatabaseString);
138+
return new CloneConfiguration<T>(rootTable, idColumnName, schema, idToDatabaseString, tableMaskers);
127139
}
128140
}

‎core/src/main/java/dev/mhh/cloner/impl/mask/CloneMasker.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
package dev.mhh.cloner.impl.mask;
2+
3+
import dev.mhh.cloner.api.CloneConfiguration;
4+
import dev.mhh.cloner.api.CloneError;
5+
import dev.mhh.cloner.api.CloneException;
6+
import dev.mhh.cloner.api.TableMasker;
7+
import org.apache.commons.csv.CSVFormat;
8+
import org.apache.commons.csv.CSVParser;
9+
import org.apache.commons.csv.CSVPrinter;
10+
import org.postgresql.copy.CopyManager;
11+
import org.postgresql.copy.CopyOut;
12+
13+
import java.io.Serializable;
14+
import java.nio.charset.StandardCharsets;
15+
import java.util.zip.ZipOutputStream;
16+
17+
public class CloneMasker {
18+
public static <T extends Serializable> void copyOut(
19+
final CopyManager copyManager,
20+
final ZipOutputStream zos,
21+
final String tableName,
22+
final String copyOutSql,
23+
final CloneConfiguration<T> cloneConfiguration
24+
) throws CloneException {
25+
final var maskerOpt = cloneConfiguration.tableMasker(tableName);
26+
if (maskerOpt.isEmpty()) {
27+
CloneException.wrapKnownExceptions(
28+
() -> copyManager.copyOut(copyOutSql, zos),
29+
CloneError.COPY_OUT_DIRECT_FAILED
30+
);
31+
return;
32+
}
33+
final var masker = maskerOpt.get();
34+
35+
final var copyOut = CloneException.wrapKnownExceptions(
36+
() -> copyManager.copyOut(copyOutSql),
37+
CloneError.COPY_OUT_MASKED_FAILED
38+
);
39+
40+
maskCopyOut(zos, copyOut, masker);
41+
}
42+
43+
private static void maskCopyOut(
44+
final ZipOutputStream zos,
45+
final CopyOut copyOut,
46+
final TableMasker masker
47+
) throws CloneException {
48+
final var headerRow = CloneException.wrapKnownExceptions(
49+
() -> copyOut.readFromCopy(),
50+
CloneError.COPY_OUT_READ_HEADER_FAILED
51+
);
52+
53+
if (headerRow == null) {
54+
throw CloneError.COPY_OUT_HEADER_EMPTY.toException();
55+
}
56+
57+
CloneException.wrapKnownExceptions(
58+
() -> zos.write(headerRow),
59+
CloneError.COPY_OUT_WRITE_HEADER_FAILED
60+
);
61+
62+
final var headerLine = new String(headerRow, StandardCharsets.UTF_8);
63+
64+
final var headerParser = CloneException.wrapKnownExceptions(
65+
() -> CSVParser.parse(headerLine, CSVFormat.POSTGRESQL_CSV),
66+
CloneError.COPY_OUT_HEADER_PARSER_FAILED
67+
);
68+
final var headers = headerParser.getRecords().getFirst().toList();
69+
CloneException.wrapKnownExceptions(
70+
headerParser::close,
71+
CloneError.COPY_OUT_HEADER_PARSER_CLOSE_FAILED
72+
);
73+
74+
byte[] row;
75+
while((row = CloneException.wrapKnownExceptions(
76+
() -> copyOut.readFromCopy(),
77+
CloneError.COPY_OUT_READ_ROW_FAILED
78+
)) != null) {
79+
final var line = new String(row, StandardCharsets.UTF_8);
80+
final var parser = CloneException.wrapKnownExceptions(
81+
() -> CSVParser.parse(line, CSVFormat.POSTGRESQL_CSV),
82+
CloneError.COPY_OUT_ROW_PARSER_FAILED
83+
);
84+
final var values = parser.getRecords().getFirst().toList();
85+
CloneException.wrapKnownExceptions(
86+
parser::close,
87+
CloneError.COPY_OUT_ROW_PARSER_CLOSE_FAILED
88+
);
89+
90+
final var maskerRow = new TableMasker.Row(headers, values);
91+
masker.mask(maskerRow);
92+
93+
final var stringBuilder = new StringBuilder();
94+
final var printer = CloneException.wrapKnownExceptions(
95+
() -> new CSVPrinter(stringBuilder, CSVFormat.POSTGRESQL_CSV),
96+
CloneError.COPY_OUT_PRINTER_CREATION_FAILED
97+
);
98+
CloneException.wrapKnownExceptions(
99+
() -> printer.printRecord(maskerRow.values()),
100+
CloneError.COPY_OUT_PRINT_ROW_FAILED
101+
);
102+
CloneException.wrapKnownExceptions(
103+
() -> printer.close(),
104+
CloneError.COPY_OUT_PRINTER_CLOSE_FAILED
105+
);
106+
CloneException.wrapKnownExceptions(
107+
() -> zos.write(stringBuilder.toString().getBytes(StandardCharsets.UTF_8)),
108+
CloneError.COPY_OUT_WRITE_ROW_FAILED
109+
);
110+
}
111+
}
112+
}

‎core/src/main/java/module-info.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
module dev.mhh.cloner {
22
requires java.sql;
33
requires org.postgresql.jdbc;
4+
requires org.apache.commons.csv;
45
exports dev.mhh.cloner.api;
56
exports dev.mhh.cloner.api.table;
67
exports dev.mhh.cloner.api.verification;

0 commit comments

Comments
 (0)