From d21530c31181b964253cebdbfcf025f541b07d31 Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 12:43:35 +0100 Subject: [PATCH 01/11] Add repository provider experiment with SQLite memory mode --- challenger/pom.xml | 8 +- .../challenge/ChallengeMain.java | 5 +- .../challenge/apimodel/ChallengeApiModel.java | 11 +- .../ChallengeApiModelRepositoryTest.java | 24 + .../compendiumdev/sparkstart/Environment.java | 2 +- ercoremodel/pom.xml | 7 +- .../thingifier/core/EntityRelModel.java | 83 ++- .../core/domain/instances/EntityInstance.java | 6 +- .../EntityInstanceRelationships.java | 4 + .../repository/InMemoryThingRepository.java | 179 ++++++ .../InMemoryThingRepositoryProvider.java | 72 +++ .../repository/SqliteThingRepository.java | 534 ++++++++++++++++++ .../SqliteThingRepositoryProvider.java | 98 ++++ .../core/repository/ThingRepository.java | 75 +++ .../repository/ThingRepositoryProvider.java | 27 + .../ThingRepositoryProviderConfig.java | 131 +++++ .../ThingRepositoryContractTest.java | 188 ++++++ .../ThingRepositoryProviderConfigTest.java | 64 +++ pom.xml | 3 +- .../compendiumdev/thingifier/Thingifier.java | 32 +- .../restapihandlers/RelationshipCreation.java | 3 +- .../restapihandlers/RelationshipCreator.java | 3 +- .../restapihandlers/RestApiDeleteHandler.java | 3 +- .../restapihandlers/RestApiPostHandler.java | 2 +- .../api/restapihandlers/ThingAmendment.java | 7 +- .../api/restapihandlers/ThingCreation.java | 13 +- 26 files changed, 1529 insertions(+), 55 deletions(-) create mode 100644 challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepositoryProvider.java create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepositoryProvider.java create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProvider.java create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProviderConfig.java create mode 100644 ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java create mode 100644 ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProviderConfigTest.java diff --git a/challenger/pom.xml b/challenger/pom.xml index 13ed900b..4c442e0c 100644 --- a/challenger/pom.xml +++ b/challenger/pom.xml @@ -19,6 +19,12 @@ ${thingifier.version} + + org.xerial + sqlite-jdbc + ${sqlite-jdbc-version} + + io.swagger.core.v3 swagger-models @@ -166,4 +172,4 @@ https://www.slf4j.org/manual.html - initial release ---> \ No newline at end of file +--> diff --git a/challenger/src/main/java/uk/co/compendiumdev/challenge/ChallengeMain.java b/challenger/src/main/java/uk/co/compendiumdev/challenge/ChallengeMain.java index 47e4022b..afacaaf0 100644 --- a/challenger/src/main/java/uk/co/compendiumdev/challenge/ChallengeMain.java +++ b/challenger/src/main/java/uk/co/compendiumdev/challenge/ChallengeMain.java @@ -7,6 +7,7 @@ import uk.co.compendiumdev.thingifier.Thingifier; import uk.co.compendiumdev.thingifier.application.MainImplementation; import uk.co.compendiumdev.thingifier.application.httprouting.ThingifierHttpApiRoutings; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepositoryProviderConfig; public class ChallengeMain { @@ -19,7 +20,9 @@ public static void main(String[] args) { logger.info("Starting Challenger"); MainImplementation app = new MainImplementation(); - Thingifier thingifier = new ChallengeApiModel().get(); + ThingRepositoryProviderConfig repositoryConfig = ThingRepositoryProviderConfig.fromArgs(args); + logger.info("Using Thingifier repository {}", repositoryConfig.describe()); + Thingifier thingifier = new ChallengeApiModel().get(repositoryConfig.createProvider()); app.registerModel("challengeapi", thingifier); // add any additional thingifier configurations here if more needed than model has defined diff --git a/challenger/src/main/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModel.java b/challenger/src/main/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModel.java index 6f10cb33..8f2836bc 100644 --- a/challenger/src/main/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModel.java +++ b/challenger/src/main/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModel.java @@ -6,12 +6,21 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; import uk.co.compendiumdev.thingifier.core.domain.definitions.validation.VRule; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepositoryProvider; import static uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType.STRING; public class ChallengeApiModel { public Thingifier get() { - Thingifier todoList = new Thingifier(); + return get(new EntityRelModel()); + } + + public Thingifier get(final ThingRepositoryProvider repositoryProvider) { + return get(new EntityRelModel(repositoryProvider)); + } + + public Thingifier get(final EntityRelModel entityRelModel) { + Thingifier todoList = new Thingifier(entityRelModel); StringBuilder para = new StringBuilder(); diff --git a/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java b/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java new file mode 100644 index 00000000..3a98e0f3 --- /dev/null +++ b/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java @@ -0,0 +1,24 @@ +package uk.co.compendiumdev.challenge.apimodel; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import uk.co.compendiumdev.thingifier.Thingifier; +import uk.co.compendiumdev.thingifier.core.EntityRelModel; +import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepository; +import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepositoryProvider; + +public class ChallengeApiModelRepositoryTest { + + @Test + public void challengeApiCanUseSqliteInMemoryRepository() { + Thingifier thingifier = new ChallengeApiModel().get(SqliteThingRepositoryProvider.inMemory()); + + Assertions.assertTrue( + thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME) + instanceof SqliteThingRepository); + Assertions.assertEquals( + 10, + thingifier.getThingInstancesNamed("todo", EntityRelModel.DEFAULT_DATABASE_NAME). + countInstances()); + } +} diff --git a/challengerAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java b/challengerAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java index 79e893bb..79ab387e 100644 --- a/challengerAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java +++ b/challengerAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java @@ -37,7 +37,7 @@ public static String getBaseUri() { String [] args; if(SINGLE_PLAYER_MODE){ - args = "".split(","); + args = "-sqlite-memory".split(","); }else{ args = "-multiplayer".split(","); }; diff --git a/ercoremodel/pom.xml b/ercoremodel/pom.xml index dd693d48..29250f06 100644 --- a/ercoremodel/pom.xml +++ b/ercoremodel/pom.xml @@ -46,6 +46,11 @@ ${junit.jupiter.version} test + + org.xerial + sqlite-jdbc + ${sqlite-jdbc-version} + @@ -100,4 +105,4 @@ - \ No newline at end of file + diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java index 5691b993..aa6cf253 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java @@ -7,6 +7,9 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.relationship.RelationshipDefinition; import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.repository.InMemoryThingRepositoryProvider; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepositoryProvider; import java.util.*; @@ -19,23 +22,35 @@ public class EntityRelModel { public static final String DEFAULT_DATABASE_NAME = "__default"; - // a Map so that key, database can be used - // e.g. key from a 'session', or 'custom' or 'default' - private final Map databases; + // a provider so that key, database can be backed by memory, files, SQLite, etc. private final ERSchema schema; // all the definitions + private final ThingRepositoryProvider repositories; private DataPopulator dataPopulator; public EntityRelModel(){ schema = new ERSchema(); - databases = new HashMap(); - databases.put(DEFAULT_DATABASE_NAME, new ERInstanceData()); + repositories = new InMemoryThingRepositoryProvider(); dataPopulator = null; } public EntityRelModel(final ERSchema schema, final ERInstanceData erInstanceData) { this.schema = schema; - this.databases = new HashMap(); - this.databases.put(DEFAULT_DATABASE_NAME,erInstanceData); + this.repositories = new InMemoryThingRepositoryProvider(erInstanceData); + dataPopulator = null; + } + + public EntityRelModel(final ThingRepositoryProvider repositories) { + this.schema = new ERSchema(); + this.repositories = repositories; + this.repositories.getDefaultRepository().initializeFrom(schema); + dataPopulator = null; + } + + public EntityRelModel(final ERSchema schema, final ThingRepositoryProvider repositories) { + this.schema = schema; + this.repositories = repositories; + this.repositories.getDefaultRepository().initializeFrom(schema); + dataPopulator = null; } public EntityDefinition createEntityDefinition(final String entityName, final String pluralName) { @@ -44,8 +59,8 @@ public EntityDefinition createEntityDefinition(final String entityName, final St public EntityDefinition createEntityDefinition(final String entityName, final String pluralName, int maximumNumberOfInstances) { EntityDefinition defn = schema.defineEntity(entityName, pluralName, maximumNumberOfInstances); - for(ERInstanceData database : databases.values()){ - database.createInstanceCollectionFor(defn); + for(String databaseKey : repositories.getRepositoryNames()){ + repositories.getRepository(databaseKey).createInstanceCollectionFor(defn); } return defn; } @@ -57,19 +72,31 @@ public ERSchema getSchema(){ // TODO: use of this is basically deprecated since is refers to the default database @Deprecated() // we should use the parameterised version public ERInstanceData getInstanceData(){ - return databases.get(DEFAULT_DATABASE_NAME); + return getInstanceData(DEFAULT_DATABASE_NAME); } public ERInstanceData getInstanceDataAsJson(){ - return databases.get(DEFAULT_DATABASE_NAME); + return getInstanceData(DEFAULT_DATABASE_NAME); } public ERInstanceData getInstanceData(String databaseKey) { - return databases.get(databaseKey); + ThingRepository repository = repositories.getRepository(databaseKey); + if(repository==null){ + return null; + } + return repository.getInstanceData(); } public Set getDatabaseNames(){ - return databases.keySet(); + return repositories.getRepositoryNames(); + } + + public ThingRepository getRepository(String databaseKey) { + return repositories.getRepository(databaseKey); + } + + public ThingRepositoryProvider getRepositoryProvider() { + return repositories; } // ERM Object Level @@ -97,7 +124,9 @@ public Collection getEntityDefinitions() { public RelationshipDefinition createRelationshipDefinition( EntityDefinition from, EntityDefinition to, final String named, final Cardinality of) { - return schema.defineRelationship(from, to, named, of); + RelationshipDefinition relationship = schema.defineRelationship(from, to, named, of); + refreshRepositorySchemas(); + return relationship; } public boolean hasRelationshipNamed(final String relationshipName) { @@ -120,7 +149,7 @@ public EntityDefinition getEntityDefinitionNamed(final String term){ // Multiple Databases public void createInstanceDatabase(String databaseKey) { - if(databases.containsKey(databaseKey)){ + if(repositories.getRepository(databaseKey)!=null){ throw new IllegalStateException("ERM Database Already Exists with name " + databaseKey); } @@ -132,22 +161,15 @@ public void deleteInstanceDatabase(String databaseKey) { if(databaseKey.equals(DEFAULT_DATABASE_NAME)){ throw new IllegalStateException("Cannot delete default database"); } - databases.remove(databaseKey); + repositories.deleteRepository(databaseKey); } public boolean createInstanceDatabaseIfNotExisting(String databaseKey) { - if(databases.containsKey(databaseKey)){ - return false; - } - - ERInstanceData aDatabase = new ERInstanceData(); - aDatabase.createInstanceCollectionFrom(this.schema); - databases.put(databaseKey, aDatabase); - return true; + return repositories.createRepositoryIfNotExisting(databaseKey, this.schema); } public boolean populateDatabase(String databaseKey){ - if(!databases.containsKey(databaseKey)){ + if(repositories.getRepository(databaseKey)==null){ return false; } @@ -155,10 +177,13 @@ public boolean populateDatabase(String databaseKey){ return false; } + repositories.getRepository(databaseKey).refreshSchema(getSchema()); + dataPopulator.populate( getSchema(), - getInstanceData(databaseKey) + repositories.getRepository(databaseKey).getInstanceData() ); + repositories.getRepository(databaseKey).flush(); return true; } @@ -166,4 +191,10 @@ public boolean populateDatabase(String databaseKey){ public void setDataGenerator(DataPopulator dataPopulator) { this.dataPopulator = dataPopulator; } + + private void refreshRepositorySchemas() { + for(String databaseKey : repositories.getRepositoryNames()){ + repositories.getRepository(databaseKey).refreshSchema(schema); + } + } } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/instances/EntityInstance.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/instances/EntityInstance.java index 541b7cac..021d1339 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/instances/EntityInstance.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/instances/EntityInstance.java @@ -20,10 +20,14 @@ public class EntityInstance { private final UUID internalId; public EntityInstance(EntityDefinition eDefn) { + this(eDefn, UUID.randomUUID()); + } + + public EntityInstance(EntityDefinition eDefn, UUID internalId) { this.entityDefinition = eDefn; this.instanceFields = eDefn.instantiateFields(); this.relationships = new EntityInstanceRelationships(this); - internalId = UUID.randomUUID(); + this.internalId = internalId; } public EntityInstance addAutoGUIDstoInstance(){ diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/instances/EntityInstanceRelationships.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/instances/EntityInstanceRelationships.java index f30e34ab..517cc5c0 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/instances/EntityInstanceRelationships.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/instances/EntityInstanceRelationships.java @@ -216,6 +216,10 @@ public boolean hasAnyRelationshipInstances() { return !relationships.isEmpty(); } + public List getRelationshipInstances() { + return new ArrayList<>(relationships); + } + public ValidationReport validateRelationships() { ValidationReport report = new ValidationReport(); diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java new file mode 100644 index 00000000..3003170f --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java @@ -0,0 +1,179 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.instance.NamedValue; +import uk.co.compendiumdev.thingifier.core.domain.instances.AutoIncrement; +import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; +import uk.co.compendiumdev.thingifier.core.reporting.ValidationReport; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class InMemoryThingRepository implements ThingRepository { + + private final String databaseKey; + private final ERInstanceData instanceData; + + public InMemoryThingRepository(final String databaseKey) { + this(databaseKey, new ERInstanceData()); + } + + public InMemoryThingRepository(final String databaseKey, final ERInstanceData instanceData) { + this.databaseKey = databaseKey; + this.instanceData = instanceData; + } + + @Override + public String databaseKey() { + return databaseKey; + } + + @Override + public void initializeFrom(final ERSchema schema) { + refreshSchema(schema); + } + + @Override + public void refreshSchema(final ERSchema schema) { + createInstanceCollectionsFrom(schema); + } + + @Override + public ERInstanceData getInstanceData() { + return instanceData; + } + + @Override + public EntityInstanceCollection createInstanceCollectionFor(final EntityDefinition definition) { + EntityInstanceCollection existing = getInstanceCollectionForEntityNamed(definition.getName()); + if (existing != null) { + return existing; + } + return instanceData.createInstanceCollectionFor(definition); + } + + @Override + public void createInstanceCollectionsFrom(final ERSchema schema) { + for (EntityDefinition defn : schema.getEntityDefinitions()) { + createInstanceCollectionFor(defn); + } + } + + @Override + public List getAllInstanceCollections() { + return instanceData.getAllInstanceCollections(); + } + + @Override + public EntityInstanceCollection getInstanceCollectionForEntityNamed(final String entityName) { + return instanceData.getInstanceCollectionForEntityNamed(entityName); + } + + @Override + public EntityInstance findEntityInstanceByGUID(final String thingGUID) { + return instanceData.findEntityInstanceByGUID(thingGUID); + } + + @Override + public EntityInstance findInstanceByPrimaryKey( + final EntityDefinition entity, final String primaryKeyValue) { + EntityInstanceCollection collection = getInstanceCollectionForEntityNamed(entity.getName()); + if (collection == null) { + return null; + } + return collection.findInstanceByPrimaryKey(primaryKeyValue); + } + + @Override + public EntityInstance findInstanceByFieldNameAndValue( + final EntityDefinition entity, final String fieldName, final String fieldValue) { + EntityInstanceCollection collection = getInstanceCollectionForEntityNamed(entity.getName()); + if (collection == null) { + return null; + } + return collection.findInstanceByFieldNameAndValue(fieldName, fieldValue); + } + + @Override + public Collection listInstances(final EntityDefinition entity) { + EntityInstanceCollection collection = getInstanceCollectionForEntityNamed(entity.getName()); + if (collection == null) { + return List.of(); + } + return collection.getInstances(); + } + + @Override + public EntityInstance addInstance(final EntityInstance instance) { + getInstanceCollectionForEntityNamed(instance.getEntity().getName()).addInstance(instance); + return instance; + } + + @Override + public EntityInstance updateInstance(final EntityInstance instance) { + return instance; + } + + @Override + public void deleteEntityInstance(final EntityInstance instance) { + instanceData.deleteEntityInstance(instance); + } + + @Override + public void clearAllData() { + instanceData.clearAllData(); + } + + @Override + public void clearInstanceDataFor(final String entityName) { + instanceData.clearInstanceDataFor(entityName); + } + + @Override + public ValidationReport checkFieldsForUniqueNess( + final EntityInstance instance, final boolean isAmendment) { + return getInstanceCollectionForEntityNamed(instance.getEntity().getName()). + checkFieldsForUniqueNess(instance, isAmendment); + } + + @Override + public Map countersFor(final EntityDefinition entity) { + return getInstanceCollectionForEntityNamed(entity.getName()).getCounters(); + } + + @Override + public void setNextIdCountersToAccomodate( + final EntityDefinition entity, final List fieldValues) { + getInstanceCollectionForEntityNamed(entity.getName()). + setNextIdCountersToAccomodate(fieldValues); + } + + @Override + public void connectRelationship( + final EntityInstance from, final String relationshipName, final EntityInstance to) { + from.getRelationships().connect(relationshipName, to); + } + + @Override + public List removeRelationshipsInvolving( + final EntityInstance parent, + final EntityInstance child, + final String relationshipName) { + return parent.getRelationships().removeRelationshipsInvolving(child, relationshipName); + } + + @Override + public List removeAllRelationships(final EntityInstance instance) { + return instance.getRelationships().removeAllRelationships(); + } + + @Override + public Collection getConnectedItems( + final EntityInstance instance, final String relationshipName) { + return instance.getRelationships().getConnectedItems(relationshipName); + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepositoryProvider.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepositoryProvider.java new file mode 100644 index 00000000..a44e07c9 --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepositoryProvider.java @@ -0,0 +1,72 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import uk.co.compendiumdev.thingifier.core.EntityRelModel; +import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; +import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class InMemoryThingRepositoryProvider implements ThingRepositoryProvider { + + private final Map repositories; + + public InMemoryThingRepositoryProvider() { + this(new InMemoryThingRepository(EntityRelModel.DEFAULT_DATABASE_NAME)); + } + + public InMemoryThingRepositoryProvider(final ERInstanceData defaultInstanceData) { + this(new InMemoryThingRepository(EntityRelModel.DEFAULT_DATABASE_NAME, defaultInstanceData)); + } + + public InMemoryThingRepositoryProvider(final ThingRepository defaultRepository) { + repositories = new HashMap<>(); + repositories.put(EntityRelModel.DEFAULT_DATABASE_NAME, defaultRepository); + } + + @Override + public ThingRepository getDefaultRepository() { + return repositories.get(EntityRelModel.DEFAULT_DATABASE_NAME); + } + + @Override + public ThingRepository getRepository(final String databaseKey) { + return repositories.get(databaseKey); + } + + @Override + public Set getRepositoryNames() { + return new HashSet<>(repositories.keySet()); + } + + @Override + public ThingRepository createRepository(final String databaseKey, final ERSchema schema) { + if (repositories.containsKey(databaseKey)) { + throw new IllegalStateException("ERM Database Already Exists with name " + databaseKey); + } + + ThingRepository repository = new InMemoryThingRepository(databaseKey); + repository.initializeFrom(schema); + repositories.put(databaseKey, repository); + return repository; + } + + @Override + public boolean createRepositoryIfNotExisting(final String databaseKey, final ERSchema schema) { + if (repositories.containsKey(databaseKey)) { + return false; + } + + ThingRepository repository = new InMemoryThingRepository(databaseKey); + repository.initializeFrom(schema); + repositories.put(databaseKey, repository); + return true; + } + + @Override + public void deleteRepository(final String databaseKey) { + repositories.remove(databaseKey); + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java new file mode 100644 index 00000000..8c922f9a --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java @@ -0,0 +1,534 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.instance.FieldValue; +import uk.co.compendiumdev.thingifier.core.domain.definitions.relationship.RelationshipDefinition; +import uk.co.compendiumdev.thingifier.core.domain.definitions.relationship.RelationshipVectorDefinition; +import uk.co.compendiumdev.thingifier.core.domain.instances.AutoIncrement; +import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; +import uk.co.compendiumdev.thingifier.core.domain.instances.RelationshipVectorInstance; +import uk.co.compendiumdev.thingifier.core.reporting.ValidationReport; + +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class SqliteThingRepository extends InMemoryThingRepository { + + private static final String INTERNAL_ID_COLUMN = "__internal_id"; + private static final String COUNTERS_TABLE = "__thingifier_counters"; + + private final String jdbcUrl; + private ERSchema schema; + private Connection connection; + private boolean loadedExistingData; + + public SqliteThingRepository(final String databaseKey, final String jdbcUrl) { + super(databaseKey, new ERInstanceData()); + this.jdbcUrl = jdbcUrl; + } + + public static SqliteThingRepository inMemory(final String databaseKey) { + return new SqliteThingRepository(databaseKey, "jdbc:sqlite::memory:"); + } + + public static SqliteThingRepository fileBacked(final String databaseKey, final Path databasePath) { + String path = databasePath.toAbsolutePath().toString().replace("\\", "/"); + return new SqliteThingRepository(databaseKey, "jdbc:sqlite:" + path); + } + + @Override + public void initializeFrom(final ERSchema schema) { + this.schema = schema; + openConnection(); + createMetadataTables(); + refreshSchema(schema); + if (!loadedExistingData) { + loadExistingData(); + loadedExistingData = true; + } + } + + @Override + public void refreshSchema(final ERSchema schema) { + this.schema = schema; + openConnection(); + super.refreshSchema(schema); + for (EntityDefinition entity : schema.getEntityDefinitions()) { + createEntityTable(entity); + } + for (RelationshipVectorDefinition vector : relationshipVectors()) { + createRelationshipTable(vector); + } + } + + @Override + public EntityInstanceCollection createInstanceCollectionFor(final EntityDefinition definition) { + EntityInstanceCollection collection = super.createInstanceCollectionFor(definition); + if (connection != null) { + createEntityTable(definition); + } + return collection; + } + + @Override + public EntityInstance addInstance(final EntityInstance instance) { + ensureSchemaReady(); + super.addInstance(instance); + persistInstance(instance); + persistCountersFor(instance.getEntity()); + return instance; + } + + @Override + public EntityInstance updateInstance(final EntityInstance instance) { + ensureSchemaReady(); + persistInstance(instance); + persistCountersFor(instance.getEntity()); + return instance; + } + + @Override + public void deleteEntityInstance(final EntityInstance instance) { + Set idsBeforeDelete = currentInternalIds(); + super.deleteEntityInstance(instance); + Set idsAfterDelete = currentInternalIds(); + + idsBeforeDelete.removeAll(idsAfterDelete); + for (String internalId : idsBeforeDelete) { + deletePersistedInstance(internalId); + } + replaceAllRelationshipRowsFromCache(); + } + + @Override + public void clearAllData() { + super.clearAllData(); + for (EntityDefinition entity : schema.getEntityDefinitions()) { + executeSql("DELETE FROM " + table(entity)); + persistCountersFor(entity); + } + clearRelationshipRows(); + } + + @Override + public void clearInstanceDataFor(final String entityName) { + EntityInstanceCollection collection = getInstanceCollectionForEntityNamed(entityName); + if (collection == null) { + return; + } + super.clearInstanceDataFor(entityName); + executeSql("DELETE FROM " + table(collection.definition())); + persistCountersFor(collection.definition()); + replaceAllRelationshipRowsFromCache(); + } + + @Override + public void connectRelationship( + final EntityInstance from, final String relationshipName, final EntityInstance to) { + super.connectRelationship(from, relationshipName, to); + replaceAllRelationshipRowsFromCache(); + } + + @Override + public List removeRelationshipsInvolving( + final EntityInstance parent, + final EntityInstance child, + final String relationshipName) { + List removed = super.removeRelationshipsInvolving(parent, child, relationshipName); + replaceAllRelationshipRowsFromCache(); + return removed; + } + + @Override + public List removeAllRelationships(final EntityInstance instance) { + List removed = super.removeAllRelationships(instance); + replaceAllRelationshipRowsFromCache(); + return removed; + } + + @Override + public void flush() { + refreshSchema(schema); + for (EntityDefinition entity : schema.getEntityDefinitions()) { + executeSql("DELETE FROM " + table(entity)); + } + clearRelationshipRows(); + + for (EntityInstanceCollection collection : getAllInstanceCollections()) { + for (EntityInstance instance : collection.getInstances()) { + persistInstance(instance); + } + persistCountersFor(collection.definition()); + } + + replaceAllRelationshipRowsFromCache(); + } + + @Override + public void close() { + if (connection == null) { + return; + } + try { + connection.close(); + connection = null; + } catch (SQLException e) { + throw new IllegalStateException("Could not close SQLite repository", e); + } + } + + private void openConnection() { + if (connection != null) { + return; + } + try { + Class.forName("org.sqlite.JDBC"); + connection = DriverManager.getConnection(jdbcUrl); + } catch (ClassNotFoundException e) { + throw new IllegalStateException( + "SQLite repository requires org.xerial:sqlite-jdbc on the runtime classpath", e); + } catch (SQLException e) { + throw new IllegalStateException("Could not open SQLite repository " + jdbcUrl, e); + } + } + + private void ensureSchemaReady() { + if (schema != null) { + refreshSchema(schema); + } + } + + private void createMetadataTables() { + executeSql( + "CREATE TABLE IF NOT EXISTS " + identifier(COUNTERS_TABLE) + " (" + + "entity_name TEXT NOT NULL, " + + "field_name TEXT NOT NULL, " + + "next_value INTEGER NOT NULL, " + + "PRIMARY KEY(entity_name, field_name))"); + } + + private void createEntityTable(final EntityDefinition entity) { + executeSql( + "CREATE TABLE IF NOT EXISTS " + table(entity) + " (" + + identifier(INTERNAL_ID_COLUMN) + " TEXT PRIMARY KEY)"); + + Set existingColumns = columnNames(entityTableName(entity)); + for (String fieldName : entity.getFieldNames()) { + if (!existingColumns.contains(fieldName)) { + Field field = entity.getField(fieldName); + executeSql( + "ALTER TABLE " + table(entity) + " ADD COLUMN " + + identifier(fieldName) + " " + sqlType(field.getType())); + } + } + + if (entity.hasPrimaryKeyField()) { + createUniqueIndex(entity, entity.getPrimaryKeyField()); + } + for (String fieldName : entity.getFieldNames()) { + Field field = entity.getField(fieldName); + if (field.mustBeUnique()) { + createUniqueIndex(entity, field); + } + } + } + + private void createUniqueIndex(final EntityDefinition entity, final Field field) { + String indexName = "__thingifier_idx_" + + Integer.toHexString((entity.getName() + "_" + field.getName()).hashCode()); + executeSql( + "CREATE UNIQUE INDEX IF NOT EXISTS " + identifier(indexName) + + " ON " + table(entity) + " (" + identifier(field.getName()) + ")"); + } + + private void createRelationshipTable(final RelationshipVectorDefinition vector) { + executeSql( + "CREATE TABLE IF NOT EXISTS " + relationshipTable(vector) + " (" + + "from_internal_id TEXT NOT NULL, " + + "to_internal_id TEXT NOT NULL, " + + "PRIMARY KEY(from_internal_id, to_internal_id))"); + } + + private void loadExistingData() { + for (EntityDefinition entity : schema.getEntityDefinitions()) { + loadEntityRows(entity); + restoreCountersFor(entity); + } + loadRelationshipRows(); + } + + private void loadEntityRows(final EntityDefinition entity) { + EntityInstanceCollection collection = getInstanceCollectionForEntityNamed(entity.getName()); + String sql = "SELECT * FROM " + table(entity); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + while (resultSet.next()) { + String internalId = resultSet.getString(INTERNAL_ID_COLUMN); + if (collection.findInstanceByInternalID(internalId) != null) { + continue; + } + + EntityInstance instance = new EntityInstance(entity, UUID.fromString(internalId)); + for (String fieldName : entity.getFieldNames()) { + String value = resultSet.getString(fieldName); + if (value != null) { + instance.overrideValue(fieldName, value); + } + } + collection.addInstance(instance); + } + } catch (SQLException e) { + throw new IllegalStateException("Could not load SQLite entity rows for " + entity.getName(), e); + } + } + + private void loadRelationshipRows() { + Map instances = instancesByInternalId(); + for (RelationshipVectorDefinition vector : relationshipVectors()) { + String sql = "SELECT from_internal_id, to_internal_id FROM " + relationshipTable(vector); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + while (resultSet.next()) { + EntityInstance from = instances.get(resultSet.getString("from_internal_id")); + EntityInstance to = instances.get(resultSet.getString("to_internal_id")); + if (from != null && to != null) { + from.getRelationships().connect(vector.getName(), to); + } + } + } catch (SQLException e) { + throw new IllegalStateException("Could not load SQLite relationship rows", e); + } + } + } + + private void restoreCountersFor(final EntityDefinition entity) { + EntityInstanceCollection collection = getInstanceCollectionForEntityNamed(entity.getName()); + String sql = "SELECT field_name, next_value FROM " + identifier(COUNTERS_TABLE) + + " WHERE entity_name = ?"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setString(1, entity.getName()); + try (ResultSet resultSet = statement.executeQuery()) { + while (resultSet.next()) { + AutoIncrement counter = collection.getCounters().get(resultSet.getString("field_name")); + if (counter != null) { + int nextValue = resultSet.getInt("next_value"); + if (counter.getCurrentValue() < nextValue) { + counter.incrementToNextAbove(nextValue - 1); + } + } + } + } + } catch (SQLException e) { + throw new IllegalStateException("Could not restore SQLite counters for " + entity.getName(), e); + } + } + + private void persistInstance(final EntityInstance instance) { + EntityDefinition entity = instance.getEntity(); + List columns = new ArrayList<>(); + columns.add(INTERNAL_ID_COLUMN); + columns.addAll(entity.getFieldNames()); + + StringBuilder names = new StringBuilder(); + StringBuilder placeholders = new StringBuilder(); + String separator = ""; + for (String column : columns) { + names.append(separator).append(identifier(column)); + placeholders.append(separator).append("?"); + separator = ", "; + } + + String sql = "INSERT OR REPLACE INTO " + table(entity) + + " (" + names + ") VALUES (" + placeholders + ")"; + + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setString(1, instance.getInternalId()); + int parameterIndex = 2; + for (String fieldName : entity.getFieldNames()) { + Field field = entity.getField(fieldName); + FieldValue value = instance.getFieldValue(fieldName); + if (value == null) { + statement.setString(parameterIndex, null); + } else { + statement.setString(parameterIndex, field.getActualValueToAdd(value)); + } + parameterIndex++; + } + statement.executeUpdate(); + } catch (SQLException e) { + throw new IllegalStateException("Could not persist SQLite instance", e); + } + } + + private void persistCountersFor(final EntityDefinition entity) { + executeSql("DELETE FROM " + identifier(COUNTERS_TABLE) + + " WHERE entity_name = " + quotedValue(entity.getName())); + + String sql = "INSERT INTO " + identifier(COUNTERS_TABLE) + + " (entity_name, field_name, next_value) VALUES (?, ?, ?)"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + for (AutoIncrement counter : countersFor(entity).values()) { + statement.setString(1, entity.getName()); + statement.setString(2, counter.getName()); + statement.setInt(3, counter.getCurrentValue()); + statement.executeUpdate(); + } + } catch (SQLException e) { + throw new IllegalStateException("Could not persist SQLite counters for " + entity.getName(), e); + } + } + + private void deletePersistedInstance(final String internalId) { + for (EntityDefinition entity : schema.getEntityDefinitions()) { + executeSql("DELETE FROM " + table(entity) + + " WHERE " + identifier(INTERNAL_ID_COLUMN) + " = " + quotedValue(internalId)); + } + for (RelationshipVectorDefinition vector : relationshipVectors()) { + executeSql("DELETE FROM " + relationshipTable(vector) + + " WHERE from_internal_id = " + quotedValue(internalId) + + " OR to_internal_id = " + quotedValue(internalId)); + } + } + + private void replaceAllRelationshipRowsFromCache() { + clearRelationshipRows(); + Set persisted = new HashSet<>(); + + String sql = "INSERT OR IGNORE INTO %s (from_internal_id, to_internal_id) VALUES (?, ?)"; + for (EntityInstance instance : instancesByInternalId().values()) { + for (RelationshipVectorInstance relationship : instance.getRelationships().getRelationshipInstances()) { + RelationshipVectorDefinition vector = relationship.getDefinition(); + String key = vector.getName() + "|" + + relationship.getFrom().getInternalId() + "|" + + relationship.getTo().getInternalId(); + if (!persisted.add(key)) { + continue; + } + try (PreparedStatement statement = connection.prepareStatement( + String.format(sql, relationshipTable(vector)))) { + statement.setString(1, relationship.getFrom().getInternalId()); + statement.setString(2, relationship.getTo().getInternalId()); + statement.executeUpdate(); + } catch (SQLException e) { + throw new IllegalStateException("Could not persist SQLite relationship", e); + } + } + } + } + + private void clearRelationshipRows() { + for (RelationshipVectorDefinition vector : relationshipVectors()) { + executeSql("DELETE FROM " + relationshipTable(vector)); + } + } + + private Set currentInternalIds() { + return instancesByInternalId().keySet(); + } + + private Map instancesByInternalId() { + Map instances = new HashMap<>(); + for (EntityInstanceCollection collection : getAllInstanceCollections()) { + for (EntityInstance instance : collection.getInstances()) { + instances.put(instance.getInternalId(), instance); + } + } + return instances; + } + + private List relationshipVectors() { + List vectors = new ArrayList<>(); + if (schema == null) { + return vectors; + } + for (RelationshipDefinition relationship : schema.getRelationships()) { + vectors.add(relationship.getFromRelationship()); + if (relationship.isTwoWay()) { + vectors.add(relationship.getReversedRelationship()); + } + } + return vectors; + } + + private Set columnNames(final String tableName) { + Set names = new HashSet<>(); + String sql = "PRAGMA table_info(" + identifier(tableName) + ")"; + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + while (resultSet.next()) { + names.add(resultSet.getString("name")); + } + } catch (SQLException e) { + throw new IllegalStateException("Could not read SQLite table info for " + tableName, e); + } + return names; + } + + private void executeSql(final String sql) { + try (Statement statement = connection.createStatement()) { + statement.executeUpdate(sql); + } catch (SQLException e) { + throw new IllegalStateException("Could not execute SQLite SQL: " + sql, e); + } + } + + private String sqlType(final FieldType type) { + switch (type) { + case AUTO_INCREMENT: + case INTEGER: + return "INTEGER"; + case FLOAT: + return "REAL"; + case BOOLEAN: + case AUTO_GUID: + case DATE: + case ENUM: + case OBJECT: + case STRING: + default: + return "TEXT"; + } + } + + private String table(final EntityDefinition entity) { + return identifier(entityTableName(entity)); + } + + private String entityTableName(final EntityDefinition entity) { + return "thing_" + entity.getName(); + } + + private String relationshipTable(final RelationshipVectorDefinition vector) { + return identifier( + "thing_rel_" + + vector.getFrom().getName() + "_" + + vector.getName() + "_" + + vector.getTo().getName()); + } + + private String identifier(final String rawIdentifier) { + return "\"" + rawIdentifier.replace("\"", "\"\"") + "\""; + } + + private String quotedValue(final String value) { + return "'" + value.replace("'", "''") + "'"; + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepositoryProvider.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepositoryProvider.java new file mode 100644 index 00000000..ca996af4 --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepositoryProvider.java @@ -0,0 +1,98 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import uk.co.compendiumdev.thingifier.core.EntityRelModel; +import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +public class SqliteThingRepositoryProvider implements ThingRepositoryProvider { + + private final Map repositories; + private final Function jdbcUrlFactory; + + private SqliteThingRepositoryProvider(final Function jdbcUrlFactory) { + this.repositories = new HashMap<>(); + this.jdbcUrlFactory = jdbcUrlFactory; + repositories.put( + EntityRelModel.DEFAULT_DATABASE_NAME, + new SqliteThingRepository( + EntityRelModel.DEFAULT_DATABASE_NAME, + jdbcUrlFactory.apply(EntityRelModel.DEFAULT_DATABASE_NAME))); + } + + public static SqliteThingRepositoryProvider inMemory() { + return new SqliteThingRepositoryProvider( + databaseKey -> "jdbc:sqlite:file:" + safeName(databaseKey) + "?mode=memory&cache=shared"); + } + + public static SqliteThingRepositoryProvider fileBacked(final Path directory) { + try { + Files.createDirectories(directory); + } catch (IOException e) { + throw new IllegalStateException("Could not create SQLite repository directory " + directory, e); + } + + return new SqliteThingRepositoryProvider(databaseKey -> { + Path databasePath = directory.resolve(safeName(databaseKey) + ".sqlite"); + return "jdbc:sqlite:" + databasePath.toAbsolutePath().toString().replace("\\", "/"); + }); + } + + @Override + public ThingRepository getDefaultRepository() { + return repositories.get(EntityRelModel.DEFAULT_DATABASE_NAME); + } + + @Override + public ThingRepository getRepository(final String databaseKey) { + return repositories.get(databaseKey); + } + + @Override + public Set getRepositoryNames() { + return new HashSet<>(repositories.keySet()); + } + + @Override + public ThingRepository createRepository(final String databaseKey, final ERSchema schema) { + if (repositories.containsKey(databaseKey)) { + throw new IllegalStateException("ERM Database Already Exists with name " + databaseKey); + } + + ThingRepository repository = new SqliteThingRepository(databaseKey, jdbcUrlFactory.apply(databaseKey)); + repository.initializeFrom(schema); + repositories.put(databaseKey, repository); + return repository; + } + + @Override + public boolean createRepositoryIfNotExisting(final String databaseKey, final ERSchema schema) { + if (repositories.containsKey(databaseKey)) { + return false; + } + + ThingRepository repository = new SqliteThingRepository(databaseKey, jdbcUrlFactory.apply(databaseKey)); + repository.initializeFrom(schema); + repositories.put(databaseKey, repository); + return true; + } + + @Override + public void deleteRepository(final String databaseKey) { + ThingRepository repository = repositories.remove(databaseKey); + if (repository != null) { + repository.close(); + } + } + + private static String safeName(final String databaseKey) { + return databaseKey.replaceAll("[^A-Za-z0-9._-]", "_"); + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java new file mode 100644 index 00000000..69c921cc --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java @@ -0,0 +1,75 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.instance.NamedValue; +import uk.co.compendiumdev.thingifier.core.domain.instances.AutoIncrement; +import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; +import uk.co.compendiumdev.thingifier.core.reporting.ValidationReport; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public interface ThingRepository extends AutoCloseable { + + String databaseKey(); + + void initializeFrom(ERSchema schema); + + void refreshSchema(ERSchema schema); + + ERInstanceData getInstanceData(); + + EntityInstanceCollection createInstanceCollectionFor(EntityDefinition definition); + + void createInstanceCollectionsFrom(ERSchema schema); + + List getAllInstanceCollections(); + + EntityInstanceCollection getInstanceCollectionForEntityNamed(String entityName); + + EntityInstance findEntityInstanceByGUID(String thingGUID); + + EntityInstance findInstanceByPrimaryKey(EntityDefinition entity, String primaryKeyValue); + + EntityInstance findInstanceByFieldNameAndValue(EntityDefinition entity, String fieldName, String fieldValue); + + Collection listInstances(EntityDefinition entity); + + EntityInstance addInstance(EntityInstance instance); + + EntityInstance updateInstance(EntityInstance instance); + + void deleteEntityInstance(EntityInstance instance); + + void clearAllData(); + + void clearInstanceDataFor(String entityName); + + ValidationReport checkFieldsForUniqueNess(EntityInstance instance, boolean isAmendment); + + Map countersFor(EntityDefinition entity); + + void setNextIdCountersToAccomodate(EntityDefinition entity, List fieldValues); + + void connectRelationship(EntityInstance from, String relationshipName, EntityInstance to); + + List removeRelationshipsInvolving( + EntityInstance parent, EntityInstance child, String relationshipName); + + List removeAllRelationships(EntityInstance instance); + + Collection getConnectedItems(EntityInstance instance, String relationshipName); + + default void flush() { + // Allows persistent repositories to sync legacy direct collection mutations. + } + + @Override + default void close() { + // Most repository implementations do not own closeable resources. + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProvider.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProvider.java new file mode 100644 index 00000000..c2721a0b --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProvider.java @@ -0,0 +1,27 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; + +import java.util.Set; + +public interface ThingRepositoryProvider extends AutoCloseable { + + ThingRepository getDefaultRepository(); + + ThingRepository getRepository(String databaseKey); + + Set getRepositoryNames(); + + ThingRepository createRepository(String databaseKey, ERSchema schema); + + boolean createRepositoryIfNotExisting(String databaseKey, ERSchema schema); + + void deleteRepository(String databaseKey); + + @Override + default void close() { + for (String databaseKey : getRepositoryNames()) { + getRepository(databaseKey).close(); + } + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProviderConfig.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProviderConfig.java new file mode 100644 index 00000000..b9c55e4b --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProviderConfig.java @@ -0,0 +1,131 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import java.nio.file.Path; +import java.nio.file.Paths; + +public class ThingRepositoryProviderConfig { + + public static final String DEFAULT_REPOSITORY_MODE = "memory"; + public static final String ARG_REPOSITORY_MODE = "-thingifier-repository"; + public static final String ARG_SQLITE_DIRECTORY = "-thingifier-sqlite-directory"; + public static final String ARG_SQLITE_MEMORY = "-sqlite-memory"; + public static final String ENV_REPOSITORY_MODE = "THINGIFIER_REPOSITORY"; + public static final String ENV_SQLITE_DIRECTORY = "THINGIFIER_SQLITE_DIRECTORY"; + public static final String PROPERTY_REPOSITORY_MODE = "thingifier.repository"; + public static final String PROPERTY_SQLITE_DIRECTORY = "thingifier.sqlite.directory"; + + private final String repositoryMode; + private final Path sqliteDirectory; + + public ThingRepositoryProviderConfig(final String repositoryMode, final Path sqliteDirectory) { + this.repositoryMode = normalize(repositoryMode); + this.sqliteDirectory = sqliteDirectory; + } + + public static ThingRepositoryProviderConfig fromArgs(final String[] args) { + String repositoryMode; + if (hasArg(args, ARG_SQLITE_MEMORY)) { + repositoryMode = "sqlite-memory"; + } else { + repositoryMode = firstNonBlank( + argValue(args, ARG_REPOSITORY_MODE), + System.getProperty(PROPERTY_REPOSITORY_MODE), + System.getenv(ENV_REPOSITORY_MODE), + DEFAULT_REPOSITORY_MODE); + } + + String sqliteDirectory = firstNonBlank( + argValue(args, ARG_SQLITE_DIRECTORY), + System.getProperty(PROPERTY_SQLITE_DIRECTORY), + System.getenv(ENV_SQLITE_DIRECTORY), + "thingifier-sqlite"); + + return new ThingRepositoryProviderConfig(repositoryMode, Paths.get(sqliteDirectory)); + } + + public ThingRepositoryProvider createProvider() { + switch (repositoryMode) { + case "memory": + case "in-memory": + case "custom-memory": + return new InMemoryThingRepositoryProvider(); + case "sqlite": + case "sqlite-memory": + case "sqlite-in-memory": + return SqliteThingRepositoryProvider.inMemory(); + case "sqlite-file": + case "sqlite-disk": + case "file": + return SqliteThingRepositoryProvider.fileBacked(sqliteDirectory); + default: + throw new IllegalArgumentException( + "Unknown Thingifier repository mode " + repositoryMode + + ". Expected memory, sqlite-memory, or sqlite-file."); + } + } + + public String getRepositoryMode() { + return repositoryMode; + } + + public Path getSqliteDirectory() { + return sqliteDirectory; + } + + public String describe() { + if (repositoryMode.equals("sqlite-file") || + repositoryMode.equals("sqlite-disk") || + repositoryMode.equals("file")) { + return repositoryMode + " at " + sqliteDirectory.toAbsolutePath(); + } + return repositoryMode; + } + + private static String argValue(final String[] args, final String argName) { + if (args == null) { + return null; + } + + for (String arg : args) { + if (arg == null) { + continue; + } + if (arg.equalsIgnoreCase(argName)) { + return ""; + } + if (arg.toLowerCase().startsWith(argName.toLowerCase() + "=")) { + return arg.substring(arg.indexOf("=") + 1).trim(); + } + } + return null; + } + + private static boolean hasArg(final String[] args, final String argName) { + if (args == null) { + return false; + } + + for (String arg : args) { + if (arg != null && arg.equalsIgnoreCase(argName)) { + return true; + } + } + return false; + } + + private static String firstNonBlank(final String... values) { + for (String value : values) { + if (value != null && !value.trim().isEmpty()) { + return value.trim(); + } + } + return ""; + } + + private static String normalize(final String mode) { + return firstNonBlank(mode, DEFAULT_REPOSITORY_MODE). + trim(). + toLowerCase(). + replace("_", "-"); + } +} diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java new file mode 100644 index 00000000..8ee6e3d0 --- /dev/null +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java @@ -0,0 +1,188 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import uk.co.compendiumdev.thingifier.core.EntityRelModel; +import uk.co.compendiumdev.thingifier.core.domain.definitions.Cardinality; +import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; + +import java.nio.file.Path; + +public class ThingRepositoryContractTest { + + @TempDir + Path tempDir; + + @Test + public void inMemoryRepositorySupportsTheContract() { + ThingRepository repository = new InMemoryThingRepository(EntityRelModel.DEFAULT_DATABASE_NAME); + + exerciseRepositoryContract(repository); + } + + @Test + public void sqliteInMemoryRepositorySupportsTheContract() { + try (ThingRepository repository = SqliteThingRepository.inMemory(EntityRelModel.DEFAULT_DATABASE_NAME)) { + exerciseRepositoryContract(repository); + } + } + + @Test + public void inMemoryProviderKeepsLogicalDatabasesSeparate() { + ThingRepositoryProvider provider = new InMemoryThingRepositoryProvider(); + + exerciseProviderIsolation(provider); + } + + @Test + public void sqliteProviderKeepsLogicalDatabasesSeparate() { + try (ThingRepositoryProvider provider = SqliteThingRepositoryProvider.inMemory()) { + exerciseProviderIsolation(provider); + } + } + + @Test + public void sqliteFileBackedRepositorySurvivesCloseAndReopen() { + ERSchema schema = todoSchema(); + Path databasePath = tempDir.resolve("thingifier.sqlite"); + + try (ThingRepository repository = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + repository.initializeFrom(schema); + + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + EntityDefinition taskDefinition = schema.getEntityDefinitionNamed("task"); + EntityInstance project = new EntityInstance(projectDefinition).setValue("title", "Persisted"); + EntityInstance task = new EntityInstance(taskDefinition).setValue("title", "Loaded relationship"); + + repository.addInstance(project); + repository.addInstance(task); + repository.connectRelationship(task, "task-of", project); + } + + try (ThingRepository reopened = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + reopened.initializeFrom(schema); + + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + EntityDefinition taskDefinition = schema.getEntityDefinitionNamed("task"); + + EntityInstance project = reopened.findInstanceByPrimaryKey(projectDefinition, "1"); + EntityInstance task = reopened.findInstanceByFieldNameAndValue( + taskDefinition, "title", "Loaded relationship"); + + Assertions.assertNotNull(project); + Assertions.assertNotNull(task); + Assertions.assertEquals("Persisted", project.getFieldValue("title").asString()); + Assertions.assertTrue(reopened.getConnectedItems(task, "task-of").contains(project)); + Assertions.assertTrue(reopened.getConnectedItems(project, "tasks").contains(task)); + } + } + + @Test + public void sqliteRepositoryQuotesGeneratedSqlIdentifiers() { + ERSchema schema = new ERSchema(); + EntityDefinition select = schema.defineEntity("select", "selects", -1); + select.addAsPrimaryKeyField(Field.is("key-id", FieldType.AUTO_GUID)); + select.addField(Field.is("field with space", FieldType.STRING)); + + try (ThingRepository repository = SqliteThingRepository.inMemory("quoted")) { + repository.initializeFrom(schema); + + EntityInstance instance = new EntityInstance(select); + instance.setValue("field with space", "works"); + repository.addInstance(instance); + + EntityInstance found = repository.findInstanceByFieldNameAndValue( + select, "field with space", "works"); + + Assertions.assertNotNull(found); + Assertions.assertEquals("works", found.getFieldValue("field with space").asString()); + } + } + + private void exerciseRepositoryContract(final ThingRepository repository) { + ERSchema schema = todoSchema(); + repository.initializeFrom(schema); + + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + EntityDefinition taskDefinition = schema.getEntityDefinitionNamed("task"); + + EntityInstance project = new EntityInstance(projectDefinition); + project.setValue("title", "Repository project"); + + repository.addInstance(project); + + Assertions.assertEquals("1", project.getPrimaryKeyValue()); + Assertions.assertEquals(project, repository.findInstanceByPrimaryKey(projectDefinition, "1")); + Assertions.assertEquals(project, repository.findInstanceByFieldNameAndValue( + projectDefinition, "title", "Repository project")); + Assertions.assertEquals(1, repository.listInstances(projectDefinition).size()); + + EntityInstance duplicate = new EntityInstance(projectDefinition); + duplicate.setValue("title", "Repository project"); + Assertions.assertFalse(repository.checkFieldsForUniqueNess(duplicate, false).isValid()); + + EntityInstance task = new EntityInstance(taskDefinition); + task.setValue("title", "Wire repository"); + repository.addInstance(task); + repository.connectRelationship(task, "task-of", project); + + Assertions.assertTrue(repository.getConnectedItems(task, "task-of").contains(project)); + Assertions.assertTrue(repository.getConnectedItems(project, "tasks").contains(task)); + + repository.removeRelationshipsInvolving(task, project, "task-of"); + + Assertions.assertTrue(repository.getConnectedItems(task, "task-of").isEmpty()); + Assertions.assertTrue(repository.getConnectedItems(project, "tasks").isEmpty()); + + repository.deleteEntityInstance(task); + + Assertions.assertEquals(0, repository.listInstances(taskDefinition).size()); + + repository.clearAllData(); + + Assertions.assertEquals(0, repository.listInstances(projectDefinition).size()); + Assertions.assertEquals(0, repository.listInstances(taskDefinition).size()); + } + + private void exerciseProviderIsolation(final ThingRepositoryProvider provider) { + ERSchema schema = todoSchema(); + ThingRepository defaultRepository = provider.getDefaultRepository(); + defaultRepository.initializeFrom(schema); + provider.createRepository("session-one", schema); + + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + EntityInstance project = new EntityInstance(projectDefinition); + project.setValue("title", "Only default"); + + defaultRepository.addInstance(project); + + Assertions.assertEquals(1, defaultRepository.listInstances(projectDefinition).size()); + Assertions.assertEquals( + 0, + provider.getRepository("session-one").listInstances(projectDefinition).size()); + } + + private ERSchema todoSchema() { + ERSchema schema = new ERSchema(); + + EntityDefinition project = schema.defineEntity("project", "projects", -1); + project.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + project.addField(Field.is("title", FieldType.STRING).setMustBeUnique(true)); + + EntityDefinition task = schema.defineEntity("task", "tasks", -1); + task.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + task.addField(Field.is("title", FieldType.STRING)); + + schema.defineRelationship(task, project, "task-of", Cardinality.ONE_TO_ONE()). + whenReversed(Cardinality.ONE_TO_MANY(), "tasks"); + + return schema; + } +} diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProviderConfigTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProviderConfigTest.java new file mode 100644 index 00000000..e4a33ed3 --- /dev/null +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryProviderConfigTest.java @@ -0,0 +1,64 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; + +public class ThingRepositoryProviderConfigTest { + + @TempDir + Path tempDir; + + @Test + public void defaultsToCurrentInMemoryRepository() { + ThingRepositoryProviderConfig config = ThingRepositoryProviderConfig.fromArgs(new String[]{}); + + Assertions.assertEquals("memory", config.getRepositoryMode()); + Assertions.assertTrue(config.createProvider() instanceof InMemoryThingRepositoryProvider); + } + + @Test + public void canCreateSqliteInMemoryRepositoryFromArgs() { + ThingRepositoryProviderConfig config = ThingRepositoryProviderConfig.fromArgs( + new String[]{"-thingifier-repository=sqlite-memory"}); + + Assertions.assertEquals("sqlite-memory", config.getRepositoryMode()); + Assertions.assertTrue(config.createProvider() instanceof SqliteThingRepositoryProvider); + } + + @Test + public void canCreateSqliteInMemoryRepositoryFromChallengeMainShortcutArg() { + ThingRepositoryProviderConfig config = ThingRepositoryProviderConfig.fromArgs( + new String[]{"-sqlite-memory"}); + + Assertions.assertEquals("sqlite-memory", config.getRepositoryMode()); + Assertions.assertTrue(config.createProvider() instanceof SqliteThingRepositoryProvider); + } + + @Test + public void canCreateSqliteFileRepositoryFromArgs() { + ThingRepositoryProviderConfig config = ThingRepositoryProviderConfig.fromArgs( + new String[]{ + "-thingifier-repository=sqlite-file", + "-thingifier-sqlite-directory=" + tempDir + }); + + Assertions.assertEquals("sqlite-file", config.getRepositoryMode()); + Assertions.assertEquals(tempDir, config.getSqliteDirectory()); + Assertions.assertTrue(config.createProvider() instanceof SqliteThingRepositoryProvider); + } + + @Test + public void rejectsUnknownRepositoryModes() { + ThingRepositoryProviderConfig config = ThingRepositoryProviderConfig.fromArgs( + new String[]{"-thingifier-repository=unknown"}); + + IllegalArgumentException thrown = Assertions.assertThrows( + IllegalArgumentException.class, + config::createProvider); + + Assertions.assertTrue(thrown.getMessage().contains("Unknown Thingifier repository mode")); + } +} diff --git a/pom.xml b/pom.xml index 9641cdb1..d73cb2c2 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ 1.5.6-SNAPSHOT 2.2.27 2.8.6 + 3.53.2.0 5.6.2 16 5.5.0 @@ -157,4 +158,4 @@ Challenger } ] ~~~~~~~~ - --> \ No newline at end of file + --> diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java index f6482ef9..922603a3 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java @@ -12,6 +12,7 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.*; import uk.co.compendiumdev.thingifier.core.domain.definitions.relationship.RelationshipDefinition; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; import uk.co.compendiumdev.thingifier.reporting.ThingReporter; import java.util.*; @@ -34,7 +35,11 @@ public final class Thingifier { private final ThingifierApiConfigProfiles apiConfigProfiles; public Thingifier(){ - erm = new EntityRelModel(); + this(new EntityRelModel()); + } + + public Thingifier(final EntityRelModel erm) { + this.erm = erm; title = ""; initialParagraph = ""; apiConfig = new ThingifierApiConfig(""); @@ -120,17 +125,17 @@ public boolean hasRelationshipNamed(final String relationshipName) { // Instances public List getThings(final String database) { - return erm.getInstanceData(database).getAllInstanceCollections(); + return erm.getRepository(database).getAllInstanceCollections(); } public EntityInstance findThingInstanceByGuid(final String thingGUID, final String database) { - return erm.getInstanceData(database).findEntityInstanceByGUID(thingGUID); + return erm.getRepository(database).findEntityInstanceByGUID(thingGUID); } public EntityInstanceCollection getThingInstancesNamed(final String aName, final String database) { - return erm.getInstanceData(database).getInstanceCollectionForEntityNamed(aName); + return erm.getRepository(database).getInstanceCollectionForEntityNamed(aName); } @@ -139,7 +144,7 @@ public EntityInstanceCollection getInstancesForSingularOrPluralNamedEntity(final final EntityDefinition defn = erm.getSchema().getDefinitionWithSingularOrPluralNamed(term); if(defn!=null){ final String entityName = defn.getName(); - return erm.getInstanceData(database).getInstanceCollectionForEntityNamed(entityName); + return erm.getRepository(database).getInstanceCollectionForEntityNamed(entityName); } return null; @@ -157,18 +162,20 @@ public void clearAllData() { } public void clearAllData(final String database) { - erm.getInstanceData(database).clearAllData(); + erm.getRepository(database).clearAllData(); } public void deleteThing(final EntityInstance aThingInstance, final String database) { - erm.getInstanceData(database).deleteEntityInstance(aThingInstance); + erm.getRepository(database).deleteEntityInstance(aThingInstance); } // data generation public void generateData(final String database) { if(dataPopulator!=null){ + erm.getRepository(database).refreshSchema(erm.getSchema()); dataPopulator.populate(erm.getSchema(), erm.getInstanceData(database)); + erm.getRepository(database).flush(); } } @@ -217,6 +224,10 @@ public EntityRelModel getERmodel() { return erm; } + public ThingRepository getRepository(final String database) { + return erm.getRepository(database); + } + /* TODO: these are documentation methods, why are they not in the @@ -256,11 +267,13 @@ public void ensureCreatedAndPopulatedInstanceDatabaseNamed(String databaseName) // if we created it then populate it if(getDefaultDataPopulator()!=null){ // Use any default data populator to populate the new database + getERmodel().getRepository(databaseName).refreshSchema(getERmodel().getSchema()); getDefaultDataPopulator(). populate( getERmodel().getSchema(), - getERmodel().getInstanceData(databaseName) + getERmodel().getRepository(databaseName).getInstanceData() ); + getERmodel().getRepository(databaseName).flush(); } } } @@ -270,8 +283,9 @@ public void ensureCreatedAndPopulatedInstanceDatabaseFromJson(String databaseNam new JsonPopulator(jsonDatabaseContents).populate( getERmodel().getSchema(), - getERmodel().getInstanceData(databaseName) + getERmodel().getRepository(databaseName).getInstanceData() ); + getERmodel().getRepository(databaseName).flush(); } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreation.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreation.java index 3faded3a..9ec26028 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreation.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreation.java @@ -131,7 +131,8 @@ public ApiResponse create(final String url, final BodyParser bodyargs, final Sim return response; } - connectThis.getRelationships().connect(relationshipToUse.getName(), relatedItem); + thingifier.getRepository(database).connectRelationship( + connectThis, relationshipToUse.getName(), relatedItem); // enforce cardinality on relationship ValidationReport validNow = relatedItem.validateRelationships(); diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreator.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreator.java index a7c8d4f6..a4dfa8f2 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreator.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreator.java @@ -19,7 +19,8 @@ public ApiResponse createRelationships(final BodyParser bodyargs, final EntityIn try { List relationships = getRelationshipsFromArgs(bodyargs, instance, database); for (RelationshipDetails relationship : relationships) { - instance.getRelationships().connect( + thingifier.getRepository(database).connectRelationship( + instance, relationship.relationshipName, thingifier.getInstancesForSingularOrPluralNamedEntity(relationship.toType, database). findInstanceByFieldNameAndValue(relationship.guidName, relationship.guidValue)); diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiDeleteHandler.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiDeleteHandler.java index b1cb0b80..8e13f798 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiDeleteHandler.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiDeleteHandler.java @@ -35,7 +35,8 @@ public ApiResponse handle(final String url, HttpHeadersBlock requestHeaders) { EntityInstance child = queryResult.getLastInstance(); // todo: this returns a list of 'items' to be removed based on // removal of the relationships, these should really be deleted from thingifier now - parent.getRelationships().removeRelationshipsInvolving(child, queryResult.getLastRelationshipName()); + thingifier.getRepository(instanceDatabaseName). + removeRelationshipsInvolving(parent, child, queryResult.getLastRelationshipName()); } else { List items = queryResult.getListEntityInstances(); if (items.isEmpty()) { diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPostHandler.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPostHandler.java index 821322d3..3d94161f 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPostHandler.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPostHandler.java @@ -47,7 +47,7 @@ public ApiResponse handle(final String url, final BodyParser args, final HttpHea if (validity.isValid()) { return response; } else { - instancesCollection.deleteInstance(response.getReturnedInstance()); + thingifier.deleteThing(response.getReturnedInstance(), instanceDatabaseName); return ApiResponse.error(400, validity.getErrorMessages()).addToErrorMessages("No new item created"); } } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingAmendment.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingAmendment.java index 04f8ca49..7b323185 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingAmendment.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingAmendment.java @@ -66,8 +66,7 @@ public ApiResponse amendInstance(final BodyParser bodyargs, final EntityInstance ValidationReport relationshipsValidation = new BodyRelationshipValidator(thingifier).validate(bodyargs, cloned.getEntity(), database); validation.combine(relationshipsValidation); - ValidationReport uniquenessCheck = thingifier.getERmodel().getInstanceData(database). - getInstanceCollectionForEntityNamed(instance.getEntity().getName()). + ValidationReport uniquenessCheck = thingifier.getRepository(database). checkFieldsForUniqueNess(cloned, true); validation.combine(uniquenessCheck); @@ -76,7 +75,7 @@ public ApiResponse amendInstance(final BodyParser bodyargs, final EntityInstance instance.clearAllFields(); // delete all existing relationships for idempotent amend // todo: this returns a list of 'items' to be removed based on relationship - instance.getRelationships().removeAllRelationships(); + thingifier.getRepository(database).removeAllRelationships(instance); } List fieldValues = FieldValues. fromListMapEntryStringString( @@ -85,6 +84,8 @@ public ApiResponse amendInstance(final BodyParser bodyargs, final EntityInstance new EntityInstanceBulkUpdater(instance).setFieldValuesFrom(fieldValues); + thingifier.getRepository(database).updateInstance(instance); + // todo: should we check that this was actually a success? final ApiResponse relresponse = new RelationshipCreator(thingifier).createRelationships(bodyargs, instance, database); // todo: should check if any of the 'removed items due to relationship removal' need to be removed diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingCreation.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingCreation.java index 8aaa1c1b..d3ad8f4a 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingCreation.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingCreation.java @@ -36,7 +36,8 @@ public ApiResponse with(final BodyParser bodyargs, final EntityInstanceCollectio // todo: separate validation for creation of 'cannot' create with ID, or cannot create with GUID EntityInstance instance = new EntityInstance(thing.definition()); instance.addAutoGUIDstoInstance(); - instance.addAutoIncrementIdsToInstance(thing.getCounters()); + instance.addAutoIncrementIdsToInstance( + thingifier.getRepository(database).countersFor(thing.definition())); return addNewThingWithFields(bodyargs, instance, thing, database); } @@ -59,7 +60,8 @@ public ApiResponse withPrimaryKey(final String primaryKey, final BodyParser body instance.overrideValue(thing.definition().getPrimaryKeyField().getName(), primaryKey); // TODO: should we really do this for a PUT when everything needs to exist, suspect this is a bug - instance.addAutoIncrementIdsToInstance(thing.getCounters()); + instance.addAutoIncrementIdsToInstance( + thingifier.getRepository(database).countersFor(thing.definition())); validated = new BodyRelationshipValidator(thingifier).validate(bodyargs, thing, database); @@ -76,7 +78,7 @@ public ApiResponse withPrimaryKey(final String primaryKey, final BodyParser body bodyargs.getFlattenedStringMap()); - thing.setNextIdCountersToAccomodate(fieldValues); + thingifier.getRepository(database).setNextIdCountersToAccomodate(thing.definition(), fieldValues); return insertNewThingWithFields(bodyargs, instance, thing, database); } @@ -148,14 +150,13 @@ private ApiResponse insertNewThingWithFields(final BodyParser bodyargs, final En private ApiResponse addValidatedInstance(BodyParser bodyargs, EntityInstance instance, EntityInstanceCollection thing, String database, ValidationReport validation) { // check for uniqueness prior to adding - ValidationReport uniquenessCheck = thingifier.getERmodel().getInstanceData(database). - getInstanceCollectionForEntityNamed(instance.getEntity().getName()). + ValidationReport uniquenessCheck = thingifier.getRepository(database). checkFieldsForUniqueNess(instance, false); validation.combine(uniquenessCheck); if (validation.isValid()) { try { - thing.addInstance(instance); + thingifier.getRepository(database).addInstance(instance); }catch(Exception e){ return ApiResponse.error(400, e.getMessage()); } From b64c1f37e2ee3d2d3c151b0df7cca616b97e2a2b Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 13:00:46 +0100 Subject: [PATCH 02/11] Push simple repository reads into SQL --- .../ChallengeApiModelRepositoryTest.java | 112 ++++++- .../thingifier/core/query/QueryResult.java | 23 ++ .../query/RepositoryBackedSimpleQuery.java | 144 +++++++++ .../thingifier/core/query/SimpleQuery.java | 9 +- .../core/query/SortByFieldName.java | 8 + .../repository/InMemoryThingRepository.java | 44 +++ .../repository/SqliteThingRepository.java | 305 +++++++++++++++++- .../core/repository/ThingRepository.java | 5 + .../ThingRepositoryContractTest.java | 65 ++++ .../restapihandlers/RestApiGetHandler.java | 31 +- 10 files changed, 725 insertions(+), 21 deletions(-) create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/QueryResult.java create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/RepositoryBackedSimpleQuery.java diff --git a/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java b/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java index 3a98e0f3..218a2504 100644 --- a/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java +++ b/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java @@ -3,22 +3,116 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import uk.co.compendiumdev.thingifier.Thingifier; +import uk.co.compendiumdev.thingifier.api.http.headers.HttpHeadersBlock; +import uk.co.compendiumdev.thingifier.api.response.ApiResponse; +import uk.co.compendiumdev.thingifier.core.domain.definitions.Cardinality; import uk.co.compendiumdev.thingifier.core.EntityRelModel; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepository; import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepositoryProvider; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; public class ChallengeApiModelRepositoryTest { @Test public void challengeApiCanUseSqliteInMemoryRepository() { - Thingifier thingifier = new ChallengeApiModel().get(SqliteThingRepositoryProvider.inMemory()); - - Assertions.assertTrue( - thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME) - instanceof SqliteThingRepository); - Assertions.assertEquals( - 10, - thingifier.getThingInstancesNamed("todo", EntityRelModel.DEFAULT_DATABASE_NAME). - countInstances()); + try (SqliteThingRepositoryProvider provider = SqliteThingRepositoryProvider.inMemory()) { + Thingifier thingifier = new ChallengeApiModel().get(provider); + + Assertions.assertTrue( + thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME) + instanceof SqliteThingRepository); + Assertions.assertEquals( + 10, + thingifier.getThingInstancesNamed("todo", EntityRelModel.DEFAULT_DATABASE_NAME). + countInstances()); + } + } + + @Test + public void challengeApiGetUsesSqliteRepositoryReadsForSimpleTodoQueries() { + try (SqliteThingRepositoryProvider provider = SqliteThingRepositoryProvider.inMemory()) { + Thingifier thingifier = new ChallengeApiModel().get(provider); + ThingRepository repository = thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME); + EntityDefinition todo = thingifier.getDefinitionNamed("todo"); + + EntityInstance firstTodo = repository.findInstanceByQueryIdentifier(todo, "1"); + firstTodo.setValue("doneStatus", "true"); + repository.updateInstance(firstTodo); + + ApiResponse allTodos = thingifier.api().get( + "/todos", new QueryFilterParams(), new HttpHeadersBlock()); + Assertions.assertEquals(200, allTodos.getStatusCode()); + Assertions.assertEquals(10, allTodos.getReturnedInstanceCollection().size()); + + ApiResponse todoById = thingifier.api().get( + "/todos/1", new QueryFilterParams(), new HttpHeadersBlock()); + Assertions.assertEquals(200, todoById.getStatusCode()); + Assertions.assertEquals(1, todoById.getReturnedInstanceCollection().size()); + Assertions.assertEquals( + "1", + todoById.getReturnedInstanceCollection().get(0).getPrimaryKeyValue()); + + QueryFilterParams doneStatusFilter = new QueryFilterParams(); + doneStatusFilter.put("doneStatus", "false"); + ApiResponse notDoneTodos = thingifier.api().get( + "/todos", doneStatusFilter, new HttpHeadersBlock()); + Assertions.assertEquals(200, notDoneTodos.getStatusCode()); + Assertions.assertEquals(9, notDoneTodos.getReturnedInstanceCollection().size()); + + QueryFilterParams idRangeFilter = new QueryFilterParams(); + idRangeFilter.put("id", ">=2"); + ApiResponse idRangeTodos = thingifier.api().get( + "/todos", idRangeFilter, new HttpHeadersBlock()); + Assertions.assertEquals(200, idRangeTodos.getStatusCode()); + Assertions.assertEquals(9, idRangeTodos.getReturnedInstanceCollection().size()); + + QueryFilterParams sortByDescendingId = new QueryFilterParams(); + sortByDescendingId.put("sortBy", "-id"); + ApiResponse sortedTodos = thingifier.api().get( + "/todos", sortByDescendingId, new HttpHeadersBlock()); + Assertions.assertEquals(200, sortedTodos.getStatusCode()); + Assertions.assertEquals( + "10", + sortedTodos.getReturnedInstanceCollection().get(0).getPrimaryKeyValue()); + } + } + + @Test + public void relationshipGetRoutesStillFallBackToLegacyQueryTraversal() { + try (SqliteThingRepositoryProvider provider = SqliteThingRepositoryProvider.inMemory()) { + Thingifier thingifier = new Thingifier(new EntityRelModel(provider)); + + EntityDefinition project = thingifier.defineThing("project", "projects"); + project.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + project.addField(Field.is("title", FieldType.STRING)); + + EntityDefinition task = thingifier.defineThing("task", "tasks"); + task.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + task.addField(Field.is("title", FieldType.STRING)); + + thingifier.defineRelationship(project, task, "tasks", Cardinality.ONE_TO_MANY()). + whenReversed(Cardinality.ONE_TO_ONE(), "task-of"); + + ThingRepository repository = thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME); + EntityInstance projectInstance = new EntityInstance(project).setValue("title", "Office"); + EntityInstance taskInstance = new EntityInstance(task).setValue("title", "File paperwork"); + repository.addInstance(projectInstance); + repository.addInstance(taskInstance); + repository.connectRelationship(projectInstance, "tasks", taskInstance); + + ApiResponse response = thingifier.api().get( + "/project/1/tasks", new QueryFilterParams(), new HttpHeadersBlock()); + + Assertions.assertEquals(200, response.getStatusCode()); + Assertions.assertEquals(1, response.getReturnedInstanceCollection().size()); + Assertions.assertEquals( + "File paperwork", + response.getReturnedInstanceCollection().get(0).getFieldValue("title").asString()); + } } } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/QueryResult.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/QueryResult.java new file mode 100644 index 00000000..25fbbb81 --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/QueryResult.java @@ -0,0 +1,23 @@ +package uk.co.compendiumdev.thingifier.core.query; + +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; + +import java.util.List; + +public interface QueryResult { + + boolean wasQueryIntendedToMatchAnInstance(); + + boolean isResultACollection(); + + List getListEntityInstances(); + + EntityInstance getLastInstance(); + + boolean lastMatchWasInstance(); + + boolean lastMatchWasNothing(); + + EntityDefinition resultContainsDefn(); +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/RepositoryBackedSimpleQuery.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/RepositoryBackedSimpleQuery.java new file mode 100644 index 00000000..83721459 --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/RepositoryBackedSimpleQuery.java @@ -0,0 +1,144 @@ +package uk.co.compendiumdev.thingifier.core.query; + +import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; + +import java.util.ArrayList; +import java.util.List; + +public class RepositoryBackedSimpleQuery implements QueryResult { + + private final ERSchema schema; + private final ThingRepository repository; + private final String query; + + private boolean isCollection; + private boolean wasIntentToMatchInstance; + private boolean lastMatchWasInstance; + private boolean lastMatchWasNothing = true; + private EntityDefinition resultContainsDefinition; + private EntityInstance currentInstance; + private List foundItems = new ArrayList<>(); + + public RepositoryBackedSimpleQuery( + final ERSchema schema, + final ThingRepository repository, + final String query) { + this.schema = schema; + this.repository = repository; + if (query.startsWith("/")) { + this.query = query.substring(1); + } else { + this.query = query; + } + } + + public static boolean canHandle(final ERSchema schema, final String query) { + String[] terms = termsFrom(query); + if (terms.length == 0 || terms.length > 2) { + return false; + } + + EntityDefinition entity = entityForTerm(schema, terms[0]); + if (entity == null) { + return false; + } + + if (terms.length == 1) { + return true; + } + + String identifierCandidate = terms[1]; + return !schema.hasRelationshipNamed(identifierCandidate) && + entityForTerm(schema, identifierCandidate) == null; + } + + public RepositoryBackedSimpleQuery performQuery(final QueryFilterParams queryParams) { + String[] terms = termsFrom(query); + EntityDefinition entity = entityForTerm(schema, terms[0]); + resultContainsDefinition = entity; + + if (terms.length == 1) { + isCollection = true; + foundItems = new ArrayList<>(repository.listInstances(entity, queryParams)); + lastMatchWasNothing = false; + return this; + } + + wasIntentToMatchInstance = true; + currentInstance = repository.findInstanceByQueryIdentifier(entity, terms[1]); + if (currentInstance == null) { + foundItems = new ArrayList<>(); + lastMatchWasNothing = true; + lastMatchWasInstance = false; + return this; + } + + foundItems = new ArrayList<>(); + foundItems.add(currentInstance); + lastMatchWasNothing = false; + lastMatchWasInstance = true; + return this; + } + + @Override + public boolean wasQueryIntendedToMatchAnInstance() { + return wasIntentToMatchInstance; + } + + @Override + public boolean isResultACollection() { + return isCollection; + } + + @Override + public List getListEntityInstances() { + return new ArrayList<>(foundItems); + } + + @Override + public EntityInstance getLastInstance() { + return currentInstance; + } + + @Override + public boolean lastMatchWasInstance() { + return lastMatchWasInstance; + } + + @Override + public boolean lastMatchWasNothing() { + return lastMatchWasNothing; + } + + @Override + public EntityDefinition resultContainsDefn() { + return resultContainsDefinition; + } + + private static EntityDefinition entityForTerm(final ERSchema schema, final String term) { + if (schema.hasEntityNamed(term)) { + return schema.getEntityDefinitionNamed(term); + } + if (schema.hasEntityWithPluralNamed(term)) { + return schema.getEntityDefinitionWithPluralNamed(term); + } + return null; + } + + private static String[] termsFrom(final String query) { + String normalized = query == null ? "" : query.trim(); + while (normalized.startsWith("/")) { + normalized = normalized.substring(1); + } + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + if (normalized.isEmpty()) { + return new String[0]; + } + return normalized.split("/"); + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SimpleQuery.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SimpleQuery.java index 561cc051..53be9fbc 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SimpleQuery.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SimpleQuery.java @@ -25,7 +25,7 @@ Use the isResultACollection, wasIntentToMatchACollection, lastMatchWasInstance in the calling method. */ -public final class SimpleQuery { +public final class SimpleQuery implements QueryResult { private final ERInstanceData database; private final ERSchema schema; // all the definitions @@ -229,10 +229,12 @@ public SimpleQuery performQuery() { } // i.e. did the query end with an identifier which was a primary key + @Override public boolean wasQueryIntendedToMatchAnInstance(){ return wasIntentToMatchInstance; } + @Override public boolean isResultACollection() { return isCollection; } @@ -259,6 +261,7 @@ public SimpleQuery performQuery(final QueryFilterParams queryParams) { return this; } + @Override public List getListEntityInstances() { List returnThis = new ArrayList<>(); @@ -303,18 +306,22 @@ public boolean wasItemFoundUnderARelationship() { return foundItemsHistoryList.get(foundItemsHistoryList.size() - 2) instanceof RelationshipVectorDefinition; } + @Override public EntityInstance getLastInstance() { return currentInstance; } + @Override public boolean lastMatchWasInstance() { return lastMatch == CURRENT_INSTANCE; } + @Override public boolean lastMatchWasNothing() { return lastMatch == NOTHING; } + @Override public EntityDefinition resultContainsDefn() { return resultContainsDefinition; } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SortByFieldName.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SortByFieldName.java index 541dae46..c4fd773b 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SortByFieldName.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SortByFieldName.java @@ -4,6 +4,14 @@ public class SortByFieldName { int order = 1; String fieldName = ""; + public int getOrder() { + return order; + } + + public String getFieldName() { + return fieldName; + } + public static boolean isSortByParam(final String key) { return (key.equalsIgnoreCase("sortby") || key.equalsIgnoreCase("sort_by")); diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java index 3003170f..c97cfe41 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java @@ -2,13 +2,19 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.instance.NamedValue; import uk.co.compendiumdev.thingifier.core.domain.instances.AutoIncrement; import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; +import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListFilter; +import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListSorter; +import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; import uk.co.compendiumdev.thingifier.core.reporting.ValidationReport; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; @@ -107,6 +113,31 @@ public Collection listInstances(final EntityDefinition entity) { return collection.getInstances(); } + @Override + public List listInstances( + final EntityDefinition entity, final QueryFilterParams queryParams) { + List instances = new ArrayList<>(listInstances(entity)); + QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; + instances = new EntityInstanceListFilter(params).filter(instances); + return new EntityInstanceListSorter(params).sort(instances); + } + + @Override + public EntityInstance findInstanceByQueryIdentifier( + final EntityDefinition entity, final String identifier) { + EntityInstanceCollection collection = getInstanceCollectionForEntityNamed(entity.getName()); + if (collection == null) { + return null; + } + + for (EntityInstance instance : collection.getInstances()) { + if (matchesQueryIdentifier(instance, identifier)) { + return instance; + } + } + return null; + } + @Override public EntityInstance addInstance(final EntityInstance instance) { getInstanceCollectionForEntityNamed(instance.getEntity().getName()).addInstance(instance); @@ -176,4 +207,17 @@ public Collection getConnectedItems( final EntityInstance instance, final String relationshipName) { return instance.getRelationships().getConnectedItems(relationshipName); } + + private boolean matchesQueryIdentifier(final EntityInstance instance, final String identifier) { + for (Field autoIncrementField : instance.getEntity().getFieldsOfType(FieldType.AUTO_INCREMENT)) { + String idValue = instance.getFieldValue(autoIncrementField.getName()).asString(); + if (idValue.contentEquals(identifier)) { + return true; + } + break; + } + + String primaryKeyValue = instance.getPrimaryKeyValue(); + return primaryKeyValue != null && primaryKeyValue.contentEquals(identifier); + } } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java index 8c922f9a..e59d3535 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java @@ -12,6 +12,12 @@ import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; import uk.co.compendiumdev.thingifier.core.domain.instances.RelationshipVectorInstance; +import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListFilter; +import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListSorter; +import uk.co.compendiumdev.thingifier.core.query.EntityListSortParamParser; +import uk.co.compendiumdev.thingifier.core.query.FilterBy; +import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; +import uk.co.compendiumdev.thingifier.core.query.SortByFieldName; import uk.co.compendiumdev.thingifier.core.reporting.ValidationReport; import java.nio.file.Path; @@ -38,7 +44,7 @@ public class SqliteThingRepository extends InMemoryThingRepository { private final String jdbcUrl; private ERSchema schema; private Connection connection; - private boolean loadedExistingData; + private boolean legacySnapshotLoaded; public SqliteThingRepository(final String databaseKey, final String jdbcUrl) { super(databaseKey, new ERInstanceData()); @@ -60,10 +66,15 @@ public void initializeFrom(final ERSchema schema) { openConnection(); createMetadataTables(); refreshSchema(schema); - if (!loadedExistingData) { + } + + @Override + public ERInstanceData getInstanceData() { + if (!legacySnapshotLoaded) { loadExistingData(); - loadedExistingData = true; + legacySnapshotLoaded = true; } + return super.getInstanceData(); } @Override @@ -88,6 +99,105 @@ public EntityInstanceCollection createInstanceCollectionFor(final EntityDefiniti return collection; } + @Override + public EntityInstance findEntityInstanceByGUID(final String thingGUID) { + ensureSchemaReady(); + for (EntityDefinition entity : schema.getEntityDefinitions()) { + for (Field guidField : entity.getFieldsOfType(FieldType.AUTO_GUID)) { + EntityInstance instance = findInstanceByFieldNameAndValue( + entity, guidField.getName(), thingGUID); + if (instance != null) { + return instance; + } + } + } + return null; + } + + @Override + public EntityInstance findInstanceByPrimaryKey( + final EntityDefinition entity, final String primaryKeyValue) { + if (!entity.hasPrimaryKeyField()) { + return null; + } + return findInstanceByFieldNameAndValue( + entity, entity.getPrimaryKeyField().getName(), primaryKeyValue); + } + + @Override + public EntityInstance findInstanceByFieldNameAndValue( + final EntityDefinition entity, final String fieldName, final String fieldValue) { + ensureSchemaReady(); + if (!entity.hasFieldNameDefined(fieldName)) { + return null; + } + + String sql = "SELECT * FROM " + table(entity) + + " WHERE " + identifier(fieldName) + " = ? LIMIT 1"; + List instances = queryEntityRows(entity, sql, List.of(fieldValue)); + if (instances.isEmpty()) { + return null; + } + return instances.get(0); + } + + @Override + public Collection listInstances(final EntityDefinition entity) { + return listInstances(entity, new QueryFilterParams()); + } + + @Override + public List listInstances( + final EntityDefinition entity, final QueryFilterParams queryParams) { + ensureSchemaReady(); + QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; + if (hasJavaOnlyFilters(params)) { + List instances = new ArrayList<>(super.listInstances(entity)); + if (!legacySnapshotLoaded) { + instances = new ArrayList<>(getInstanceData(). + getInstanceCollectionForEntityNamed(entity.getName()). + getInstances()); + } + instances = new EntityInstanceListFilter(params).filter(instances); + return new EntityInstanceListSorter(params).sort(instances); + } + + SqlQuery sqlQuery = selectInstancesSql(entity, params); + return queryEntityRows(entity, sqlQuery.sql, sqlQuery.parameters); + } + + @Override + public EntityInstance findInstanceByQueryIdentifier( + final EntityDefinition entity, final String identifierValue) { + ensureSchemaReady(); + List clauses = new ArrayList<>(); + List parameters = new ArrayList<>(); + + for (Field autoIncrementField : entity.getFieldsOfType(FieldType.AUTO_INCREMENT)) { + clauses.add(identifier(autoIncrementField.getName()) + " = ?"); + parameters.add(identifierValue); + break; + } + + if (entity.hasPrimaryKeyField()) { + clauses.add(identifier(entity.getPrimaryKeyField().getName()) + " = ?"); + parameters.add(identifierValue); + } + + if (clauses.isEmpty()) { + return null; + } + + String sql = "SELECT * FROM " + table(entity) + + " WHERE " + String.join(" OR ", clauses) + + " LIMIT 1"; + List instances = queryEntityRows(entity, sql, parameters); + if (instances.isEmpty()) { + return null; + } + return instances.get(0); + } + @Override public EntityInstance addInstance(final EntityInstance instance) { ensureSchemaReady(); @@ -107,6 +217,12 @@ public EntityInstance updateInstance(final EntityInstance instance) { @Override public void deleteEntityInstance(final EntityInstance instance) { + if (!legacySnapshotLoaded) { + super.deleteEntityInstance(instance); + deletePersistedInstance(instance.getInternalId()); + return; + } + Set idsBeforeDelete = currentInternalIds(); super.deleteEntityInstance(instance); Set idsAfterDelete = currentInternalIds(); @@ -164,8 +280,35 @@ public List removeAllRelationships(final EntityInstance instance return removed; } + @Override + public Collection getConnectedItems( + final EntityInstance instance, final String relationshipName) { + ensureSchemaReady(); + Set connected = new HashSet<>(); + + for (RelationshipVectorDefinition vector : + instance.getEntity().related().getRelationships(relationshipName)) { + addConnectedItemsFromVectorTable(connected, instance, vector); + if (vector.getRelationshipDefinition().isTwoWay()) { + RelationshipVectorDefinition otherVector = + vector.getRelationshipDefinition().otherVectorOf(vector); + if (otherVector != null) { + addConnectedItemsFromVectorTable(connected, instance, otherVector); + } + } + } + + if (connected.isEmpty() && legacySnapshotLoaded) { + return super.getConnectedItems(instance, relationshipName); + } + return connected; + } + @Override public void flush() { + if (!legacySnapshotLoaded) { + return; + } refreshSchema(schema); for (EntityDefinition entity : schema.getEntityDefinitions()) { executeSql("DELETE FROM " + table(entity)); @@ -309,7 +452,8 @@ private void loadRelationshipRows() { while (resultSet.next()) { EntityInstance from = instances.get(resultSet.getString("from_internal_id")); EntityInstance to = instances.get(resultSet.getString("to_internal_id")); - if (from != null && to != null) { + if (from != null && to != null && + !from.getRelationships().getConnectedItems(vector.getName()).contains(to)) { from.getRelationships().connect(vector.getName(), to); } } @@ -341,6 +485,149 @@ private void restoreCountersFor(final EntityDefinition entity) { } } + private SqlQuery selectInstancesSql( + final EntityDefinition entity, final QueryFilterParams queryParams) { + StringBuilder sql = new StringBuilder("SELECT * FROM " + table(entity)); + List whereClauses = new ArrayList<>(); + List parameters = new ArrayList<>(); + + for (FilterBy filterBy : queryParams.toList()) { + if (EntityListSortParamParser.isSortByParam(filterBy.fieldName)) { + continue; + } + if (!entity.hasFieldNameDefined(filterBy.fieldName)) { + continue; + } + + String column = identifier(filterBy.fieldName); + switch (filterBy.filterOperation) { + case "=": + whereClauses.add(column + " = ?"); + parameters.add(filterBy.fieldValue); + break; + case "!": + case "!=": + whereClauses.add(column + " <> ?"); + parameters.add(filterBy.fieldValue); + break; + case "<": + case ">": + case "<=": + case ">=": + whereClauses.add(column + " " + filterBy.filterOperation + " ?"); + parameters.add(filterBy.fieldValue); + break; + case "*=": + whereClauses.add(column + " LIKE ? ESCAPE '\\'"); + parameters.add(sqlLikeWildcard(filterBy.fieldValue)); + break; + default: + break; + } + } + + if (!whereClauses.isEmpty()) { + sql.append(" WHERE ").append(String.join(" AND ", whereClauses)); + } + + List orderClauses = new ArrayList<>(); + for (SortByFieldName sortBy : new EntityListSortParamParser(queryParams).sortBys()) { + if (!entity.hasFieldNameDefined(sortBy.getFieldName())) { + continue; + } + orderClauses.add(identifier(sortBy.getFieldName()) + + (sortBy.getOrder() < 0 ? " ASC" : " DESC")); + } + if (!orderClauses.isEmpty()) { + sql.append(" ORDER BY ").append(String.join(", ", orderClauses)); + } + + return new SqlQuery(sql.toString(), parameters); + } + + private boolean hasJavaOnlyFilters(final QueryFilterParams queryParams) { + for (FilterBy filterBy : queryParams.toList()) { + if ("~=".equals(filterBy.filterOperation)) { + return true; + } + } + return false; + } + + private List queryEntityRows( + final EntityDefinition entity, final String sql, final List parameters) { + List instances = new ArrayList<>(); + try (PreparedStatement statement = connection.prepareStatement(sql)) { + for (int index = 0; index < parameters.size(); index++) { + statement.setObject(index + 1, parameters.get(index)); + } + try (ResultSet resultSet = statement.executeQuery()) { + while (resultSet.next()) { + instances.add(instanceFromRow(entity, resultSet)); + } + } + } catch (SQLException e) { + throw new IllegalStateException("Could not query SQLite entity rows for " + entity.getName(), e); + } + return instances; + } + + private EntityInstance instanceFromRow( + final EntityDefinition entity, final ResultSet resultSet) throws SQLException { + String internalId = resultSet.getString(INTERNAL_ID_COLUMN); + EntityInstanceCollection collection = getInstanceCollectionForEntityNamed(entity.getName()); + EntityInstance existing = collection.findInstanceByInternalID(internalId); + if (existing != null) { + return existing; + } + + EntityInstance instance = new EntityInstance( + entity, UUID.fromString(internalId)); + for (String fieldName : entity.getFieldNames()) { + String value = resultSet.getString(fieldName); + if (value != null) { + instance.overrideValue(fieldName, value); + } + } + collection.addInstance(instance); + return instance; + } + + private String sqlLikeWildcard(final String wildcard) { + return wildcard. + replace("\\", "\\\\"). + replace("%", "\\%"). + replace("_", "\\_"). + replace("*", "%"). + replace("?", "_"); + } + + private void addConnectedItemsFromVectorTable( + final Set connected, + final EntityInstance instance, + final RelationshipVectorDefinition vector) { + EntityDefinition connectedEntity; + String sql; + + if (vector.getFrom() == instance.getEntity()) { + connectedEntity = vector.getTo(); + sql = "SELECT target.* FROM " + relationshipTable(vector) + " rel " + + "JOIN " + table(connectedEntity) + " target " + + "ON target." + identifier(INTERNAL_ID_COLUMN) + " = rel.to_internal_id " + + "WHERE rel.from_internal_id = ?"; + } else if (vector.getTo() == instance.getEntity()) { + connectedEntity = vector.getFrom(); + sql = "SELECT target.* FROM " + relationshipTable(vector) + " rel " + + "JOIN " + table(connectedEntity) + " target " + + "ON target." + identifier(INTERNAL_ID_COLUMN) + " = rel.from_internal_id " + + "WHERE rel.to_internal_id = ?"; + } else { + return; + } + + connected.addAll(queryEntityRows(connectedEntity, sql, List.of(instance.getInternalId()))); + } + private void persistInstance(final EntityInstance instance) { EntityDefinition entity = instance.getEntity(); List columns = new ArrayList<>(); @@ -531,4 +818,14 @@ private String identifier(final String rawIdentifier) { private String quotedValue(final String value) { return "'" + value.replace("'", "''") + "'"; } + + private static class SqlQuery { + private final String sql; + private final List parameters; + + private SqlQuery(final String sql, final List parameters) { + this.sql = sql; + this.parameters = parameters; + } + } } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java index 69c921cc..722a9565 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java @@ -7,6 +7,7 @@ import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; +import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; import uk.co.compendiumdev.thingifier.core.reporting.ValidationReport; import java.util.Collection; @@ -39,6 +40,10 @@ public interface ThingRepository extends AutoCloseable { Collection listInstances(EntityDefinition entity); + List listInstances(EntityDefinition entity, QueryFilterParams queryParams); + + EntityInstance findInstanceByQueryIdentifier(EntityDefinition entity, String identifier); + EntityInstance addInstance(EntityInstance instance); EntityInstance updateInstance(EntityInstance instance); diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java index 8ee6e3d0..75d1c793 100644 --- a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java @@ -10,8 +10,11 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; +import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; import java.nio.file.Path; +import java.util.List; public class ThingRepositoryContractTest { @@ -106,6 +109,42 @@ public void sqliteRepositoryQuotesGeneratedSqlIdentifiers() { } } + @Test + public void sqliteFileBackedRepositoryOnlyHydratesCompatibilitySnapshotWhenRequested() { + ERSchema schema = todoSchema(); + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + Path databasePath = tempDir.resolve("lazy.sqlite"); + + try (ThingRepository repository = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + repository.initializeFrom(schema); + + repository.addInstance(new EntityInstance(projectDefinition).setValue("title", "First")); + repository.addInstance(new EntityInstance(projectDefinition).setValue("title", "Second")); + } + + try (ThingRepository reopened = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + reopened.initializeFrom(schema); + + EntityInstanceCollection cachedProjects = + reopened.getInstanceCollectionForEntityNamed("project"); + Assertions.assertEquals(0, cachedProjects.countInstances()); + + QueryFilterParams params = new QueryFilterParams(); + params.put("title", "First"); + + List filtered = reopened.listInstances(projectDefinition, params); + + Assertions.assertEquals(1, filtered.size()); + Assertions.assertEquals(1, cachedProjects.countInstances()); + + reopened.getInstanceData(); + + Assertions.assertEquals(2, cachedProjects.countInstances()); + } + } + private void exerciseRepositoryContract(final ThingRepository repository) { ERSchema schema = todoSchema(); repository.initializeFrom(schema); @@ -120,6 +159,7 @@ private void exerciseRepositoryContract(final ThingRepository repository) { Assertions.assertEquals("1", project.getPrimaryKeyValue()); Assertions.assertEquals(project, repository.findInstanceByPrimaryKey(projectDefinition, "1")); + Assertions.assertEquals(project, repository.findInstanceByQueryIdentifier(projectDefinition, "1")); Assertions.assertEquals(project, repository.findInstanceByFieldNameAndValue( projectDefinition, "title", "Repository project")); Assertions.assertEquals(1, repository.listInstances(projectDefinition).size()); @@ -128,6 +168,31 @@ private void exerciseRepositoryContract(final ThingRepository repository) { duplicate.setValue("title", "Repository project"); Assertions.assertFalse(repository.checkFieldsForUniqueNess(duplicate, false).isValid()); + EntityInstance secondProject = new EntityInstance(projectDefinition); + secondProject.setValue("title", "Another project"); + repository.addInstance(secondProject); + + QueryFilterParams filteredParams = new QueryFilterParams(); + filteredParams.put("id", ">=2"); + List filteredProjects = + repository.listInstances(projectDefinition, filteredParams); + Assertions.assertEquals(1, filteredProjects.size()); + Assertions.assertEquals(secondProject, filteredProjects.get(0)); + + QueryFilterParams sortedParams = new QueryFilterParams(); + sortedParams.put("sortBy", "-id"); + List sortedProjects = + repository.listInstances(projectDefinition, sortedParams); + Assertions.assertEquals("2", sortedProjects.get(0).getPrimaryKeyValue()); + Assertions.assertEquals("1", sortedProjects.get(1).getPrimaryKeyValue()); + + QueryFilterParams regexParams = new QueryFilterParams(); + regexParams.put("title", "~=Repository.*"); + List regexProjects = + repository.listInstances(projectDefinition, regexParams); + Assertions.assertEquals(1, regexProjects.size()); + Assertions.assertEquals(project, regexProjects.get(0)); + EntityInstance task = new EntityInstance(taskDefinition); task.setValue("title", "Wire repository"); repository.addInstance(task); diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandler.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandler.java index 730a6878..7c6e2d3b 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandler.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandler.java @@ -4,7 +4,9 @@ import uk.co.compendiumdev.thingifier.api.http.headers.HttpHeadersBlock; import uk.co.compendiumdev.thingifier.api.response.ApiResponse; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.query.QueryResult; import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; +import uk.co.compendiumdev.thingifier.core.query.RepositoryBackedSimpleQuery; import uk.co.compendiumdev.thingifier.core.query.SimpleQuery; import java.util.List; @@ -28,13 +30,28 @@ public ApiResponse handle(final String url, final QueryFilterParams queryParams, String instanceDatabaseName = SessionHeaderParser.getDatabaseNameFromHeaderValue(requestHeaders); - SimpleQuery queryResults; - - if(thingifier.apiConfig().forParams().willAllowFilteringThroughUrlParams()){ - queryResults = new SimpleQuery(thingifier.getERmodel().getSchema(), thingifier.getERmodel().getInstanceData(instanceDatabaseName), url).performQuery( - queryParams); - }else{ - queryResults = new SimpleQuery(thingifier.getERmodel().getSchema(), thingifier.getERmodel().getInstanceData(instanceDatabaseName), url).performQuery(); + QueryResult queryResults; + boolean allowFiltering = thingifier.apiConfig().forParams().willAllowFilteringThroughUrlParams(); + QueryFilterParams effectiveQueryParams = allowFiltering ? queryParams : new QueryFilterParams(); + + if (RepositoryBackedSimpleQuery.canHandle(thingifier.getERmodel().getSchema(), url)) { + queryResults = new RepositoryBackedSimpleQuery( + thingifier.getERmodel().getSchema(), + thingifier.getRepository(instanceDatabaseName), + url). + performQuery(effectiveQueryParams); + } else if(allowFiltering){ + queryResults = new SimpleQuery( + thingifier.getERmodel().getSchema(), + thingifier.getERmodel().getInstanceData(instanceDatabaseName), + url). + performQuery(queryParams); + } else { + queryResults = new SimpleQuery( + thingifier.getERmodel().getSchema(), + thingifier.getERmodel().getInstanceData(instanceDatabaseName), + url). + performQuery(); } // TODO: we should support pagination through query params From e0960f26f7efeb4e562fd34ccc0cbc1e3db2e750 Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 14:06:33 +0100 Subject: [PATCH 03/11] Add SQLite JDBC to challengerAuto test runtime --- challengerAuto/pom.xml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/challengerAuto/pom.xml b/challengerAuto/pom.xml index 34beb8d4..6d2eda4e 100644 --- a/challengerAuto/pom.xml +++ b/challengerAuto/pom.xml @@ -24,6 +24,13 @@ test + + org.xerial + sqlite-jdbc + ${sqlite-jdbc-version} + test + + org.junit.jupiter junit-jupiter-api @@ -100,4 +107,4 @@ - \ No newline at end of file + From 2cccb8eebc67e68b0f88f7b2c6e97bb2b7be215a Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 14:24:25 +0100 Subject: [PATCH 04/11] Add standalone SQLite todo manager app --- .../examples/TodoManagerThingifier.java | 11 ++- pom.xml | 1 + .../pom.xml | 70 +++++++++++++++++++ .../todolist/sqlite/application/Main.java | 35 ++++++++++ standAloneTodoListManagerRestApiAuto/pom.xml | 9 ++- .../compendiumdev/sparkstart/Environment.java | 35 ++++++++-- 6 files changed, 153 insertions(+), 8 deletions(-) create mode 100644 standAloneSqlLiteTodoListManagerRestApi/pom.xml create mode 100644 standAloneSqlLiteTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/sqlite/application/Main.java diff --git a/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/examples/TodoManagerThingifier.java b/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/examples/TodoManagerThingifier.java index 4558d572..a5f1d285 100644 --- a/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/examples/TodoManagerThingifier.java +++ b/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/examples/TodoManagerThingifier.java @@ -10,6 +10,7 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; import uk.co.compendiumdev.thingifier.core.domain.definitions.validation.VRule; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepositoryProvider; import static uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType.STRING; @@ -19,6 +20,14 @@ public class TodoManagerThingifier { REMEMBER - THIS IS THE PRODUCTION DEFINITION */ public Thingifier get() { + return get(new EntityRelModel()); + } + + public Thingifier get(final ThingRepositoryProvider repositoryProvider) { + return get(new EntityRelModel(repositoryProvider)); + } + + public Thingifier get(final EntityRelModel entityRelModel) { // this is basically an Entity Relationship diagram as source // TODO: should expand functionality based on E-R diagrams @@ -40,7 +49,7 @@ public Thingifier get() { // e.g. (.|\s)*\S(.|\s)* non empty - Thingifier todoManager = new Thingifier(); + Thingifier todoManager = new Thingifier(entityRelModel); StringBuilder para = new StringBuilder(); diff --git a/pom.xml b/pom.xml index d73cb2c2..26950ade 100644 --- a/pom.xml +++ b/pom.xml @@ -57,6 +57,7 @@ challengerAuto standAloneTodoListRestApi standAloneTodoListManagerRestApi + standAloneSqlLiteTodoListManagerRestApi standAloneTodoListManagerRestApiAuto thingifierapp swaggerizer diff --git a/standAloneSqlLiteTodoListManagerRestApi/pom.xml b/standAloneSqlLiteTodoListManagerRestApi/pom.xml new file mode 100644 index 00000000..f1393042 --- /dev/null +++ b/standAloneSqlLiteTodoListManagerRestApi/pom.xml @@ -0,0 +1,70 @@ + + + 4.0.0 + + + uk.co.compendiumdev.thingifier + thingifier-root + 1.5.6-SNAPSHOT + + + standAloneSqlLiteTodoListManagerRestApi + + + + uk.co.compendiumdev + thingifier + ${thingifier.version} + + + uk.co.compendiumdev.thingifier + examplemodels + ${thingifier.version} + + + org.xerial + sqlite-jdbc + ${sqlite-jdbc-version} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + ${java.version} + ${java.version} + + + + maven-assembly-plugin + 3.3.0 + + + package + + single + + + + + + jar-with-dependencies + + runSqlLiteTodoManagerRestAPI-${project.version} + false + + + uk.co.compendiumdev.todolist.sqlite.application.Main + + + + + + + diff --git a/standAloneSqlLiteTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/sqlite/application/Main.java b/standAloneSqlLiteTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/sqlite/application/Main.java new file mode 100644 index 00000000..dd3e5035 --- /dev/null +++ b/standAloneSqlLiteTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/sqlite/application/Main.java @@ -0,0 +1,35 @@ +package uk.co.compendiumdev.todolist.sqlite.application; + +import uk.co.compendiumdev.thingifier.application.MainImplementation; +import uk.co.compendiumdev.thingifier.application.examples.TodoManagerThingifier; +import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepositoryProvider; + +import static spark.Spark.get; + +public class Main { + + public static void main(String[] args) { + + MainImplementation app = new MainImplementation(); + app.registerModel("todoManager", + new TodoManagerThingifier().get(SqliteThingRepositoryProvider.inMemory())); + + app.setDefaultsFromArgs(args); + + app.configurePortAndDefaultRoutes(); + app.setupBuiltInConfigurableRoutes(); + + app.chooseThingifier(); + app.configureThingifierWithProfile(); + + app.setupDefaultGui(); + + get("/", (request, response) -> { + response.redirect("/gui"); + return ""; + }); + + app.startRestServer(); + app.addBuiltInArgConfiguredHooks(); + } +} diff --git a/standAloneTodoListManagerRestApiAuto/pom.xml b/standAloneTodoListManagerRestApiAuto/pom.xml index 6714e449..d8579b61 100644 --- a/standAloneTodoListManagerRestApiAuto/pom.xml +++ b/standAloneTodoListManagerRestApiAuto/pom.xml @@ -20,6 +20,13 @@ test + + uk.co.compendiumdev.thingifier + standAloneSqlLiteTodoListManagerRestApi + ${thingifier.version} + test + + org.junit.jupiter junit-jupiter-api @@ -65,4 +72,4 @@ - \ No newline at end of file + diff --git a/standAloneTodoListManagerRestApiAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java b/standAloneTodoListManagerRestApiAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java index e1ae3505..5befd4b1 100644 --- a/standAloneTodoListManagerRestApiAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java +++ b/standAloneTodoListManagerRestApiAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java @@ -5,10 +5,13 @@ import io.restassured.filter.log.RequestLoggingFilter; import io.restassured.filter.log.ResponseLoggingFilter; import spark.Spark; -import uk.co.compendiumdev.todolist.application.Main; public class Environment { + private static final String APP_MODE_PROPERTY = "standalone.todo.manager.app"; + private static final String APP_MODE_ENV = "STANDALONE_TODO_MANAGER_APP"; + private static final String APP_ARGS_PROPERTY = "standalone.todo.manager.args"; + public static String getEnv(String urlPath){ return getBaseUri() + urlPath; } @@ -30,12 +33,13 @@ public static String getBaseUri() { }else{ //start it up Spark.port(4567); - String [] args; - - // todo support configuring to write tests against different versions - args = "".split(","); + String[] args = configuredAppArgs(); - Main.main(args); + if (useSqliteMemoryApp()) { + uk.co.compendiumdev.todolist.sqlite.application.Main.main(args); + } else { + uk.co.compendiumdev.todolist.application.Main.main(args); + } // wait till running int maxtries=10; @@ -54,4 +58,23 @@ public static String getBaseUri() { // TODO: incorporate browsermob proxy and allow configuration of all // requests through a proxy file to output a HAR file of all requests for later review } + + private static boolean useSqliteMemoryApp() { + String appMode = System.getProperty(APP_MODE_PROPERTY); + if (appMode == null || appMode.isBlank()) { + appMode = System.getenv(APP_MODE_ENV); + } + + return "sqlite-memory".equalsIgnoreCase(appMode) || + "sqlite".equalsIgnoreCase(appMode); + } + + private static String[] configuredAppArgs() { + String configuredArgs = System.getProperty(APP_ARGS_PROPERTY, ""); + if (configuredArgs.isBlank()) { + return new String[0]; + } + + return configuredArgs.split(","); + } } From 7a3e3af0c99feb3261d643ba814c341371e3bb11 Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 15:19:55 +0100 Subject: [PATCH 05/11] Route simple entity writes through repositories --- .../challenge/ChallengeMain.java | 3 + .../ChallengeApiModelRepositoryTest.java | 97 +++++++++++++++++++ docs/repository-sqlite-migration-audit.md | 39 ++++++++ .../thingifier/core/EntityRelModel.java | 7 +- .../repository/SqliteThingRepository.java | 10 ++ .../SqliteThingRepositoryProvider.java | 5 +- .../core/repository/ThingRepository.java | 4 + .../ThingRepositoryContractTest.java | 13 +++ .../compendiumdev/thingifier/Thingifier.java | 7 +- .../restapihandlers/BodyArgsProcessor.java | 8 +- .../BodyCreationValidator.java | 42 +++++++- .../BodyRelationshipValidator.java | 17 ++-- .../api/restapihandlers/EntityUrlMatcher.java | 74 ++++++++++++++ .../restapihandlers/RelationshipCreation.java | 12 +-- .../restapihandlers/RelationshipCreator.java | 8 +- .../restapihandlers/RestApiDeleteHandler.java | 15 ++- .../restapihandlers/RestApiPostHandler.java | 41 ++++---- .../restapihandlers/RestApiPutHandler.java | 36 ++++--- .../api/restapihandlers/ThingCreation.java | 35 ++++--- .../application/MainImplementation.java | 11 ++- .../routehandlers/ShutdownRouteHandler.java | 9 ++ 21 files changed, 412 insertions(+), 81 deletions(-) create mode 100644 docs/repository-sqlite-migration-audit.md create mode 100644 thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/EntityUrlMatcher.java diff --git a/challenger/src/main/java/uk/co/compendiumdev/challenge/ChallengeMain.java b/challenger/src/main/java/uk/co/compendiumdev/challenge/ChallengeMain.java index afacaaf0..e3996999 100644 --- a/challenger/src/main/java/uk/co/compendiumdev/challenge/ChallengeMain.java +++ b/challenger/src/main/java/uk/co/compendiumdev/challenge/ChallengeMain.java @@ -118,6 +118,9 @@ public static ChallengeRouteHandler getChallenger(){ } public static void stop(){ + if (challenger != null) { + challenger.getThingifier().close(); + } challenger = null; } } diff --git a/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java b/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java index 218a2504..1db0df1b 100644 --- a/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java +++ b/challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java @@ -3,6 +3,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import uk.co.compendiumdev.thingifier.Thingifier; +import uk.co.compendiumdev.thingifier.api.http.HttpApiRequest; +import uk.co.compendiumdev.thingifier.api.http.bodyparser.BodyParser; import uk.co.compendiumdev.thingifier.api.http.headers.HttpHeadersBlock; import uk.co.compendiumdev.thingifier.api.response.ApiResponse; import uk.co.compendiumdev.thingifier.core.domain.definitions.Cardinality; @@ -115,4 +117,99 @@ public void relationshipGetRoutesStillFallBackToLegacyQueryTraversal() { response.getReturnedInstanceCollection().get(0).getFieldValue("title").asString()); } } + + @Test + public void simpleEntityWritesUseSqliteRepositoryWithoutLoadingCompatibilitySnapshot() { + try (SqliteThingRepositoryProvider provider = SqliteThingRepositoryProvider.inMemory()) { + Thingifier thingifier = new Thingifier(new EntityRelModel(provider)); + + EntityDefinition note = thingifier.defineThing("note", "notes"); + note.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + note.addField(Field.is("title", FieldType.STRING)); + + ThingRepository repository = thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME); + + ApiResponse create = thingifier.api().post( + "/notes", + bodyParser(thingifier, "{\"title\":\"created\"}"), + new HttpHeadersBlock()); + + Assertions.assertEquals(201, create.getStatusCode()); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + Assertions.assertEquals(1, repository.listInstances(note).size()); + + ApiResponse postAmend = thingifier.api().post( + "/notes/1", + bodyParser(thingifier, "{\"title\":\"posted\"}"), + new HttpHeadersBlock()); + + Assertions.assertEquals(200, postAmend.getStatusCode()); + Assertions.assertEquals( + "posted", + repository.findInstanceByQueryIdentifier(note, "1"). + getFieldValue("title").asString()); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + + ApiResponse putAmend = thingifier.api().put( + "/notes/1", + bodyParser(thingifier, "{\"title\":\"put\"}"), + new HttpHeadersBlock()); + + Assertions.assertEquals(200, putAmend.getStatusCode()); + Assertions.assertEquals( + "put", + repository.findInstanceByQueryIdentifier(note, "1"). + getFieldValue("title").asString()); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + + ApiResponse delete = thingifier.api().delete( + "/notes/1", new HttpHeadersBlock()); + + Assertions.assertEquals(200, delete.getStatusCode()); + Assertions.assertEquals(0, repository.listInstances(note).size()); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + } + } + + @Test + public void sqliteMemoryProvidersHaveSeparateNamedDatabaseLifetimes() { + EntityRelModel firstModel = null; + EntityRelModel secondModel = null; + try (SqliteThingRepositoryProvider firstProvider = SqliteThingRepositoryProvider.inMemory(); + SqliteThingRepositoryProvider secondProvider = SqliteThingRepositoryProvider.inMemory()) { + firstModel = new EntityRelModel(firstProvider); + secondModel = new EntityRelModel(secondProvider); + + EntityDefinition firstNote = firstModel.createEntityDefinition("note", "notes"); + firstNote.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + firstNote.addField(Field.is("title", FieldType.STRING)); + + EntityDefinition secondNote = secondModel.createEntityDefinition("note", "notes"); + secondNote.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + secondNote.addField(Field.is("title", FieldType.STRING)); + + firstModel.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME). + addInstance(new EntityInstance(firstNote).setValue("title", "first")); + + Assertions.assertEquals( + 1, + firstModel.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME). + listInstances(firstNote).size()); + Assertions.assertEquals( + 0, + secondModel.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME). + listInstances(secondNote).size()); + } finally { + if (firstModel != null) { + firstModel.close(); + } + if (secondModel != null) { + secondModel.close(); + } + } + } + + private BodyParser bodyParser(final Thingifier thingifier, final String json) { + return new BodyParser(new HttpApiRequest("/path").setBody(json), thingifier.getThingNames()); + } } diff --git a/docs/repository-sqlite-migration-audit.md b/docs/repository-sqlite-migration-audit.md new file mode 100644 index 00000000..d9ecc923 --- /dev/null +++ b/docs/repository-sqlite-migration-audit.md @@ -0,0 +1,39 @@ +# Repository SQLite Migration Audit + +This note classifies the current `getInstanceData()` and `getThingInstancesNamed()` usage during the SQLite repository experiment. + +## Normal API Request Paths + +Simple entity `GET` requests already use `ThingRepository` for `/entity`, `/entities`, `/entity/{id}`, and `/entities/{id}` when the path does not traverse a relationship. + +Simple entity `POST`, `PUT`, and `DELETE` requests now resolve entity definitions and entity instances through `ThingRepository` for root collection and root instance paths. These paths should not call `getInstanceData()` for SQLite unless a payload uses unsupported legacy behavior. + +Relationship URL paths still use the legacy `SimpleQuery` fallback. This includes relationship traversal and relationship create/delete paths such as `/projects/{id}/tasks` and `/projects/{id}/tasks/{id}`. + +## Relationship Payload Helpers + +Relationship references in request bodies now resolve target instances through `ThingRepository` rather than `EntityInstanceCollection` scans. This keeps normal create/amend payloads from requiring a full SQLite compatibility snapshot merely to find a related target by id, guid, or other supported field. + +## Compatibility Snapshot Paths + +`EntityRelModel.getInstanceData()` and `ThingRepository.getInstanceData()` are compatibility/export paths. For SQLite, `getInstanceData()` explicitly hydrates the legacy in-memory snapshot. + +Known compatibility users: + +- data populator and JSON import paths; +- admin/debug/export views and reporting; +- Challenger persistence/session restore; +- GUI data explorer pages; +- relationship URL fallback paths. + +These are acceptable for this phase, but they are the next places to reduce if memory efficiency needs to extend beyond simple entity API operations. + +## Test Helpers + +Most `getInstanceData()` and `getThingInstancesNamed()` usage is in `src/test`. These are test fixture and assertion helpers, not runtime leaks, unless a test is specifically exercising SQLite memory efficiency. + +## Runtime Lifecycle + +SQLite in-memory repositories use named database URLs scoped to a single `SqliteThingRepositoryProvider` lifetime. Each logical database key receives its own SQLite in-memory database name inside that provider namespace. + +The repository/provider owns the SQLite connection. `Thingifier.close()`, `EntityRelModel.close()`, `MainImplementation.close()`, and the shutdown route close repository providers so the in-memory database lifetime is explicit and connection-bound. diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java index aa6cf253..49c0ab31 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java @@ -18,7 +18,7 @@ Schema and instances are separate to allow us to have multiple 'databases' in memory at the same time built from the same schema. */ -public class EntityRelModel { +public class EntityRelModel implements AutoCloseable { public static final String DEFAULT_DATABASE_NAME = "__default"; @@ -99,6 +99,11 @@ public ThingRepositoryProvider getRepositoryProvider() { return repositories; } + @Override + public void close() { + repositories.close(); + } + // ERM Object Level public EntityRelModel cloneWithDifferentData(final List instances) { return new EntityRelModel(schema, new ERInstanceData(instances)); diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java index e59d3535..841a7f28 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java @@ -45,6 +45,7 @@ public class SqliteThingRepository extends InMemoryThingRepository { private ERSchema schema; private Connection connection; private boolean legacySnapshotLoaded; + private boolean closed; public SqliteThingRepository(final String databaseKey, final String jdbcUrl) { super(databaseKey, new ERInstanceData()); @@ -77,6 +78,11 @@ public ERInstanceData getInstanceData() { return super.getInstanceData(); } + @Override + public boolean hasLoadedCompatibilitySnapshot() { + return legacySnapshotLoaded; + } + @Override public void refreshSchema(final ERSchema schema) { this.schema = schema; @@ -327,6 +333,7 @@ public void flush() { @Override public void close() { + closed = true; if (connection == null) { return; } @@ -339,6 +346,9 @@ public void close() { } private void openConnection() { + if (closed) { + throw new IllegalStateException("SQLite repository has been closed"); + } if (connection != null) { return; } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepositoryProvider.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepositoryProvider.java index ca996af4..835c629d 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepositoryProvider.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepositoryProvider.java @@ -10,6 +10,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.function.Function; public class SqliteThingRepositoryProvider implements ThingRepositoryProvider { @@ -28,8 +29,10 @@ private SqliteThingRepositoryProvider(final Function jdbcUrlFact } public static SqliteThingRepositoryProvider inMemory() { + String providerName = "thingifier_" + UUID.randomUUID().toString().replace("-", ""); return new SqliteThingRepositoryProvider( - databaseKey -> "jdbc:sqlite:file:" + safeName(databaseKey) + "?mode=memory&cache=shared"); + databaseKey -> "jdbc:sqlite:file:" + providerName + "_" + safeName(databaseKey) + + "?mode=memory&cache=shared"); } public static SqliteThingRepositoryProvider fileBacked(final Path directory) { diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java index 722a9565..2c0461c3 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java @@ -69,6 +69,10 @@ List removeRelationshipsInvolving( Collection getConnectedItems(EntityInstance instance, String relationshipName); + default boolean hasLoadedCompatibilitySnapshot() { + return true; + } + default void flush() { // Allows persistent repositories to sync legacy direct collection mutations. } diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java index 75d1c793..20943497 100644 --- a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java @@ -49,6 +49,19 @@ public void sqliteProviderKeepsLogicalDatabasesSeparate() { } } + @Test + public void sqliteRepositoryCannotBeUsedAfterClose() { + ERSchema schema = todoSchema(); + ThingRepository repository = SqliteThingRepository.inMemory(EntityRelModel.DEFAULT_DATABASE_NAME); + repository.initializeFrom(schema); + + repository.close(); + + Assertions.assertThrows( + IllegalStateException.class, + () -> repository.listInstances(schema.getEntityDefinitionNamed("project"))); + } + @Test public void sqliteFileBackedRepositorySurvivesCloseAndReopen() { ERSchema schema = todoSchema(); diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java index 922603a3..a9c1b8dc 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java @@ -24,7 +24,7 @@ is the main class that allows access to: - the API Definition and config - TODO: why is the API documentation not in here? */ -public final class Thingifier { +public final class Thingifier implements AutoCloseable { private final EntityRelModel erm; private ApiDocsConfig apiDocsConfig; @@ -228,6 +228,11 @@ public ThingRepository getRepository(final String database) { return erm.getRepository(database); } + @Override + public void close() { + erm.close(); + } + /* TODO: these are documentation methods, why are they not in the diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyArgsProcessor.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyArgsProcessor.java index 79e2a2a8..9af76633 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyArgsProcessor.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyArgsProcessor.java @@ -73,9 +73,11 @@ public void identifyRelationships(List> fullargs, instance.getEntity().related().getRelationships(relationshipName); for(RelationshipVectorDefinition relate : relationshipsAre){ final EntityDefinition typeOfThing = relate.getTo(); - instanceToRelateTo = thingifier.getThingInstancesNamed(typeOfThing.getName(), database). - findInstanceByFieldNameAndValue(relationshipFieldName, - complexKeyValue.getValue()); + instanceToRelateTo = thingifier.getRepository(database). + findInstanceByFieldNameAndValue( + typeOfThing, + relationshipFieldName, + complexKeyValue.getValue()); if(instanceToRelateTo!=null){ break; } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyCreationValidator.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyCreationValidator.java index d74a5f6e..43d1b17c 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyCreationValidator.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyCreationValidator.java @@ -7,6 +7,7 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; import java.util.List; import java.util.Map; @@ -43,6 +44,36 @@ public ValidationReport validate(final BodyParser bodyargs, final EntityDefiniti public ValidationReport areFieldsUnique(final BodyParser bodyargs, final EntityInstanceCollection thing, List uniqueFields) { + final ValidationReport report = new ValidationReport(); + + for (Map.Entry entry : bodyargs.getFlattenedStringMap()) { + + if (uniqueFields.contains(entry.getKey())) { + String existingValue = entry.getValue(); + + if (existingValue != null && existingValue.trim().length() > 0) { + final EntityInstance foundInstance = thing.findInstanceByFieldNameAndValue( + entry.getKey(), + entry.getValue() + ); + + if (foundInstance != null) { + report.setValid(false); + report.addErrorMessage( + String.format("Found Existing item with %s of %s", + entry.getKey(), entry.getValue())); + } + } + } + } + return report; + } + + public ValidationReport areFieldsUnique( + final BodyParser bodyargs, + final EntityDefinition thingDefinition, + final ThingRepository repository, + final List uniqueFields) { final ValidationReport report = new ValidationReport(); @@ -54,10 +85,13 @@ public ValidationReport areFieldsUnique(final BodyParser bodyargs, final EntityI if (existingValue != null && existingValue.trim().length() > 0) { // not unique if we can find something by that field value - final EntityInstance foundInstance = thing.findInstanceByFieldNameAndValue( - entry.getKey(), - entry.getValue() - ); + final EntityInstance foundInstance; + if (repository == null) { + foundInstance = null; + } else { + foundInstance = repository.findInstanceByFieldNameAndValue( + thingDefinition, entry.getKey(), entry.getValue()); + } if (foundInstance!=null) { report.setValid(false); diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyRelationshipValidator.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyRelationshipValidator.java index 25998f49..49ff580a 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyRelationshipValidator.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/BodyRelationshipValidator.java @@ -86,8 +86,8 @@ private boolean validateCompressedRelationshipDefinition( if(thingToRelateTo==null) { final List relationshipsNamed = thingDefinition.related().getRelationships(relationShipName); for(RelationshipVectorDefinition vector : relationshipsNamed){ - final EntityInstanceCollection thingToRelationship = thingifier.getThingInstancesNamed(vector.getTo().getName(), database); - thingToRelateTo = thingToRelationship.findInstanceByPrimaryKey(guidValue); + thingToRelateTo = thingifier.getRepository(database). + findInstanceByQueryIdentifier(vector.getTo(), guidValue); if(thingToRelateTo!=null){ break; } @@ -143,19 +143,22 @@ private boolean validateComplexFourPartRelationshipDefinition( String uniqueId = complexKeyValue; EntityInstance thingToRelateTo = null; - EntityInstanceCollection things = thingifier. - getInstancesForSingularOrPluralNamedEntity(relationshipToPart, database); + EntityDefinition things = thingifier.getERmodel().getSchema(). + getDefinitionWithSingularOrPluralNamed(relationshipToPart); if(things!=null){ - thingToRelateTo = things.findInstanceByPrimaryKey(uniqueId); + thingToRelateTo = thingifier.getRepository(database). + findInstanceByQueryIdentifier(things, uniqueId); } // haven't found it yet if(thingToRelateTo==null){ - things = thingifier.getInstancesForSingularOrPluralNamedEntity(relationshipToPart, database); + things = thingifier.getERmodel().getSchema(). + getDefinitionWithSingularOrPluralNamed(relationshipToPart); if(things!=null) { - thingToRelateTo = things.findInstanceByFieldNameAndValue(relationshipFieldPart, uniqueId); + thingToRelateTo = thingifier.getRepository(database). + findInstanceByFieldNameAndValue(things, relationshipFieldPart, uniqueId); } } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/EntityUrlMatcher.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/EntityUrlMatcher.java new file mode 100644 index 00000000..f1e79243 --- /dev/null +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/EntityUrlMatcher.java @@ -0,0 +1,74 @@ +package uk.co.compendiumdev.thingifier.api.restapihandlers; + +import uk.co.compendiumdev.thingifier.Thingifier; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; + +final class EntityUrlMatcher { + + private EntityUrlMatcher() { + } + + static EntityDefinition entityFromCollectionUrl(final Thingifier thingifier, final String url) { + String[] parts = parts(url); + if (parts.length != 1) { + return null; + } + return thingifier.getERmodel().getSchema(). + getDefinitionWithSingularOrPluralNamed(parts[0]); + } + + static EntityDefinition entityFromInstanceUrl(final Thingifier thingifier, final String url) { + String[] parts = parts(url); + if (parts.length != 2) { + return null; + } + return thingifier.getERmodel().getSchema(). + getDefinitionWithSingularOrPluralNamed(parts[0]); + } + + static String identifierFromInstanceUrl(final String url) { + String[] parts = parts(url); + if (parts.length != 2) { + return null; + } + return parts[1]; + } + + static String entityTermFromUrl(final String url) { + String[] parts = parts(url); + if (parts.length == 0) { + return ""; + } + return parts[0]; + } + + static boolean hasPartCount(final String url, final int count) { + return parts(url).length == count; + } + + static EntityInstance findInstanceFromUrl( + final Thingifier thingifier, final String url, final String database) { + EntityDefinition entity = entityFromInstanceUrl(thingifier, url); + String identifier = identifierFromInstanceUrl(url); + if (entity == null || identifier == null) { + return null; + } + return thingifier.getRepository(database). + findInstanceByQueryIdentifier(entity, identifier); + } + + private static String[] parts(final String url) { + String normalized = url == null ? "" : url.trim(); + while (normalized.startsWith("/")) { + normalized = normalized.substring(1); + } + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + if (normalized.isEmpty()) { + return new String[0]; + } + return normalized.split("/"); + } +} diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreation.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreation.java index 9ec26028..2d6584b9 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreation.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreation.java @@ -1,6 +1,5 @@ package uk.co.compendiumdev.thingifier.api.restapihandlers; -import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; import uk.co.compendiumdev.thingifier.Thingifier; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; @@ -46,21 +45,22 @@ public ApiResponse create(final String url, final BodyParser bodyargs, final Sim // if no way to narrow it down then use the first one TODO: potential bug if multiple named relationshps relationshipToUse = possibleRelationships.get(0); - EntityInstanceCollection thingTo = thingifier.getThingInstancesNamed(relationshipToUse.getTo().getName(), database); + EntityDefinition thingTo = relationshipToUse.getTo(); // if there is a guid in the body then use that to try and find a thing that matches it // if there is a guid field or an id field then use whichever first matches a thing boolean amExpectingARelatedItem = false; String matchingFieldNames = ""; for(String fieldName : args.keySet()){ - final Field field = thingTo.definition().getField(fieldName); + final Field field = thingTo.getField(fieldName); // in theory this is any 'key' unique field if(field.getType()== FieldType.AUTO_GUID || field.getType() == FieldType.AUTO_INCREMENT){ amExpectingARelatedItem=true; if(!matchingFieldNames.contains(fieldName+ " ")){ matchingFieldNames = matchingFieldNames + fieldName +" "; } - relatedItem = thingTo.findInstanceByFieldNameAndValue(fieldName, args.get(fieldName)); + relatedItem = thingifier.getRepository(database). + findInstanceByFieldNameAndValue(thingTo, fieldName, args.get(fieldName)); if(relatedItem!=null){ // found something break; @@ -74,14 +74,14 @@ public ApiResponse create(final String url, final BodyParser bodyargs, final Sim EntityInstance returnThing = null; - EntityInstanceCollection thingToCreate = null; + EntityDefinition thingToCreate = null; ApiResponse response = null; // if we have a parent thing, but no GUID then can we create a Thing and connect it later? if (relatedItem == null) { EntityDefinition createThing = relationshipToUse.getTo(); - thingToCreate = thingifier.getThingInstancesNamed(createThing.getName(), database); + thingToCreate = createThing; response = new ThingCreation(thingifier).with(bodyargs, thingToCreate, database); if(response.isErrorResponse()){ diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreator.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreator.java index a4dfa8f2..cbafff1d 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreator.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RelationshipCreator.java @@ -3,6 +3,7 @@ import uk.co.compendiumdev.thingifier.Thingifier; import uk.co.compendiumdev.thingifier.api.http.bodyparser.BodyParser; import uk.co.compendiumdev.thingifier.api.response.ApiResponse; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import java.util.ArrayList; @@ -19,11 +20,14 @@ public ApiResponse createRelationships(final BodyParser bodyargs, final EntityIn try { List relationships = getRelationshipsFromArgs(bodyargs, instance, database); for (RelationshipDetails relationship : relationships) { + EntityDefinition relatedEntity = thingifier.getERmodel().getSchema(). + getDefinitionWithSingularOrPluralNamed(relationship.toType); thingifier.getRepository(database).connectRelationship( instance, relationship.relationshipName, - thingifier.getInstancesForSingularOrPluralNamedEntity(relationship.toType, database). - findInstanceByFieldNameAndValue(relationship.guidName, relationship.guidValue)); + thingifier.getRepository(database). + findInstanceByFieldNameAndValue( + relatedEntity, relationship.guidName, relationship.guidValue)); } return ApiResponse.created(instance, thingifier.apiConfig()); diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiDeleteHandler.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiDeleteHandler.java index 8e13f798..780363f2 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiDeleteHandler.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiDeleteHandler.java @@ -1,9 +1,9 @@ package uk.co.compendiumdev.thingifier.api.restapihandlers; import uk.co.compendiumdev.thingifier.api.http.headers.HttpHeadersBlock; -import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; import uk.co.compendiumdev.thingifier.Thingifier; import uk.co.compendiumdev.thingifier.api.response.ApiResponse; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.query.SimpleQuery; @@ -21,12 +21,23 @@ public ApiResponse handle(final String url, HttpHeadersBlock requestHeaders) { String instanceDatabaseName = SessionHeaderParser.getDatabaseNameFromHeaderValue(requestHeaders); // this should probably not delete root items - EntityInstanceCollection thing = thingifier.getInstancesForSingularOrPluralNamedEntity(url, instanceDatabaseName); + EntityDefinition thing = EntityUrlMatcher.entityFromCollectionUrl(thingifier, url); if (thing != null) { // can't delete root level with a DELETE return ApiResponse.error(405, "Cannot delete root level entity"); } + EntityDefinition entity = EntityUrlMatcher.entityFromInstanceUrl(thingifier, url); + if (entity != null) { + EntityInstance instance = EntityUrlMatcher.findInstanceFromUrl( + thingifier, url, instanceDatabaseName); + if (instance == null) { + return ApiResponse.error404(String.format("Could not find any instances with %s", url)); + } + thingifier.deleteThing(instance, instanceDatabaseName); + return ApiResponse.success(); + } + SimpleQuery queryResult = new SimpleQuery(thingifier.getERmodel().getSchema(), thingifier.getERmodel().getInstanceData(instanceDatabaseName), url).performQuery(); if (queryResult.wasItemFoundUnderARelationship()) { diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPostHandler.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPostHandler.java index 3d94161f..b5b28e90 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPostHandler.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPostHandler.java @@ -1,8 +1,8 @@ package uk.co.compendiumdev.thingifier.api.restapihandlers; import uk.co.compendiumdev.thingifier.api.http.headers.HttpHeadersBlock; -import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; import uk.co.compendiumdev.thingifier.Thingifier; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; import uk.co.compendiumdev.thingifier.core.reporting.ValidationReport; import uk.co.compendiumdev.thingifier.api.http.bodyparser.BodyParser; import uk.co.compendiumdev.thingifier.api.response.ApiResponse; @@ -30,11 +30,12 @@ public ApiResponse handle(final String url, final BodyParser args, final HttpHea */ // if queryis empty then need a way to check if the query matched // create a thing - EntityInstanceCollection instancesCollection = thingifier.getInstancesForSingularOrPluralNamedEntity(url, instanceDatabaseName); - if (instancesCollection != null) { + EntityDefinition entityDefinition = EntityUrlMatcher.entityFromCollectionUrl(thingifier, url); + if (entityDefinition != null) { // create a new thing does not enforce relationships // TODO: validate before creation so as to only delete in an 'emergency' not as default - final ApiResponse response = new ThingCreation(thingifier).with(args, instancesCollection, instanceDatabaseName); + final ApiResponse response = new ThingCreation(thingifier).with( + args, entityDefinition, instanceDatabaseName); if (response.isErrorResponse()) { return response; } @@ -58,40 +59,36 @@ public ApiResponse handle(final String url, final BodyParser args, final HttpHea */ // amend a thing // thing/guid - String[] urlParts = url.split("/"); - if (urlParts.length == 2) { + entityDefinition = EntityUrlMatcher.entityFromInstanceUrl(thingifier, url); + if (entityDefinition != null) { - String thingName = urlParts[0]; - instancesCollection = thingifier.getInstancesForSingularOrPluralNamedEntity(thingName, instanceDatabaseName); + String primaryKey = EntityUrlMatcher.identifierFromInstanceUrl(url); - if (instancesCollection == null) { - // this is not a URL for thing/guid - // unknown thing - return NoSuchEntity.response(thingName); - - } - - String primaryKey = urlParts[1]; - - if(instancesCollection.definition().hasPrimaryKeyField()){ - EntityInstance instance = instancesCollection.findInstanceByPrimaryKey(primaryKey); + if(entityDefinition.hasPrimaryKeyField()){ + EntityInstance instance = thingifier.getRepository(instanceDatabaseName). + findInstanceByQueryIdentifier(entityDefinition, primaryKey); if (instance == null) { // cannot amend something that does not exist return ApiResponse.error404(String.format("No such %s entity instance with %s == %s found", - instancesCollection.definition().getName(), - instancesCollection.definition().getPrimaryKeyField().getName(), + entityDefinition.getName(), + entityDefinition.getPrimaryKeyField().getName(), primaryKey) ); } return amendAThingWithPost(args, instance, instanceDatabaseName); }else{ - return ApiResponse.error404(String.format("Entity %s does not have a primary key defined", instancesCollection.definition().getName())); + return ApiResponse.error404(String.format("Entity %s does not have a primary key defined", entityDefinition.getName())); } } + String thingName = EntityUrlMatcher.entityTermFromUrl(url); + if (!thingName.isEmpty() && EntityUrlMatcher.hasPartCount(url, 2)) { + return NoSuchEntity.response(thingName); + } + /* Match a Relationship diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPutHandler.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPutHandler.java index 2553eeec..c73a16fe 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPutHandler.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiPutHandler.java @@ -1,10 +1,10 @@ package uk.co.compendiumdev.thingifier.api.restapihandlers; import uk.co.compendiumdev.thingifier.api.http.headers.HttpHeadersBlock; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.instance.NamedValue; -import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; import uk.co.compendiumdev.thingifier.Thingifier; import uk.co.compendiumdev.thingifier.api.http.bodyparser.BodyParser; import uk.co.compendiumdev.thingifier.api.response.ApiResponse; @@ -26,7 +26,7 @@ public ApiResponse handle(final String url, final BodyParser args, final HttpHea // if queryis empty then need a way to check if the query matched // create a thing - EntityInstanceCollection thing = thingifier.getInstancesForSingularOrPluralNamedEntity(url, instanceDatabaseName); + EntityDefinition thing = EntityUrlMatcher.entityFromCollectionUrl(thingifier, url); if (thing != null) { // can't create a new thing at root level with PUT return ApiResponse.error(405, "Cannot create root level entity with a PUT"); @@ -35,24 +35,22 @@ public ApiResponse handle(final String url, final BodyParser args, final HttpHea // amend a thing // thing/guid - String[] urlParts = url.split("/"); - if (urlParts.length != 2) { + thing = EntityUrlMatcher.entityFromInstanceUrl(thingifier, url); + if (thing == null) { // WHAT was that query? + if (EntityUrlMatcher.hasPartCount(url, 2)) { + String thingName = EntityUrlMatcher.entityTermFromUrl(url); + if (!thingName.isEmpty()) { + return NoSuchEntity.response(thingName); + } + } return ApiResponse.error(400, "Your request was not understood"); } - String thingName = urlParts[0]; - thing = thingifier.getInstancesForSingularOrPluralNamedEntity(thingName, instanceDatabaseName); - - if (thing == null) { - // this is not a URL for thing/guid - // unknown thing - return NoSuchEntity.response(urlParts[0]); - } - - String instanceGuid = urlParts[1]; + String instanceGuid = EntityUrlMatcher.identifierFromInstanceUrl(url); - EntityInstance instance = thing.findInstanceByPrimaryKey(instanceGuid); + EntityInstance instance = thingifier.getRepository(instanceDatabaseName). + findInstanceByQueryIdentifier(thing, instanceGuid); if (instance == null) { @@ -60,7 +58,7 @@ public ApiResponse handle(final String url, final BodyParser args, final HttpHea // if the primary key is an AUTO i.e. AUTO_GUID or AUTO_INCREMENT then we should not be able to create it // in fact if there are any fields at all which are AUTO then we should not be able to create it with PUT - List forbiddenPutCreationFields = thing.definition().getFieldsOfType(FieldType.AUTO_INCREMENT, FieldType.AUTO_GUID); + List forbiddenPutCreationFields = thing.getFieldsOfType(FieldType.AUTO_INCREMENT, FieldType.AUTO_GUID); if(forbiddenPutCreationFields.size()>0){ String names = ""; @@ -70,7 +68,7 @@ public ApiResponse handle(final String url, final BodyParser args, final HttpHea } names = names + field.getName(); } - return ApiResponse.error(400, String.format("Cannot create %s with PUT due to Auto fields %s", thing.definition().getName(), names)); + return ApiResponse.error(400, String.format("Cannot create %s with PUT due to Auto fields %s", thing.getName(), names)); } @@ -78,10 +76,10 @@ public ApiResponse handle(final String url, final BodyParser args, final HttpHea // any field in the body for the primary key must match the primarykey field List fieldValues = FieldValues.fromListMapEntryStringString(args.getFlattenedStringMap()); for( NamedValue namedValue : fieldValues){ - if(namedValue.name.equals(thing.definition().getPrimaryKeyField().getName())){ + if(namedValue.name.equals(thing.getPrimaryKeyField().getName())){ if(!namedValue.value.equals(instanceGuid)){ // primary key does not match the value in the message - return ApiResponse.error(400, String.format("Cannot create %s with PUT as key does not match body value %s != %s", thing.definition().getName(), instanceGuid, namedValue.value)); + return ApiResponse.error(400, String.format("Cannot create %s with PUT as key does not match body value %s != %s", thing.getName(), instanceGuid, namedValue.value)); } } } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingCreation.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingCreation.java index d3ad8f4a..4eb2d6b5 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingCreation.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/ThingCreation.java @@ -1,6 +1,7 @@ package uk.co.compendiumdev.thingifier.api.restapihandlers; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.instance.NamedValue; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; import uk.co.compendiumdev.thingifier.Thingifier; import uk.co.compendiumdev.thingifier.core.reporting.ValidationReport; @@ -21,6 +22,10 @@ public ThingCreation(final Thingifier thingifier) { } public ApiResponse with(final BodyParser bodyargs, final EntityInstanceCollection thing, final String database) { + return with(bodyargs, thing.definition(), database); + } + + public ApiResponse with(final BodyParser bodyargs, final EntityDefinition thing, final String database) { ValidationReport validated = new BodyRelationshipValidator(thingifier).validate(bodyargs, thing, database); @@ -34,34 +39,42 @@ public ApiResponse with(final BodyParser bodyargs, final EntityInstanceCollectio } // todo: separate validation for creation of 'cannot' create with ID, or cannot create with GUID - EntityInstance instance = new EntityInstance(thing.definition()); + EntityInstance instance = new EntityInstance(thing); instance.addAutoGUIDstoInstance(); instance.addAutoIncrementIdsToInstance( - thingifier.getRepository(database).countersFor(thing.definition())); + thingifier.getRepository(database).countersFor(thing)); return addNewThingWithFields(bodyargs, instance, thing, database); } // create with GUID and IDs is normally associated with PUT or 'insert' public ApiResponse withPrimaryKey(final String primaryKey, final BodyParser bodyargs, final EntityInstanceCollection thing, final String database) { + return withPrimaryKey(primaryKey, bodyargs, thing.definition(), database); + } + + public ApiResponse withPrimaryKey(final String primaryKey, final BodyParser bodyargs, + final EntityDefinition thing, final String database) { EntityInstance instance; ValidationReport validated; validated = new BodyCreationValidator(thingifier). - areFieldsUnique(bodyargs, thing, - thing.definition().getFieldNamesOfType(FieldType.AUTO_INCREMENT, FieldType.AUTO_GUID)); + areFieldsUnique( + bodyargs, + thing, + thingifier.getRepository(database), + thing.getFieldNamesOfType(FieldType.AUTO_INCREMENT, FieldType.AUTO_GUID)); if(!validated.isValid()){ return ApiResponse.error(409,"Cannot Create with duplicate values: "+ validated.getCombinedErrorMessages()); } - instance = new EntityInstance(thing.definition()); - instance.overrideValue(thing.definition().getPrimaryKeyField().getName(), primaryKey); + instance = new EntityInstance(thing); + instance.overrideValue(thing.getPrimaryKeyField().getName(), primaryKey); // TODO: should we really do this for a PUT when everything needs to exist, suspect this is a bug instance.addAutoIncrementIdsToInstance( - thingifier.getRepository(database).countersFor(thing.definition())); + thingifier.getRepository(database).countersFor(thing)); validated = new BodyRelationshipValidator(thingifier).validate(bodyargs, thing, database); @@ -78,14 +91,14 @@ public ApiResponse withPrimaryKey(final String primaryKey, final BodyParser body bodyargs.getFlattenedStringMap()); - thingifier.getRepository(database).setNextIdCountersToAccomodate(thing.definition(), fieldValues); + thingifier.getRepository(database).setNextIdCountersToAccomodate(thing, fieldValues); return insertNewThingWithFields(bodyargs, instance, thing, database); } private ApiResponse addNewThingWithFields(final BodyParser bodyargs, final EntityInstance instance, - final EntityInstanceCollection thing, final String database) { + final EntityDefinition thing, final String database) { if(thingifier.apiConfig().willApiEnforceDeclaredTypesInInput()) { ValidationReport validatedTypes = bodyargs.validateAgainstType(instance.getEntity()); @@ -118,7 +131,7 @@ private ApiResponse addNewThingWithFields(final BodyParser bodyargs, final Entit } private ApiResponse insertNewThingWithFields(final BodyParser bodyargs, final EntityInstance instance, - final EntityInstanceCollection thing, final String database) { + final EntityDefinition thing, final String database) { if(thingifier.apiConfig().willApiEnforceDeclaredTypesInInput()) { ValidationReport validatedTypes = bodyargs.validateAgainstType(instance.getEntity()); @@ -147,7 +160,7 @@ private ApiResponse insertNewThingWithFields(final BodyParser bodyargs, final En return addValidatedInstance(bodyargs, instance, thing, database, validation); } - private ApiResponse addValidatedInstance(BodyParser bodyargs, EntityInstance instance, EntityInstanceCollection thing, String database, ValidationReport validation) { + private ApiResponse addValidatedInstance(BodyParser bodyargs, EntityInstance instance, EntityDefinition thing, String database, ValidationReport validation) { // check for uniqueness prior to adding ValidationReport uniquenessCheck = thingifier.getRepository(database). diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/application/MainImplementation.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/application/MainImplementation.java index e1563d65..7f043e08 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/application/MainImplementation.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/application/MainImplementation.java @@ -28,7 +28,7 @@ This is a bit too tied down to the concept of one app, one thingifier at a time though. We should be able to have multiple thingifiers at different endpoints e.g. /api1/, /api2/ etc. */ -public class MainImplementation { +public class MainImplementation implements AutoCloseable { Integer proxyport; private Map thingifierModels; @@ -253,7 +253,7 @@ public void configurePortAndDefaultRoutes() { public void setupBuiltInConfigurableRoutes() { if(allowShutdown) { apiDefn.addRoutesToDocumentation( - new ShutdownRouteHandler(). + new ShutdownRouteHandler(this). configureRoutes(). getRoutes()); } @@ -391,4 +391,11 @@ public DefaultGUIHTML getGuiManagement() { public ThingifierApiDocumentationDefn getApiDefn() { return apiDefn; } + + @Override + public void close() { + for (Thingifier model : thingifierModels.values()) { + model.close(); + } + } } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/application/routehandlers/ShutdownRouteHandler.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/application/routehandlers/ShutdownRouteHandler.java index 6db9ed56..c4d64cbd 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/application/routehandlers/ShutdownRouteHandler.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/application/routehandlers/ShutdownRouteHandler.java @@ -12,9 +12,15 @@ public class ShutdownRouteHandler { List routes; + private final AutoCloseable closeable; public ShutdownRouteHandler(){ + this(null); + } + + public ShutdownRouteHandler(final AutoCloseable closeable){ routes = new ArrayList(); + this.closeable = closeable; } public List getRoutes(){ @@ -24,6 +30,9 @@ public List getRoutes(){ public ShutdownRouteHandler configureRoutes() { get("/shutdown", (request, result) -> { + if (closeable != null) { + closeable.close(); + } System.exit(0); return ""; }); From 52f13f2c7920fbef4d8f1f38909c35ed484088b4 Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 15:30:42 +0100 Subject: [PATCH 06/11] amended content text --- .../test/java/uk/co/compendiumdev/sparkstart/Environment.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/challengerAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java b/challengerAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java index 79ab387e..5f06988f 100644 --- a/challengerAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java +++ b/challengerAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java @@ -39,8 +39,8 @@ public static String getBaseUri() { if(SINGLE_PLAYER_MODE){ args = "-sqlite-memory".split(","); }else{ - args = "-multiplayer".split(","); - }; + args = "-sqlite-memory,-multiplayer".split(","); + } ChallengeMain.main(args); From 369d51939151a77eac77b576914653215f682e45 Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 16:14:58 +0100 Subject: [PATCH 07/11] Complete base repository SQLite migration paths --- docs/repository-sqlite-migration-audit.md | 19 +- .../thingifier/core/EntityRelModel.java | 23 +- .../RepositoryDataPopulator.java | 9 + .../query/RepositoryBackedSimpleQuery.java | 33 +- .../repository/InMemoryThingRepository.java | 12 + .../repository/SqliteThingRepository.java | 336 +++++++++++++++--- .../core/repository/ThingRepository.java | 10 + .../thingifier/core/EntityRelModelTest.java | 40 +++ .../ThingRepositoryContractTest.java | 88 ++++- .../data/TodoAPIDataPopulator.java | 38 +- .../data/TodoManagerAPIDataPopulator.java | 33 +- .../compendiumdev/thingifier/Thingifier.java | 55 ++- .../htmlgui/htmlgen/DefaultGuiHtmlPages.java | 43 ++- .../RelationshipApiSqliteRepositoryTest.java | 70 ++++ .../DefaultGuiHtmlPagesRepositoryTest.java | 63 ++++ 15 files changed, 764 insertions(+), 108 deletions(-) create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/datapopulator/RepositoryDataPopulator.java create mode 100644 thingifier/src/test/java/uk/co/compendiumdev/casestudy/todomanager/api_non_http/RelationshipApiSqliteRepositoryTest.java create mode 100644 thingifier/src/test/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/DefaultGuiHtmlPagesRepositoryTest.java diff --git a/docs/repository-sqlite-migration-audit.md b/docs/repository-sqlite-migration-audit.md index d9ecc923..92b3c50f 100644 --- a/docs/repository-sqlite-migration-audit.md +++ b/docs/repository-sqlite-migration-audit.md @@ -4,11 +4,13 @@ This note classifies the current `getInstanceData()` and `getThingInstancesNamed ## Normal API Request Paths -Simple entity `GET` requests already use `ThingRepository` for `/entity`, `/entities`, `/entity/{id}`, and `/entities/{id}` when the path does not traverse a relationship. +Simple entity `GET` requests use `ThingRepository` for `/entity`, `/entities`, `/entity/{id}`, and `/entities/{id}`. -Simple entity `POST`, `PUT`, and `DELETE` requests now resolve entity definitions and entity instances through `ThingRepository` for root collection and root instance paths. These paths should not call `getInstanceData()` for SQLite unless a payload uses unsupported legacy behavior. +Simple relationship `GET` traversal, such as `/projects/{id}/tasks`, now uses `ThingRepository.listRelatedInstances(...)`. SQLite executes the relationship lookup against relationship tables and materializes only the returned rows. -Relationship URL paths still use the legacy `SimpleQuery` fallback. This includes relationship traversal and relationship create/delete paths such as `/projects/{id}/tasks` and `/projects/{id}/tasks/{id}`. +Simple entity `POST`, `PUT`, and `DELETE` requests resolve entity definitions and entity instances through `ThingRepository` for root collection and root instance paths. These paths should not call `getInstanceData()` for SQLite unless a payload uses unsupported legacy behavior. + +Relationship create/delete URL paths still use the legacy `SimpleQuery` fallback. This includes write paths such as `POST /projects/{id}/tasks` and `DELETE /projects/{id}/tasks/{id}`. ## Relationship Payload Helpers @@ -22,11 +24,14 @@ Known compatibility users: - data populator and JSON import paths; - admin/debug/export views and reporting; -- Challenger persistence/session restore; -- GUI data explorer pages; -- relationship URL fallback paths. +- legacy data populators that have not implemented `RepositoryDataPopulator`; +- relationship write URL fallback paths. + +The default Todo API and Todo Manager seed populators now implement `RepositoryDataPopulator`, so SQLite-backed Todo apps can seed through repository writes without first hydrating `ERInstanceData`. + +The base GUI data explorer list/detail pages now read via `ThingRepository` and render relationship sections via `ThingRepository.listRelatedInstances(...)`. -These are acceptable for this phase, but they are the next places to reduce if memory efficiency needs to extend beyond simple entity API operations. +These remaining compatibility paths are acceptable for this phase, but they are the next places to reduce if memory efficiency needs to extend beyond simple entity reads and Todo-style population. ## Test Helpers diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java index 49c0ab31..1e469f75 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/EntityRelModel.java @@ -1,6 +1,7 @@ package uk.co.compendiumdev.thingifier.core; import uk.co.compendiumdev.thingifier.core.domain.datapopulator.DataPopulator; +import uk.co.compendiumdev.thingifier.core.domain.datapopulator.RepositoryDataPopulator; import uk.co.compendiumdev.thingifier.core.domain.definitions.Cardinality; import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; @@ -174,7 +175,8 @@ public boolean createInstanceDatabaseIfNotExisting(String databaseKey) { } public boolean populateDatabase(String databaseKey){ - if(repositories.getRepository(databaseKey)==null){ + ThingRepository repository = repositories.getRepository(databaseKey); + if(repository==null){ return false; } @@ -182,13 +184,8 @@ public boolean populateDatabase(String databaseKey){ return false; } - repositories.getRepository(databaseKey).refreshSchema(getSchema()); - - dataPopulator.populate( - getSchema(), - repositories.getRepository(databaseKey).getInstanceData() - ); - repositories.getRepository(databaseKey).flush(); + repository.refreshSchema(getSchema()); + populateRepository(repository); return true; } @@ -202,4 +199,14 @@ private void refreshRepositorySchemas() { repositories.getRepository(databaseKey).refreshSchema(schema); } } + + private void populateRepository(final ThingRepository repository) { + if(dataPopulator instanceof RepositoryDataPopulator) { + ((RepositoryDataPopulator) dataPopulator).populate(getSchema(), repository); + return; + } + + dataPopulator.populate(getSchema(), repository.getInstanceData()); + repository.flush(); + } } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/datapopulator/RepositoryDataPopulator.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/datapopulator/RepositoryDataPopulator.java new file mode 100644 index 00000000..8a30d2ff --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/domain/datapopulator/RepositoryDataPopulator.java @@ -0,0 +1,9 @@ +package uk.co.compendiumdev.thingifier.core.domain.datapopulator; + +import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; + +public interface RepositoryDataPopulator extends DataPopulator { + + void populate(ERSchema schema, ThingRepository repository); +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/RepositoryBackedSimpleQuery.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/RepositoryBackedSimpleQuery.java index 83721459..2f1a35df 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/RepositoryBackedSimpleQuery.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/RepositoryBackedSimpleQuery.java @@ -37,7 +37,7 @@ public RepositoryBackedSimpleQuery( public static boolean canHandle(final ERSchema schema, final String query) { String[] terms = termsFrom(query); - if (terms.length == 0 || terms.length > 2) { + if (terms.length == 0 || terms.length > 3) { return false; } @@ -51,8 +51,16 @@ public static boolean canHandle(final ERSchema schema, final String query) { } String identifierCandidate = terms[1]; - return !schema.hasRelationshipNamed(identifierCandidate) && - entityForTerm(schema, identifierCandidate) == null; + if (schema.hasRelationshipNamed(identifierCandidate) || + entityForTerm(schema, identifierCandidate) != null) { + return false; + } + + if (terms.length == 2) { + return true; + } + + return entity.related().hasRelationship(terms[2]); } public RepositoryBackedSimpleQuery performQuery(final QueryFilterParams queryParams) { @@ -76,6 +84,17 @@ public RepositoryBackedSimpleQuery performQuery(final QueryFilterParams queryPar return this; } + if (terms.length == 3) { + wasIntentToMatchInstance = true; + isCollection = true; + foundItems = new ArrayList<>( + repository.listRelatedInstances(currentInstance, terms[2], queryParams)); + resultContainsDefinition = relatedEntityFor(currentInstance, terms[2]); + lastMatchWasNothing = false; + lastMatchWasInstance = false; + return this; + } + foundItems = new ArrayList<>(); foundItems.add(currentInstance); lastMatchWasNothing = false; @@ -141,4 +160,12 @@ private static String[] termsFrom(final String query) { } return normalized.split("/"); } + + private EntityDefinition relatedEntityFor( + final EntityInstance instance, final String relationshipName) { + if (instance.getEntity().related().getRelationships(relationshipName).isEmpty()) { + return null; + } + return instance.getEntity().related().getRelationships(relationshipName).get(0).getTo(); + } } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java index c97cfe41..d5c05165 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/InMemoryThingRepository.java @@ -208,6 +208,18 @@ public Collection getConnectedItems( return instance.getRelationships().getConnectedItems(relationshipName); } + @Override + public List listRelatedInstances( + final EntityInstance instance, + final String relationshipName, + final QueryFilterParams queryParams) { + QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; + List instances = new ArrayList<>( + getConnectedItems(instance, relationshipName)); + instances = new EntityInstanceListFilter(params).filter(instances); + return new EntityInstanceListSorter(params).sort(instances); + } + private boolean matchesQueryIdentifier(final EntityInstance instance, final String identifier) { for (Field autoIncrementField : instance.getEntity().getFieldsOfType(FieldType.AUTO_INCREMENT)) { String idValue = instance.getFieldValue(autoIncrementField.getName()).asString(); diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java index 841a7f28..6765270c 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java @@ -31,6 +31,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -164,8 +165,7 @@ public List listInstances( getInstanceCollectionForEntityNamed(entity.getName()). getInstances()); } - instances = new EntityInstanceListFilter(params).filter(instances); - return new EntityInstanceListSorter(params).sort(instances); + return filterAndSort(instances, params); } SqlQuery sqlQuery = selectInstancesSql(entity, params); @@ -207,25 +207,32 @@ public EntityInstance findInstanceByQueryIdentifier( @Override public EntityInstance addInstance(final EntityInstance instance) { ensureSchemaReady(); - super.addInstance(instance); - persistInstance(instance); - persistCountersFor(instance.getEntity()); + runInTransaction(() -> { + super.addInstance(instance); + persistInstance(instance); + persistCountersFor(instance.getEntity()); + }); return instance; } @Override public EntityInstance updateInstance(final EntityInstance instance) { ensureSchemaReady(); - persistInstance(instance); - persistCountersFor(instance.getEntity()); + runInTransaction(() -> { + persistInstance(instance); + persistCountersFor(instance.getEntity()); + }); return instance; } @Override public void deleteEntityInstance(final EntityInstance instance) { + ensureSchemaReady(); if (!legacySnapshotLoaded) { - super.deleteEntityInstance(instance); - deletePersistedInstance(instance.getInternalId()); + runInTransaction(() -> { + super.deleteEntityInstance(instance); + deletePersistedInstance(instance.getInternalId()); + }); return; } @@ -233,40 +240,49 @@ public void deleteEntityInstance(final EntityInstance instance) { super.deleteEntityInstance(instance); Set idsAfterDelete = currentInternalIds(); - idsBeforeDelete.removeAll(idsAfterDelete); - for (String internalId : idsBeforeDelete) { - deletePersistedInstance(internalId); - } - replaceAllRelationshipRowsFromCache(); + runInTransaction(() -> { + idsBeforeDelete.removeAll(idsAfterDelete); + for (String internalId : idsBeforeDelete) { + deletePersistedInstance(internalId); + } + replaceAllRelationshipRowsFromCache(); + }); } @Override public void clearAllData() { + ensureSchemaReady(); super.clearAllData(); - for (EntityDefinition entity : schema.getEntityDefinitions()) { - executeSql("DELETE FROM " + table(entity)); - persistCountersFor(entity); - } - clearRelationshipRows(); + runInTransaction(() -> { + for (EntityDefinition entity : schema.getEntityDefinitions()) { + executeSql("DELETE FROM " + table(entity)); + persistCountersFor(entity); + } + clearRelationshipRows(); + }); } @Override public void clearInstanceDataFor(final String entityName) { + ensureSchemaReady(); EntityInstanceCollection collection = getInstanceCollectionForEntityNamed(entityName); if (collection == null) { return; } super.clearInstanceDataFor(entityName); - executeSql("DELETE FROM " + table(collection.definition())); - persistCountersFor(collection.definition()); - replaceAllRelationshipRowsFromCache(); + runInTransaction(() -> { + executeSql("DELETE FROM " + table(collection.definition())); + persistCountersFor(collection.definition()); + replaceAllRelationshipRowsFromCache(); + }); } @Override public void connectRelationship( final EntityInstance from, final String relationshipName, final EntityInstance to) { + ensureSchemaReady(); super.connectRelationship(from, relationshipName, to); - replaceAllRelationshipRowsFromCache(); + runInTransaction(() -> persistRelationship(from, relationshipName, to)); } @Override @@ -274,15 +290,17 @@ public List removeRelationshipsInvolving( final EntityInstance parent, final EntityInstance child, final String relationshipName) { + ensureSchemaReady(); List removed = super.removeRelationshipsInvolving(parent, child, relationshipName); - replaceAllRelationshipRowsFromCache(); + runInTransaction(() -> deleteRelationshipRowsInvolving(parent, child, relationshipName)); return removed; } @Override public List removeAllRelationships(final EntityInstance instance) { + ensureSchemaReady(); List removed = super.removeAllRelationships(instance); - replaceAllRelationshipRowsFromCache(); + runInTransaction(() -> deleteRelationshipRowsInvolving(instance)); return removed; } @@ -310,25 +328,57 @@ public Collection getConnectedItems( return connected; } + @Override + public List listRelatedInstances( + final EntityInstance instance, + final String relationshipName, + final QueryFilterParams queryParams) { + ensureSchemaReady(); + QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; + + if (hasJavaOnlyFilters(params)) { + return filterAndSort( + new ArrayList<>(getConnectedItems(instance, relationshipName)), + params); + } + + Map related = new LinkedHashMap<>(); + for (RelationshipVectorDefinition vector : + relationshipVectorsFor(instance, relationshipName)) { + SqlQuery query = selectRelatedInstancesSql(instance, vector, params); + if (query == null) { + continue; + } + for (EntityInstance item : queryEntityRows(query.entity, query.sql, query.parameters)) { + related.put(item.getInternalId(), item); + } + } + + return new EntityInstanceListSorter(params). + sort(new ArrayList<>(related.values())); + } + @Override public void flush() { if (!legacySnapshotLoaded) { return; } refreshSchema(schema); - for (EntityDefinition entity : schema.getEntityDefinitions()) { - executeSql("DELETE FROM " + table(entity)); - } - clearRelationshipRows(); + runInTransaction(() -> { + for (EntityDefinition entity : schema.getEntityDefinitions()) { + executeSql("DELETE FROM " + table(entity)); + } + clearRelationshipRows(); - for (EntityInstanceCollection collection : getAllInstanceCollections()) { - for (EntityInstance instance : collection.getInstances()) { - persistInstance(instance); + for (EntityInstanceCollection collection : getAllInstanceCollections()) { + for (EntityInstance instance : collection.getInstances()) { + persistInstance(instance); + } + persistCountersFor(collection.definition()); } - persistCountersFor(collection.definition()); - } - replaceAllRelationshipRowsFromCache(); + replaceAllRelationshipRowsFromCache(); + }); } @Override @@ -418,6 +468,18 @@ private void createRelationshipTable(final RelationshipVectorDefinition vector) "from_internal_id TEXT NOT NULL, " + "to_internal_id TEXT NOT NULL, " + "PRIMARY KEY(from_internal_id, to_internal_id))"); + createRelationshipIndex(vector, "from_internal_id"); + createRelationshipIndex(vector, "to_internal_id"); + } + + private void createRelationshipIndex( + final RelationshipVectorDefinition vector, final String columnName) { + String indexName = "__thingifier_rel_idx_" + + Integer.toHexString((relationshipTableName(vector) + "_" + columnName).hashCode()); + executeSql( + "CREATE INDEX IF NOT EXISTS " + identifier(indexName) + + " ON " + relationshipTable(vector) + + " (" + identifier(columnName) + ")"); } private void loadExistingData() { @@ -540,19 +602,97 @@ private SqlQuery selectInstancesSql( sql.append(" WHERE ").append(String.join(" AND ", whereClauses)); } + appendOrderBy(sql, entity, queryParams, ""); + + return new SqlQuery(entity, sql.toString(), parameters); + } + + private SqlQuery selectRelatedInstancesSql( + final EntityInstance instance, + final RelationshipVectorDefinition vector, + final QueryFilterParams queryParams) { + EntityDefinition connectedEntity; + String targetJoinColumn; + String sourceWhereColumn; + + if (vector.getFrom() == instance.getEntity()) { + connectedEntity = vector.getTo(); + targetJoinColumn = "to_internal_id"; + sourceWhereColumn = "from_internal_id"; + } else if (vector.getTo() == instance.getEntity()) { + connectedEntity = vector.getFrom(); + targetJoinColumn = "from_internal_id"; + sourceWhereColumn = "to_internal_id"; + } else { + return null; + } + + StringBuilder sql = new StringBuilder( + "SELECT target.* FROM " + relationshipTable(vector) + " rel " + + "JOIN " + table(connectedEntity) + " target " + + "ON target." + identifier(INTERNAL_ID_COLUMN) + + " = rel." + targetJoinColumn + + " WHERE rel." + sourceWhereColumn + " = ?"); + List parameters = new ArrayList<>(); + parameters.add(instance.getInternalId()); + + for (FilterBy filterBy : queryParams.toList()) { + if (EntityListSortParamParser.isSortByParam(filterBy.fieldName)) { + continue; + } + if (!connectedEntity.hasFieldNameDefined(filterBy.fieldName)) { + continue; + } + + String column = "target." + identifier(filterBy.fieldName); + switch (filterBy.filterOperation) { + case "=": + sql.append(" AND ").append(column).append(" = ?"); + parameters.add(filterBy.fieldValue); + break; + case "!": + case "!=": + sql.append(" AND ").append(column).append(" <> ?"); + parameters.add(filterBy.fieldValue); + break; + case "<": + case ">": + case "<=": + case ">=": + sql.append(" AND ").append(column).append(" ") + .append(filterBy.filterOperation).append(" ?"); + parameters.add(filterBy.fieldValue); + break; + case "*=": + sql.append(" AND ").append(column).append(" LIKE ? ESCAPE '\\'"); + parameters.add(sqlLikeWildcard(filterBy.fieldValue)); + break; + default: + break; + } + } + + appendOrderBy(sql, connectedEntity, queryParams, "target."); + + return new SqlQuery(connectedEntity, sql.toString(), parameters); + } + + private void appendOrderBy( + final StringBuilder sql, + final EntityDefinition entity, + final QueryFilterParams queryParams, + final String columnPrefix) { List orderClauses = new ArrayList<>(); for (SortByFieldName sortBy : new EntityListSortParamParser(queryParams).sortBys()) { if (!entity.hasFieldNameDefined(sortBy.getFieldName())) { continue; } - orderClauses.add(identifier(sortBy.getFieldName()) + + orderClauses.add(columnPrefix + identifier(sortBy.getFieldName()) + (sortBy.getOrder() < 0 ? " ASC" : " DESC")); } if (!orderClauses.isEmpty()) { sql.append(" ORDER BY ").append(String.join(", ", orderClauses)); } - - return new SqlQuery(sql.toString(), parameters); } private boolean hasJavaOnlyFilters(final QueryFilterParams queryParams) { @@ -638,6 +778,36 @@ private void addConnectedItemsFromVectorTable( connected.addAll(queryEntityRows(connectedEntity, sql, List.of(instance.getInternalId()))); } + private List relationshipVectorsFor( + final EntityInstance instance, final String relationshipName) { + List vectors = new ArrayList<>(); + Set seenTables = new HashSet<>(); + + for (RelationshipVectorDefinition vector : + instance.getEntity().related().getRelationships(relationshipName)) { + addRelationshipVectorIfNew(vectors, seenTables, vector); + if (vector.getRelationshipDefinition().isTwoWay()) { + RelationshipVectorDefinition otherVector = + vector.getRelationshipDefinition().otherVectorOf(vector); + if (otherVector != null) { + addRelationshipVectorIfNew(vectors, seenTables, otherVector); + } + } + } + + return vectors; + } + + private void addRelationshipVectorIfNew( + final List vectors, + final Set seenTables, + final RelationshipVectorDefinition vector) { + String tableName = relationshipTableName(vector); + if (seenTables.add(tableName)) { + vectors.add(vector); + } + } + private void persistInstance(final EntityInstance instance) { EntityDefinition entity = instance.getEntity(); List columns = new ArrayList<>(); @@ -675,6 +845,27 @@ private void persistInstance(final EntityInstance instance) { } } + private void persistRelationship( + final EntityInstance from, final String relationshipName, final EntityInstance to) { + RelationshipVectorDefinition vector = + from.getEntity().getNamedRelationshipTo(relationshipName, to.getEntity()); + if (vector == null) { + throw new IllegalArgumentException( + "Unknown relationship " + relationshipName + " between " + + from.getEntity().getName() + " and " + to.getEntity().getName()); + } + + String sql = "INSERT OR IGNORE INTO " + relationshipTable(vector) + + " (from_internal_id, to_internal_id) VALUES (?, ?)"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setString(1, from.getInternalId()); + statement.setString(2, to.getInternalId()); + statement.executeUpdate(); + } catch (SQLException e) { + throw new IllegalStateException("Could not persist SQLite relationship", e); + } + } + private void persistCountersFor(final EntityDefinition entity) { executeSql("DELETE FROM " + identifier(COUNTERS_TABLE) + " WHERE entity_name = " + quotedValue(entity.getName())); @@ -705,6 +896,30 @@ private void deletePersistedInstance(final String internalId) { } } + private void deleteRelationshipRowsInvolving( + final EntityInstance parent, + final EntityInstance child, + final String relationshipName) { + for (RelationshipVectorDefinition vector : relationshipVectors()) { + if (!vector.getRelationshipDefinition().isKnownAs(relationshipName)) { + continue; + } + executeSql("DELETE FROM " + relationshipTable(vector) + + " WHERE (from_internal_id = " + quotedValue(parent.getInternalId()) + + " AND to_internal_id = " + quotedValue(child.getInternalId()) + ")" + + " OR (from_internal_id = " + quotedValue(child.getInternalId()) + + " AND to_internal_id = " + quotedValue(parent.getInternalId()) + ")"); + } + } + + private void deleteRelationshipRowsInvolving(final EntityInstance instance) { + for (RelationshipVectorDefinition vector : relationshipVectors()) { + executeSql("DELETE FROM " + relationshipTable(vector) + + " WHERE from_internal_id = " + quotedValue(instance.getInternalId()) + + " OR to_internal_id = " + quotedValue(instance.getInternalId())); + } + } + private void replaceAllRelationshipRowsFromCache() { clearRelationshipRows(); Set persisted = new HashSet<>(); @@ -765,6 +980,13 @@ private List relationshipVectors() { return vectors; } + private List filterAndSort( + final List instances, + final QueryFilterParams params) { + List filtered = new EntityInstanceListFilter(params).filter(instances); + return new EntityInstanceListSorter(params).sort(filtered); + } + private Set columnNames(final String tableName) { Set names = new HashSet<>(); String sql = "PRAGMA table_info(" + identifier(tableName) + ")"; @@ -787,6 +1009,24 @@ private void executeSql(final String sql) { } } + private void runInTransaction(final Runnable operation) { + try { + boolean originalAutoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + try { + operation.run(); + connection.commit(); + } catch (RuntimeException e) { + connection.rollback(); + throw e; + } finally { + connection.setAutoCommit(originalAutoCommit); + } + } catch (SQLException e) { + throw new IllegalStateException("Could not run SQLite transaction", e); + } + } + private String sqlType(final FieldType type) { switch (type) { case AUTO_INCREMENT: @@ -814,11 +1054,14 @@ private String entityTableName(final EntityDefinition entity) { } private String relationshipTable(final RelationshipVectorDefinition vector) { - return identifier( - "thing_rel_" + - vector.getFrom().getName() + "_" + - vector.getName() + "_" + - vector.getTo().getName()); + return identifier(relationshipTableName(vector)); + } + + private String relationshipTableName(final RelationshipVectorDefinition vector) { + return "thing_rel_" + + vector.getFrom().getName() + "_" + + vector.getName() + "_" + + vector.getTo().getName(); } private String identifier(final String rawIdentifier) { @@ -830,10 +1073,15 @@ private String quotedValue(final String value) { } private static class SqlQuery { + private final EntityDefinition entity; private final String sql; private final List parameters; - private SqlQuery(final String sql, final List parameters) { + private SqlQuery( + final EntityDefinition entity, + final String sql, + final List parameters) { + this.entity = entity; this.sql = sql; this.parameters = parameters; } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java index 2c0461c3..049e30bb 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepository.java @@ -22,6 +22,7 @@ public interface ThingRepository extends AutoCloseable { void refreshSchema(ERSchema schema); + @Deprecated ERInstanceData getInstanceData(); EntityInstanceCollection createInstanceCollectionFor(EntityDefinition definition); @@ -30,6 +31,7 @@ public interface ThingRepository extends AutoCloseable { List getAllInstanceCollections(); + @Deprecated EntityInstanceCollection getInstanceCollectionForEntityNamed(String entityName); EntityInstance findEntityInstanceByGUID(String thingGUID); @@ -44,6 +46,14 @@ public interface ThingRepository extends AutoCloseable { EntityInstance findInstanceByQueryIdentifier(EntityDefinition entity, String identifier); + default List listRelatedInstances( + EntityInstance instance, String relationshipName) { + return listRelatedInstances(instance, relationshipName, new QueryFilterParams()); + } + + List listRelatedInstances( + EntityInstance instance, String relationshipName, QueryFilterParams queryParams); + EntityInstance addInstance(EntityInstance instance); EntityInstance updateInstance(EntityInstance instance); diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/EntityRelModelTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/EntityRelModelTest.java index 49f05091..7f39fb19 100644 --- a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/EntityRelModelTest.java +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/EntityRelModelTest.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import uk.co.compendiumdev.thingifier.core.domain.datapopulator.DataPopulator; +import uk.co.compendiumdev.thingifier.core.domain.datapopulator.RepositoryDataPopulator; import uk.co.compendiumdev.thingifier.core.domain.definitions.Cardinality; import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; @@ -12,6 +13,11 @@ import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; +import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepositoryProvider; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; + +import java.util.ArrayList; +import java.util.List; public class EntityRelModelTest { @@ -245,4 +251,38 @@ public void populate(final ERSchema schema, final ERInstanceData database) { Assertions.assertEquals(1, erm.getInstanceData().getAllInstanceCollections().size()); } + + @Test + public void populateDatabaseUsesRepositoryDataPopulatorWithoutCompatibilitySnapshot(){ + try (EntityRelModel erm = new EntityRelModel(SqliteThingRepositoryProvider.inMemory())) { + EntityDefinition thing = erm.createEntityDefinition("thing", "things"); + thing.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + thing.addField(Field.is("name", FieldType.STRING)); + + RepositoryDataPopulator dataPopulator = new RepositoryDataPopulator() { + @Override + public void populate(final ERSchema schema, final ERInstanceData database) { + Assertions.fail("Repository-aware data populators should not use ERInstanceData"); + } + + @Override + public void populate(final ERSchema schema, final ThingRepository repository) { + repository.addInstance(new EntityInstance( + schema.getEntityDefinitionNamed("thing")). + setValue("name", "repository")); + } + }; + + erm.setDataGenerator(dataPopulator); + + Assertions.assertTrue(erm.populateDatabase(EntityRelModel.DEFAULT_DATABASE_NAME)); + + ThingRepository repository = erm.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME); + List instances = new ArrayList<>(repository.listInstances(thing)); + + Assertions.assertEquals(1, instances.size()); + Assertions.assertEquals("repository", instances.get(0).getFieldValue("name").asString()); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + } + } } diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java index 20943497..4b54176d 100644 --- a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java @@ -12,6 +12,7 @@ import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; +import uk.co.compendiumdev.thingifier.core.query.RepositoryBackedSimpleQuery; import java.nio.file.Path; import java.util.List; @@ -158,6 +159,57 @@ public void sqliteFileBackedRepositoryOnlyHydratesCompatibilitySnapshotWhenReque } } + @Test + public void sqliteRelationshipReadsDoNotHydrateCompatibilitySnapshot() { + ERSchema schema = todoSchema(); + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + EntityDefinition taskDefinition = schema.getEntityDefinitionNamed("task"); + Path databasePath = tempDir.resolve("relationship-lazy.sqlite"); + + try (ThingRepository repository = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + repository.initializeFrom(schema); + + EntityInstance project = repository.addInstance( + new EntityInstance(projectDefinition).setValue("title", "SQLite project")); + EntityInstance firstTask = repository.addInstance( + new EntityInstance(taskDefinition).setValue("title", "Wire repository")); + EntityInstance secondTask = repository.addInstance( + new EntityInstance(taskDefinition).setValue("title", "Write tests")); + + repository.connectRelationship(project, "tasks", firstTask); + repository.connectRelationship(project, "tasks", secondTask); + } + + try (ThingRepository reopened = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + reopened.initializeFrom(schema); + + Assertions.assertFalse(reopened.hasLoadedCompatibilitySnapshot()); + + EntityInstance project = reopened.findInstanceByPrimaryKey(projectDefinition, "1"); + + QueryFilterParams params = new QueryFilterParams(); + params.put("title", "*=Wire*"); + + List filteredTasks = + reopened.listRelatedInstances(project, "tasks", params); + + Assertions.assertEquals(1, filteredTasks.size()); + Assertions.assertEquals("Wire repository", + filteredTasks.get(0).getFieldValue("title").asString()); + Assertions.assertFalse(reopened.hasLoadedCompatibilitySnapshot()); + + RepositoryBackedSimpleQuery query = + new RepositoryBackedSimpleQuery(schema, reopened, "project/1/tasks"). + performQuery(new QueryFilterParams()); + + Assertions.assertTrue(query.isResultACollection()); + Assertions.assertEquals(2, query.getListEntityInstances().size()); + Assertions.assertFalse(reopened.hasLoadedCompatibilitySnapshot()); + } + } + private void exerciseRepositoryContract(final ThingRepository repository) { ERSchema schema = todoSchema(); repository.initializeFrom(schema); @@ -209,17 +261,49 @@ private void exerciseRepositoryContract(final ThingRepository repository) { EntityInstance task = new EntityInstance(taskDefinition); task.setValue("title", "Wire repository"); repository.addInstance(task); - repository.connectRelationship(task, "task-of", project); + repository.connectRelationship(project, "tasks", task); + + EntityInstance secondTask = new EntityInstance(taskDefinition); + secondTask.setValue("title", "Write tests"); + repository.addInstance(secondTask); + repository.connectRelationship(project, "tasks", secondTask); Assertions.assertTrue(repository.getConnectedItems(task, "task-of").contains(project)); Assertions.assertTrue(repository.getConnectedItems(project, "tasks").contains(task)); - repository.removeRelationshipsInvolving(task, project, "task-of"); + Assertions.assertTrue( + repository.listRelatedInstances(task, "task-of").contains(project)); + Assertions.assertTrue( + repository.listRelatedInstances(project, "tasks").contains(task)); + Assertions.assertTrue( + repository.listRelatedInstances(project, "tasks").contains(secondTask)); + + QueryFilterParams relationshipFilterParams = new QueryFilterParams(); + relationshipFilterParams.put("title", "*=Wire*"); + List filteredTasks = + repository.listRelatedInstances(project, "tasks", relationshipFilterParams); + Assertions.assertEquals(1, filteredTasks.size()); + Assertions.assertEquals(task, filteredTasks.get(0)); + + QueryFilterParams relationshipSortParams = new QueryFilterParams(); + relationshipSortParams.put("sortBy", "-id"); + List sortedTasks = + repository.listRelatedInstances(project, "tasks", relationshipSortParams); + Assertions.assertEquals("2", sortedTasks.get(0).getPrimaryKeyValue()); + Assertions.assertEquals("1", sortedTasks.get(1).getPrimaryKeyValue()); + + repository.removeRelationshipsInvolving(project, task, "tasks"); Assertions.assertTrue(repository.getConnectedItems(task, "task-of").isEmpty()); + Assertions.assertFalse(repository.getConnectedItems(project, "tasks").contains(task)); + Assertions.assertTrue(repository.getConnectedItems(project, "tasks").contains(secondTask)); + + repository.removeRelationshipsInvolving(project, secondTask, "tasks"); + Assertions.assertTrue(repository.getConnectedItems(project, "tasks").isEmpty()); repository.deleteEntityInstance(task); + repository.deleteEntityInstance(secondTask); Assertions.assertEquals(0, repository.listInstances(taskDefinition).size()); diff --git a/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/data/TodoAPIDataPopulator.java b/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/data/TodoAPIDataPopulator.java index a28df1ed..9f4e6ea4 100644 --- a/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/data/TodoAPIDataPopulator.java +++ b/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/data/TodoAPIDataPopulator.java @@ -1,31 +1,43 @@ package uk.co.compendiumdev.thingifier.application.data; +import uk.co.compendiumdev.thingifier.core.domain.datapopulator.RepositoryDataPopulator; import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; -import uk.co.compendiumdev.thingifier.core.domain.datapopulator.DataPopulator; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; -public class TodoAPIDataPopulator implements DataPopulator { +public class TodoAPIDataPopulator implements RepositoryDataPopulator { + + private static final String[] TODOS = { + "scan paperwork", + "file paperwork", + "process payments", + "escalate late payments", + "pay invoices", + "process payroll", + "train staff", + "schedule meeting"}; @Override public void populate(final ERSchema schema, final ERInstanceData database) { - String [] todos={ - "scan paperwork", - "file paperwork", - "process payments", - "escalate late payments", - "pay invoices", - "process payroll", - "train staff", - "schedule meeting"}; - EntityInstanceCollection todo = database.getInstanceCollectionForEntityNamed("todo"); - for(String todoItem : todos){ + for(String todoItem : TODOS){ todo.addInstance(new EntityInstance(todo.definition())). setValue("title", todoItem); } } + + @Override + public void populate(final ERSchema schema, final ThingRepository repository) { + EntityDefinition todo = schema.getEntityDefinitionNamed("todo"); + + for (String todoItem : TODOS) { + repository.addInstance(new EntityInstance(todo). + setValue("title", todoItem)); + } + } } diff --git a/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/data/TodoManagerAPIDataPopulator.java b/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/data/TodoManagerAPIDataPopulator.java index a34aef69..d0146087 100644 --- a/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/data/TodoManagerAPIDataPopulator.java +++ b/examplemodels/src/main/java/uk/co/compendiumdev/thingifier/application/data/TodoManagerAPIDataPopulator.java @@ -1,12 +1,14 @@ package uk.co.compendiumdev.thingifier.application.data; +import uk.co.compendiumdev.thingifier.core.domain.datapopulator.RepositoryDataPopulator; import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; import uk.co.compendiumdev.thingifier.core.domain.instances.ERInstanceData; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; -import uk.co.compendiumdev.thingifier.core.domain.datapopulator.DataPopulator; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; -public class TodoManagerAPIDataPopulator implements DataPopulator { +public class TodoManagerAPIDataPopulator implements RepositoryDataPopulator { @Override public void populate(final ERSchema schema, final ERInstanceData database) { @@ -37,4 +39,31 @@ public void populate(final ERSchema schema, final ERInstanceData database) { paperwork.getRelationships().connect("categories", officeCategory); } + + @Override + public void populate(final ERSchema schema, final ThingRepository repository) { + EntityDefinition todo = schema.getEntityDefinitionNamed("todo"); + EntityDefinition category = schema.getEntityDefinitionNamed("category"); + EntityDefinition project = schema.getEntityDefinitionNamed("project"); + + EntityInstance paperwork = repository.addInstance(new EntityInstance(todo). + setValue("title", "scan paperwork")); + + EntityInstance filework = repository.addInstance(new EntityInstance(todo). + setValue("title", "file paperwork")); + + EntityInstance officeCategory = repository.addInstance(new EntityInstance(category). + setValue("title", "Office")); + + repository.addInstance(new EntityInstance(category). + setValue("title", "Home")); + + EntityInstance officeWork = repository.addInstance(new EntityInstance(project). + setValue("title", "Office Work")); + + repository.connectRelationship(officeWork, "tasks", paperwork); + repository.connectRelationship(officeWork, "tasks", filework); + + repository.connectRelationship(paperwork, "categories", officeCategory); + } } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java index a9c1b8dc..86e1b7da 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/Thingifier.java @@ -7,6 +7,7 @@ import uk.co.compendiumdev.thingifier.apiconfig.ThingifierApiConfigProfiles; import uk.co.compendiumdev.thingifier.core.EntityRelModel; import uk.co.compendiumdev.thingifier.api.ermodelconversion.JsonPopulator; +import uk.co.compendiumdev.thingifier.core.domain.datapopulator.RepositoryDataPopulator; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; import uk.co.compendiumdev.thingifier.core.domain.datapopulator.DataPopulator; import uk.co.compendiumdev.thingifier.core.domain.definitions.*; @@ -133,13 +134,34 @@ public EntityInstance findThingInstanceByGuid(final String thingGUID, final Stri return erm.getRepository(database).findEntityInstanceByGUID(thingGUID); } - + @Deprecated public EntityInstanceCollection getThingInstancesNamed(final String aName, final String database) { return erm.getRepository(database).getInstanceCollectionForEntityNamed(aName); } + public List listThingInstancesNamed(final String aName, final String database) { + EntityDefinition definition = erm.getSchema().getDefinitionWithSingularOrPluralNamed(aName); + ThingRepository repository = erm.getRepository(database); + if (definition == null || repository == null) { + return Collections.emptyList(); + } + return new ArrayList<>(repository.listInstances(definition)); + } + + public EntityInstance findThingInstanceByFieldNameAndValue( + final String entityName, + final String fieldName, + final String fieldValue, + final String database) { + EntityDefinition definition = erm.getSchema().getDefinitionWithSingularOrPluralNamed(entityName); + ThingRepository repository = erm.getRepository(database); + if (definition == null || repository == null) { + return null; + } + return repository.findInstanceByFieldNameAndValue(definition, fieldName, fieldValue); + } - + @Deprecated public EntityInstanceCollection getInstancesForSingularOrPluralNamedEntity(final String term, final String database) { final EntityDefinition defn = erm.getSchema().getDefinitionWithSingularOrPluralNamed(term); if(defn!=null){ @@ -173,9 +195,12 @@ public void deleteThing(final EntityInstance aThingInstance, final String databa // data generation public void generateData(final String database) { if(dataPopulator!=null){ - erm.getRepository(database).refreshSchema(erm.getSchema()); - dataPopulator.populate(erm.getSchema(), erm.getInstanceData(database)); - erm.getRepository(database).flush(); + ThingRepository repository = erm.getRepository(database); + if (repository == null) { + return; + } + repository.refreshSchema(erm.getSchema()); + populateRepository(repository); } } @@ -272,13 +297,9 @@ public void ensureCreatedAndPopulatedInstanceDatabaseNamed(String databaseName) // if we created it then populate it if(getDefaultDataPopulator()!=null){ // Use any default data populator to populate the new database - getERmodel().getRepository(databaseName).refreshSchema(getERmodel().getSchema()); - getDefaultDataPopulator(). - populate( - getERmodel().getSchema(), - getERmodel().getRepository(databaseName).getInstanceData() - ); - getERmodel().getRepository(databaseName).flush(); + ThingRepository repository = getERmodel().getRepository(databaseName); + repository.refreshSchema(getERmodel().getSchema()); + populateRepository(repository); } } } @@ -297,4 +318,14 @@ public void ensureCreatedAndPopulatedInstanceDatabaseFromJson(String databaseNam public ApiDocsConfig apidocsconfig() { return apiDocsConfig; } + + private void populateRepository(final ThingRepository repository) { + if (dataPopulator instanceof RepositoryDataPopulator) { + ((RepositoryDataPopulator) dataPopulator).populate(erm.getSchema(), repository); + return; + } + + dataPopulator.populate(erm.getSchema(), repository.getInstanceData()); + repository.flush(); + } } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/DefaultGuiHtmlPages.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/DefaultGuiHtmlPages.java index b2aa114c..82c93496 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/DefaultGuiHtmlPages.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/DefaultGuiHtmlPages.java @@ -12,9 +12,10 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; import uk.co.compendiumdev.thingifier.core.domain.definitions.relationship.RelationshipVectorDefinition; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; -import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; -import java.util.Collection; +import java.util.ArrayList; +import java.util.List; import java.util.Map; public class DefaultGuiHtmlPages { @@ -119,16 +120,20 @@ public String getInstancesListPage(String database, String entityName) { htmlErrorMessage = htmlErrorMessage + "

Database Named " + HtmlUtils.sanitise(database) + " not found. Have you made any API Calls?" + tryDefault + "

"; } - EntityInstanceCollection thing = null; + EntityDefinition definition = thingifier.getERmodel(). + getSchema(). + getDefinitionWithSingularOrPluralNamed(entityName); + List instances = new ArrayList<>(); if (htmlErrorMessage.isEmpty()) { try { - thing = thingifier.getThingInstancesNamed(entityName, database); + ThingRepository repository = thingifier.getRepository(database); + instances = new ArrayList<>(repository.listInstances(definition)); } catch (Exception e) { //htmlErrorMessage = htmlErrorMessage + "

Database Access Error: " + e.getMessage() + ".

"; } - if (thing == null) { + if (definition == null) { htmlErrorMessage = htmlErrorMessage + "

Entity instances not found in database, have you made any API calls?" + tryDefault + "

"; } } @@ -141,13 +146,11 @@ public String getInstancesListPage(String database, String entityName) { try { - final EntityDefinition definition = thing.definition(); - html.append("

" + definition.getPlural() + "

"); html.append(startHtmlTableFor(definition)); - for (EntityInstance instance : thing.getInstances()) { + for (EntityInstance instance : instances) { html.append(htmlTableRowFor(instance, database)); } @@ -276,16 +279,19 @@ public String getInstanceDetailsPage(String database, String entityName, MapDatabase Access Error: " + e.getMessage() + ".

"; } - if(thing == null){ + if(definition == null || repository == null){ htmlErrorMessage = htmlErrorMessage + "

Entity instances not found in database, have you made any API calls?" + tryDefault + "

"; } } @@ -297,9 +303,11 @@ public String getInstanceDetailsPage(String database, String entityName, MapInstances not found in database, have you made any API calls?" + tryDefault + "

"; } @@ -324,7 +333,6 @@ public String getInstanceDetailsPage(String database, String entityName, Map"); - if (instance.getRelationships().hasAnyRelationshipInstances()) { + if (!definition.related().getRelationships().isEmpty()) { html.append("

Relationships

"); for (RelationshipVectorDefinition relationship : definition.related().getRelationships()) { - final Collection relatedItems = instance.getRelationships().getConnectedItems(relationship.getName()); + final List relatedItems = + repository.listRelatedInstances(instance, relationship.getName()); html.append("

" + relationship.getName() + "

"); if (!relatedItems.isEmpty()) { boolean header = true; diff --git a/thingifier/src/test/java/uk/co/compendiumdev/casestudy/todomanager/api_non_http/RelationshipApiSqliteRepositoryTest.java b/thingifier/src/test/java/uk/co/compendiumdev/casestudy/todomanager/api_non_http/RelationshipApiSqliteRepositoryTest.java new file mode 100644 index 00000000..60379606 --- /dev/null +++ b/thingifier/src/test/java/uk/co/compendiumdev/casestudy/todomanager/api_non_http/RelationshipApiSqliteRepositoryTest.java @@ -0,0 +1,70 @@ +package uk.co.compendiumdev.casestudy.todomanager.api_non_http; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import uk.co.compendiumdev.thingifier.Thingifier; +import uk.co.compendiumdev.thingifier.api.http.headers.HttpHeadersBlock; +import uk.co.compendiumdev.thingifier.api.response.ApiResponse; +import uk.co.compendiumdev.thingifier.core.EntityRelModel; +import uk.co.compendiumdev.thingifier.core.domain.definitions.Cardinality; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; +import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepositoryProvider; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; + +public class RelationshipApiSqliteRepositoryTest { + + @Test + public void getRelationshipPathsUseRepositoryWithoutLoadingCompatibilitySnapshot() { + try (Thingifier todoManager = new Thingifier( + new EntityRelModel(SqliteThingRepositoryProvider.inMemory()))) { + EntityDefinition todo = todoManager.defineThing("todo", "todos"); + todo.addAsPrimaryKeyField(Field.is("guid", FieldType.AUTO_GUID)); + todo.addField(Field.is("title", FieldType.STRING)); + + EntityDefinition project = todoManager.defineThing("project", "projects"); + project.addAsPrimaryKeyField(Field.is("guid", FieldType.AUTO_GUID)); + project.addField(Field.is("title", FieldType.STRING)); + + todoManager.defineRelationship(project, todo, "tasks", Cardinality.ONE_TO_MANY()). + whenReversed(Cardinality.ONE_TO_MANY(), "task-of"); + + ThingRepository repository = + todoManager.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME); + + EntityInstance task = repository.addInstance(new EntityInstance(todo). + setValue("title", "SQLite relationship task")); + EntityInstance projectInstance = repository.addInstance(new EntityInstance(project). + setValue("title", "SQLite relationship project")); + + repository.connectRelationship(projectInstance, "tasks", task); + + ApiResponse tasksResponse = todoManager.api().get( + String.format("project/%s/tasks", projectInstance.getPrimaryKeyValue()), + new QueryFilterParams(), + new HttpHeadersBlock()); + + Assertions.assertEquals(200, tasksResponse.getStatusCode()); + Assertions.assertTrue(tasksResponse.isCollection()); + Assertions.assertEquals(1, tasksResponse.getReturnedInstanceCollection().size()); + Assertions.assertEquals(task.getPrimaryKeyValue(), + tasksResponse.getReturnedInstanceCollection().get(0).getPrimaryKeyValue()); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + + ApiResponse projectResponse = todoManager.api().get( + String.format("todo/%s/task-of", task.getPrimaryKeyValue()), + new QueryFilterParams(), + new HttpHeadersBlock()); + + Assertions.assertEquals(200, projectResponse.getStatusCode()); + Assertions.assertTrue(projectResponse.isCollection()); + Assertions.assertEquals(1, projectResponse.getReturnedInstanceCollection().size()); + Assertions.assertEquals(projectInstance.getPrimaryKeyValue(), + projectResponse.getReturnedInstanceCollection().get(0).getPrimaryKeyValue()); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + } + } +} diff --git a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/DefaultGuiHtmlPagesRepositoryTest.java b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/DefaultGuiHtmlPagesRepositoryTest.java new file mode 100644 index 00000000..6b16428f --- /dev/null +++ b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/DefaultGuiHtmlPagesRepositoryTest.java @@ -0,0 +1,63 @@ +package uk.co.compendiumdev.thingifier.htmlgui.htmlgen; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import uk.co.compendiumdev.thingifier.Thingifier; +import uk.co.compendiumdev.thingifier.core.EntityRelModel; +import uk.co.compendiumdev.thingifier.core.domain.definitions.Cardinality; +import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; +import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; +import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepositoryProvider; +import uk.co.compendiumdev.thingifier.core.repository.ThingRepository; + +import java.util.Map; + +public class DefaultGuiHtmlPagesRepositoryTest { + + @Test + public void guiInstancePagesReadFromRepositoryWithoutLoadingCompatibilitySnapshot() { + try (Thingifier thingifier = new Thingifier( + new EntityRelModel(SqliteThingRepositoryProvider.inMemory()))) { + + EntityDefinition project = thingifier.defineThing("project", "projects"); + project.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + project.addField(Field.is("title", FieldType.STRING)); + + EntityDefinition todo = thingifier.defineThing("todo", "todos"); + todo.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + todo.addField(Field.is("title", FieldType.STRING)); + + thingifier.defineRelationship(project, todo, "tasks", Cardinality.ONE_TO_MANY()). + whenReversed(Cardinality.ONE_TO_MANY(), "tasksof"); + + ThingRepository repository = + thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME); + + EntityInstance projectInstance = repository.addInstance(new EntityInstance(project). + setValue("title", "Repository Project")); + EntityInstance todoInstance = repository.addInstance(new EntityInstance(todo). + setValue("title", "Repository Todo")); + + repository.connectRelationship(projectInstance, "tasks", todoInstance); + + DefaultGuiHtmlPages pages = new DefaultGuiHtmlPages( + new DefaultGUIHTML(), thingifier, "/gui"); + + String listHtml = pages.getInstancesListPage( + EntityRelModel.DEFAULT_DATABASE_NAME, "todo"); + Assertions.assertTrue(listHtml.contains("Repository Todo"), listHtml); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + + String detailHtml = pages.getInstanceDetailsPage( + EntityRelModel.DEFAULT_DATABASE_NAME, + "project", + Map.of("id", projectInstance.getPrimaryKeyValue())); + + Assertions.assertTrue(detailHtml.contains("Repository Project"), detailHtml); + Assertions.assertTrue(detailHtml.contains("Repository Todo"), detailHtml); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + } + } +} From df824b0b313b39c4e13a45d99a1a73d4e57f2991 Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 16:19:40 +0100 Subject: [PATCH 08/11] redirect to ui --- .../uk/co/compendiumdev/todolist/sqlite/application/Main.java | 2 +- .../java/uk/co/compendiumdev/todolist/application/Main.java | 2 +- .../java/uk/co/compendiumdev/todolist/application/Main.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/standAloneSqlLiteTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/sqlite/application/Main.java b/standAloneSqlLiteTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/sqlite/application/Main.java index dd3e5035..094e1654 100644 --- a/standAloneSqlLiteTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/sqlite/application/Main.java +++ b/standAloneSqlLiteTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/sqlite/application/Main.java @@ -25,7 +25,7 @@ public static void main(String[] args) { app.setupDefaultGui(); get("/", (request, response) -> { - response.redirect("/gui"); + response.redirect("/gui/entities"); return ""; }); diff --git a/standAloneTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/application/Main.java b/standAloneTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/application/Main.java index 656d6f3b..88476646 100644 --- a/standAloneTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/application/Main.java +++ b/standAloneTodoListManagerRestApi/src/main/java/uk/co/compendiumdev/todolist/application/Main.java @@ -26,7 +26,7 @@ public static void main(String[] args) { app.setupDefaultGui(); get("/", (request, response) -> { - response.redirect("/gui"); + response.redirect("/gui/entities"); return ""; }); diff --git a/standAloneTodoListRestApi/src/main/java/uk/co/compendiumdev/todolist/application/Main.java b/standAloneTodoListRestApi/src/main/java/uk/co/compendiumdev/todolist/application/Main.java index 51dcdce6..00b73ccf 100644 --- a/standAloneTodoListRestApi/src/main/java/uk/co/compendiumdev/todolist/application/Main.java +++ b/standAloneTodoListRestApi/src/main/java/uk/co/compendiumdev/todolist/application/Main.java @@ -27,7 +27,7 @@ public static void main(String[] args) { app.setupDefaultGui(); get("/", (request, response) -> { - response.redirect("/gui"); + response.redirect("/gui/entities"); return ""; }); From 557973e9738068a9d577a0a205096866655737c0 Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 17:11:16 +0100 Subject: [PATCH 09/11] Add SQLite regex-to-LIKE conversion --- .../SqliteRegexToLikeConverter.java | 156 ++++++++++++++++++ .../repository/SqliteThingRepository.java | 153 +++++++++++------ .../SqliteRegexToLikeConverterTest.java | 105 ++++++++++++ .../ThingRepositoryContractTest.java | 70 ++++++++ 4 files changed, 432 insertions(+), 52 deletions(-) create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexToLikeConverter.java create mode 100644 ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexToLikeConverterTest.java diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexToLikeConverter.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexToLikeConverter.java new file mode 100644 index 00000000..416913ca --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexToLikeConverter.java @@ -0,0 +1,156 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +final class SqliteRegexToLikeConverter { + + private SqliteRegexToLikeConverter() { + } + + static Conversion convert(final String regex) { + validateRegex(regex); + + String pattern = regex; + if (pattern.startsWith("^")) { + pattern = pattern.substring(1); + } + if (hasTrailingUnescapedDollar(pattern)) { + pattern = pattern.substring(0, pattern.length() - 1); + } + + StringBuilder exactValue = new StringBuilder(); + StringBuilder likeValue = new StringBuilder(); + boolean usesLikeWildcard = false; + + for (int index = 0; index < pattern.length(); index++) { + char current = pattern.charAt(index); + + if (current == '\\') { + if (index == pattern.length() - 1) { + throw new RegexToLikeConversionException("Trailing regex escape"); + } + char escaped = pattern.charAt(++index); + if (isUnsupportedEscape(escaped)) { + throw new RegexToLikeConversionException("Unsupported regex escape"); + } + appendLiteral(exactValue, likeValue, escaped); + continue; + } + + if (current == '.') { + if (index < pattern.length() - 1 && pattern.charAt(index + 1) == '*') { + likeValue.append('%'); + usesLikeWildcard = true; + index++; + } else { + likeValue.append('_'); + usesLikeWildcard = true; + } + continue; + } + + if (isUnsupportedRegexMeta(current)) { + throw new RegexToLikeConversionException("Unsupported regex operator"); + } + + appendLiteral(exactValue, likeValue, current); + } + + if (usesLikeWildcard) { + return Conversion.like(likeValue.toString()); + } + return Conversion.equalTo(exactValue.toString()); + } + + private static void validateRegex(final String regex) { + if (regex == null) { + throw new RegexToLikeConversionException("Regex value is null"); + } + try { + Pattern.compile(regex); + } catch (PatternSyntaxException e) { + throw new RegexToLikeConversionException("Invalid regex", e); + } + } + + private static void appendLiteral( + final StringBuilder exactValue, + final StringBuilder likeValue, + final char literal) { + exactValue.append(literal); + if (literal == '\\' || literal == '%' || literal == '_') { + likeValue.append('\\'); + } + likeValue.append(literal); + } + + private static boolean isUnsupportedEscape(final char escaped) { + return Character.isLetterOrDigit(escaped); + } + + private static boolean isUnsupportedRegexMeta(final char current) { + return current == '[' || + current == ']' || + current == '(' || + current == ')' || + current == '{' || + current == '}' || + current == '+' || + current == '?' || + current == '*' || + current == '|' || + current == '^' || + current == '$'; + } + + private static boolean hasTrailingUnescapedDollar(final String pattern) { + if (pattern.isEmpty() || pattern.charAt(pattern.length() - 1) != '$') { + return false; + } + + int backslashes = 0; + for (int index = pattern.length() - 2; + index >= 0 && pattern.charAt(index) == '\\'; + index--) { + backslashes++; + } + return backslashes % 2 == 0; + } + + static final class Conversion { + private final boolean equality; + private final String sqlValue; + + private Conversion(final boolean equality, final String sqlValue) { + this.equality = equality; + this.sqlValue = sqlValue; + } + + static Conversion equalTo(final String sqlValue) { + return new Conversion(true, sqlValue); + } + + static Conversion like(final String sqlValue) { + return new Conversion(false, sqlValue); + } + + boolean isEquality() { + return equality; + } + + String sqlValue() { + return sqlValue; + } + } + + static final class RegexToLikeConversionException extends RuntimeException { + RegexToLikeConversionException(final String message) { + super(message); + } + + RegexToLikeConversionException(final String message, final Throwable cause) { + super(message, cause); + } + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java index 6765270c..79d16c6e 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java @@ -36,9 +36,12 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.logging.Logger; public class SqliteThingRepository extends InMemoryThingRepository { + private static final Logger LOGGER = + Logger.getLogger(SqliteThingRepository.class.getName()); private static final String INTERNAL_ID_COLUMN = "__internal_id"; private static final String COUNTERS_TABLE = "__thingifier_counters"; @@ -158,18 +161,16 @@ public List listInstances( final EntityDefinition entity, final QueryFilterParams queryParams) { ensureSchemaReady(); QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; - if (hasJavaOnlyFilters(params)) { - List instances = new ArrayList<>(super.listInstances(entity)); - if (!legacySnapshotLoaded) { - instances = new ArrayList<>(getInstanceData(). - getInstanceCollectionForEntityNamed(entity.getName()). - getInstances()); - } - return filterAndSort(instances, params); - } - SqlQuery sqlQuery = selectInstancesSql(entity, params); - return queryEntityRows(entity, sqlQuery.sql, sqlQuery.parameters); + try { + SqlQuery sqlQuery = selectInstancesSql(entity, params); + return queryEntityRows(entity, sqlQuery.sql, sqlQuery.parameters); + } catch (SqliteRegexToLikeConverter.RegexToLikeConversionException e) { + LOGGER.warning("SQLite regex filter for " + entity.getName() + + " could not be converted to LIKE; using Java compatibility filtering: " + + e.getMessage()); + return listInstancesUsingJavaFilter(entity, params); + } } @Override @@ -336,26 +337,29 @@ public List listRelatedInstances( ensureSchemaReady(); QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; - if (hasJavaOnlyFilters(params)) { + try { + Map related = new LinkedHashMap<>(); + for (RelationshipVectorDefinition vector : + relationshipVectorsFor(instance, relationshipName)) { + SqlQuery query = selectRelatedInstancesSql(instance, vector, params); + if (query == null) { + continue; + } + for (EntityInstance item : queryEntityRows(query.entity, query.sql, query.parameters)) { + related.put(item.getInternalId(), item); + } + } + + return new EntityInstanceListSorter(params). + sort(new ArrayList<>(related.values())); + } catch (SqliteRegexToLikeConverter.RegexToLikeConversionException e) { + LOGGER.warning("SQLite regex relationship filter for " + relationshipName + + " could not be converted to LIKE; using Java compatibility filtering: " + + e.getMessage()); return filterAndSort( new ArrayList<>(getConnectedItems(instance, relationshipName)), params); } - - Map related = new LinkedHashMap<>(); - for (RelationshipVectorDefinition vector : - relationshipVectorsFor(instance, relationshipName)) { - SqlQuery query = selectRelatedInstancesSql(instance, vector, params); - if (query == null) { - continue; - } - for (EntityInstance item : queryEntityRows(query.entity, query.sql, query.parameters)) { - related.put(item.getInternalId(), item); - } - } - - return new EntityInstanceListSorter(params). - sort(new ArrayList<>(related.values())); } @Override @@ -405,6 +409,7 @@ private void openConnection() { try { Class.forName("org.sqlite.JDBC"); connection = DriverManager.getConnection(jdbcUrl); + configureConnection(); } catch (ClassNotFoundException e) { throw new IllegalStateException( "SQLite repository requires org.xerial:sqlite-jdbc on the runtime classpath", e); @@ -413,6 +418,10 @@ private void openConnection() { } } + private void configureConnection() { + executeSql("PRAGMA case_sensitive_like = ON"); + } + private void ensureSchemaReady() { if (schema != null) { refreshSchema(schema); @@ -593,6 +602,9 @@ private SqlQuery selectInstancesSql( whereClauses.add(column + " LIKE ? ESCAPE '\\'"); parameters.add(sqlLikeWildcard(filterBy.fieldValue)); break; + case "~=": + whereClauses.add(regexLikeSqlCondition(column, filterBy.fieldValue, parameters)); + break; default: break; } @@ -667,6 +679,10 @@ private SqlQuery selectRelatedInstancesSql( sql.append(" AND ").append(column).append(" LIKE ? ESCAPE '\\'"); parameters.add(sqlLikeWildcard(filterBy.fieldValue)); break; + case "~=": + sql.append(" AND "). + append(regexLikeSqlCondition(column, filterBy.fieldValue, parameters)); + break; default: break; } @@ -695,13 +711,17 @@ private void appendOrderBy( } } - private boolean hasJavaOnlyFilters(final QueryFilterParams queryParams) { - for (FilterBy filterBy : queryParams.toList()) { - if ("~=".equals(filterBy.filterOperation)) { - return true; - } + private String regexLikeSqlCondition( + final String column, + final String regex, + final List parameters) { + SqliteRegexToLikeConverter.Conversion conversion = + SqliteRegexToLikeConverter.convert(regex); + parameters.add(conversion.sqlValue()); + if (conversion.isEquality()) { + return column + " = ?"; } - return false; + return column + " LIKE ? ESCAPE '\\'"; } private List queryEntityRows( @@ -722,6 +742,18 @@ private List queryEntityRows( return instances; } + private List listInstancesUsingJavaFilter( + final EntityDefinition entity, + final QueryFilterParams params) { + List instances = new ArrayList<>(super.listInstances(entity)); + if (!legacySnapshotLoaded) { + instances = new ArrayList<>(getInstanceData(). + getInstanceCollectionForEntityNamed(entity.getName()). + getInstances()); + } + return filterAndSort(instances, params); + } + private EntityInstance instanceFromRow( final EntityDefinition entity, final ResultSet resultSet) throws SQLException { String internalId = resultSet.getString(INTERNAL_ID_COLUMN); @@ -867,8 +899,10 @@ private void persistRelationship( } private void persistCountersFor(final EntityDefinition entity) { - executeSql("DELETE FROM " + identifier(COUNTERS_TABLE) + - " WHERE entity_name = " + quotedValue(entity.getName())); + executePreparedSql( + "DELETE FROM " + identifier(COUNTERS_TABLE) + + " WHERE entity_name = ?", + List.of(entity.getName())); String sql = "INSERT INTO " + identifier(COUNTERS_TABLE) + " (entity_name, field_name, next_value) VALUES (?, ?, ?)"; @@ -886,13 +920,16 @@ private void persistCountersFor(final EntityDefinition entity) { private void deletePersistedInstance(final String internalId) { for (EntityDefinition entity : schema.getEntityDefinitions()) { - executeSql("DELETE FROM " + table(entity) + - " WHERE " + identifier(INTERNAL_ID_COLUMN) + " = " + quotedValue(internalId)); + executePreparedSql( + "DELETE FROM " + table(entity) + + " WHERE " + identifier(INTERNAL_ID_COLUMN) + " = ?", + List.of(internalId)); } for (RelationshipVectorDefinition vector : relationshipVectors()) { - executeSql("DELETE FROM " + relationshipTable(vector) + - " WHERE from_internal_id = " + quotedValue(internalId) + - " OR to_internal_id = " + quotedValue(internalId)); + executePreparedSql( + "DELETE FROM " + relationshipTable(vector) + + " WHERE from_internal_id = ? OR to_internal_id = ?", + List.of(internalId, internalId)); } } @@ -904,19 +941,24 @@ private void deleteRelationshipRowsInvolving( if (!vector.getRelationshipDefinition().isKnownAs(relationshipName)) { continue; } - executeSql("DELETE FROM " + relationshipTable(vector) + - " WHERE (from_internal_id = " + quotedValue(parent.getInternalId()) + - " AND to_internal_id = " + quotedValue(child.getInternalId()) + ")" + - " OR (from_internal_id = " + quotedValue(child.getInternalId()) + - " AND to_internal_id = " + quotedValue(parent.getInternalId()) + ")"); + executePreparedSql( + "DELETE FROM " + relationshipTable(vector) + + " WHERE (from_internal_id = ? AND to_internal_id = ?)" + + " OR (from_internal_id = ? AND to_internal_id = ?)", + List.of( + parent.getInternalId(), + child.getInternalId(), + child.getInternalId(), + parent.getInternalId())); } } private void deleteRelationshipRowsInvolving(final EntityInstance instance) { for (RelationshipVectorDefinition vector : relationshipVectors()) { - executeSql("DELETE FROM " + relationshipTable(vector) + - " WHERE from_internal_id = " + quotedValue(instance.getInternalId()) + - " OR to_internal_id = " + quotedValue(instance.getInternalId())); + executePreparedSql( + "DELETE FROM " + relationshipTable(vector) + + " WHERE from_internal_id = ? OR to_internal_id = ?", + List.of(instance.getInternalId(), instance.getInternalId())); } } @@ -1009,6 +1051,17 @@ private void executeSql(final String sql) { } } + private void executePreparedSql(final String sql, final List parameters) { + try (PreparedStatement statement = connection.prepareStatement(sql)) { + for (int index = 0; index < parameters.size(); index++) { + statement.setObject(index + 1, parameters.get(index)); + } + statement.executeUpdate(); + } catch (SQLException e) { + throw new IllegalStateException("Could not execute SQLite SQL: " + sql, e); + } + } + private void runInTransaction(final Runnable operation) { try { boolean originalAutoCommit = connection.getAutoCommit(); @@ -1068,10 +1121,6 @@ private String identifier(final String rawIdentifier) { return "\"" + rawIdentifier.replace("\"", "\"\"") + "\""; } - private String quotedValue(final String value) { - return "'" + value.replace("'", "''") + "'"; - } - private static class SqlQuery { private final EntityDefinition entity; private final String sql; diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexToLikeConverterTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexToLikeConverterTest.java new file mode 100644 index 00000000..8bded696 --- /dev/null +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexToLikeConverterTest.java @@ -0,0 +1,105 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SqliteRegexToLikeConverterTest { + + @Test + public void convertsExactLiteralRegexToEquality() { + SqliteRegexToLikeConverter.Conversion conversion = + SqliteRegexToLikeConverter.convert("^abc$"); + + Assertions.assertTrue(conversion.isEquality()); + Assertions.assertEquals("abc", conversion.sqlValue()); + } + + @Test + public void convertsPrefixRegexToLike() { + SqliteRegexToLikeConverter.Conversion conversion = + SqliteRegexToLikeConverter.convert("^abc.*"); + + Assertions.assertFalse(conversion.isEquality()); + Assertions.assertEquals("abc%", conversion.sqlValue()); + } + + @Test + public void convertsContainsRegexToLike() { + SqliteRegexToLikeConverter.Conversion conversion = + SqliteRegexToLikeConverter.convert(".*abc.*"); + + Assertions.assertFalse(conversion.isEquality()); + Assertions.assertEquals("%abc%", conversion.sqlValue()); + } + + @Test + public void convertsSuffixRegexToLike() { + SqliteRegexToLikeConverter.Conversion conversion = + SqliteRegexToLikeConverter.convert(".*abc$"); + + Assertions.assertFalse(conversion.isEquality()); + Assertions.assertEquals("%abc", conversion.sqlValue()); + } + + @Test + public void convertsSingleCharacterWildcardRegexToLike() { + SqliteRegexToLikeConverter.Conversion conversion = + SqliteRegexToLikeConverter.convert("a.b"); + + Assertions.assertFalse(conversion.isEquality()); + Assertions.assertEquals("a_b", conversion.sqlValue()); + } + + @Test + public void convertsMultiCharacterWildcardRegexToLike() { + SqliteRegexToLikeConverter.Conversion conversion = + SqliteRegexToLikeConverter.convert("a.*b"); + + Assertions.assertFalse(conversion.isEquality()); + Assertions.assertEquals("a%b", conversion.sqlValue()); + } + + @Test + public void escapesLikeWildcardsInRegexLiterals() { + SqliteRegexToLikeConverter.Conversion conversion = + SqliteRegexToLikeConverter.convert("a.*%_"); + + Assertions.assertFalse(conversion.isEquality()); + Assertions.assertEquals("a%\\%\\_", conversion.sqlValue()); + } + + @Test + public void rejectsRegexCharacterClasses() { + Assertions.assertThrows( + SqliteRegexToLikeConverter.RegexToLikeConversionException.class, + () -> SqliteRegexToLikeConverter.convert("Repository [Pp]roject")); + } + + @Test + public void rejectsRegexAlternation() { + Assertions.assertThrows( + SqliteRegexToLikeConverter.RegexToLikeConversionException.class, + () -> SqliteRegexToLikeConverter.convert("cat|dog")); + } + + @Test + public void rejectsRegexGroups() { + Assertions.assertThrows( + SqliteRegexToLikeConverter.RegexToLikeConversionException.class, + () -> SqliteRegexToLikeConverter.convert("(cat)")); + } + + @Test + public void rejectsRegexShorthandCharacterClasses() { + Assertions.assertThrows( + SqliteRegexToLikeConverter.RegexToLikeConversionException.class, + () -> SqliteRegexToLikeConverter.convert("\\d+")); + } + + @Test + public void rejectsMalformedRegex() { + Assertions.assertThrows( + SqliteRegexToLikeConverter.RegexToLikeConversionException.class, + () -> SqliteRegexToLikeConverter.convert("[")); + } +} diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java index 4b54176d..ea83e113 100644 --- a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java @@ -210,6 +210,76 @@ public void sqliteRelationshipReadsDoNotHydrateCompatibilitySnapshot() { } } + @Test + public void sqliteConvertibleRegexFilterDoesNotHydrateCompatibilitySnapshot() { + ERSchema schema = todoSchema(); + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + Path databasePath = tempDir.resolve("regex-like-lazy.sqlite"); + + try (ThingRepository repository = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + repository.initializeFrom(schema); + + repository.addInstance(new EntityInstance(projectDefinition). + setValue("title", "Repository project")); + repository.addInstance(new EntityInstance(projectDefinition). + setValue("title", "repository project")); + repository.addInstance(new EntityInstance(projectDefinition). + setValue("title", "Another project")); + } + + try (ThingRepository reopened = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + reopened.initializeFrom(schema); + + QueryFilterParams regexParams = new QueryFilterParams(); + regexParams.put("title", "~=Repository.*"); + + List filtered = + reopened.listInstances(projectDefinition, regexParams); + + Assertions.assertEquals(1, filtered.size()); + Assertions.assertEquals("Repository project", + filtered.get(0).getFieldValue("title").asString()); + Assertions.assertFalse(reopened.hasLoadedCompatibilitySnapshot()); + Assertions.assertEquals(1, + reopened.getInstanceCollectionForEntityNamed("project").countInstances()); + } + } + + @Test + public void sqliteUnsupportedRegexFilterUsesCompatibilityFallback() { + ERSchema schema = todoSchema(); + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + Path databasePath = tempDir.resolve("regex-java-fallback.sqlite"); + + try (ThingRepository repository = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + repository.initializeFrom(schema); + + repository.addInstance(new EntityInstance(projectDefinition). + setValue("title", "Repository project")); + repository.addInstance(new EntityInstance(projectDefinition). + setValue("title", "Another project")); + } + + try (ThingRepository reopened = + SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { + reopened.initializeFrom(schema); + + QueryFilterParams regexParams = new QueryFilterParams(); + regexParams.put("title", "~=Repository [Pp]roject"); + + List filtered = + reopened.listInstances(projectDefinition, regexParams); + + Assertions.assertEquals(1, filtered.size()); + Assertions.assertEquals("Repository project", + filtered.get(0).getFieldValue("title").asString()); + Assertions.assertTrue(reopened.hasLoadedCompatibilitySnapshot()); + } + } + private void exerciseRepositoryContract(final ThingRepository repository) { ERSchema schema = todoSchema(); repository.initializeFrom(schema); From dd00eab084b8299c99f9983aa4896d620e143d72 Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 17:14:17 +0100 Subject: [PATCH 10/11] Add SQLite REGEXP filter support --- .../repository/SqliteRegexFilterPolicy.java | 44 ++++++ .../repository/SqliteThingRepository.java | 141 ++++++++++-------- .../ThingRepositoryContractTest.java | 88 ++++++++++- 3 files changed, 211 insertions(+), 62 deletions(-) create mode 100644 ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexFilterPolicy.java diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexFilterPolicy.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexFilterPolicy.java new file mode 100644 index 00000000..07f1bff3 --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteRegexFilterPolicy.java @@ -0,0 +1,44 @@ +package uk.co.compendiumdev.thingifier.core.repository; + +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +final class SqliteRegexFilterPolicy { + + private static final int MAX_REGEX_LENGTH = 512; + private static final Pattern NESTED_QUANTIFIER = + Pattern.compile("\\((?:\\\\.|[^\\\\)])*[+*](?:\\\\.|[^\\\\)])*\\)\\s*[+*?{]"); + + private SqliteRegexFilterPolicy() { + } + + static Pattern compileSupported(final String regex) { + if (regex == null) { + throw new UnsupportedRegexFilterException("Regex value is null"); + } + if (regex.length() > MAX_REGEX_LENGTH) { + throw new UnsupportedRegexFilterException( + "Regex exceeds maximum supported length of " + MAX_REGEX_LENGTH); + } + if (NESTED_QUANTIFIER.matcher(regex).find()) { + throw new UnsupportedRegexFilterException( + "Regex uses nested quantifiers that are not allowed for SQLite filtering"); + } + + try { + return Pattern.compile(regex); + } catch (PatternSyntaxException e) { + throw new UnsupportedRegexFilterException("Invalid regex", e); + } + } + + static final class UnsupportedRegexFilterException extends RuntimeException { + UnsupportedRegexFilterException(final String message) { + super(message); + } + + UnsupportedRegexFilterException(final String message, final Throwable cause) { + super(message, cause); + } + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java index 79d16c6e..624feceb 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/SqliteThingRepository.java @@ -1,5 +1,6 @@ package uk.co.compendiumdev.thingifier.core.repository; +import org.sqlite.Function; import uk.co.compendiumdev.thingifier.core.domain.definitions.ERSchema; import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; @@ -12,7 +13,6 @@ import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceCollection; import uk.co.compendiumdev.thingifier.core.domain.instances.RelationshipVectorInstance; -import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListFilter; import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListSorter; import uk.co.compendiumdev.thingifier.core.query.EntityListSortParamParser; import uk.co.compendiumdev.thingifier.core.query.FilterBy; @@ -37,6 +37,7 @@ import java.util.Set; import java.util.UUID; import java.util.logging.Logger; +import java.util.regex.Pattern; public class SqliteThingRepository extends InMemoryThingRepository { @@ -162,15 +163,8 @@ public List listInstances( ensureSchemaReady(); QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; - try { - SqlQuery sqlQuery = selectInstancesSql(entity, params); - return queryEntityRows(entity, sqlQuery.sql, sqlQuery.parameters); - } catch (SqliteRegexToLikeConverter.RegexToLikeConversionException e) { - LOGGER.warning("SQLite regex filter for " + entity.getName() + - " could not be converted to LIKE; using Java compatibility filtering: " + - e.getMessage()); - return listInstancesUsingJavaFilter(entity, params); - } + SqlQuery sqlQuery = selectInstancesSql(entity, params); + return queryEntityRows(entity, sqlQuery.sql, sqlQuery.parameters); } @Override @@ -337,29 +331,20 @@ public List listRelatedInstances( ensureSchemaReady(); QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; - try { - Map related = new LinkedHashMap<>(); - for (RelationshipVectorDefinition vector : - relationshipVectorsFor(instance, relationshipName)) { - SqlQuery query = selectRelatedInstancesSql(instance, vector, params); - if (query == null) { - continue; - } - for (EntityInstance item : queryEntityRows(query.entity, query.sql, query.parameters)) { - related.put(item.getInternalId(), item); - } + Map related = new LinkedHashMap<>(); + for (RelationshipVectorDefinition vector : + relationshipVectorsFor(instance, relationshipName)) { + SqlQuery query = selectRelatedInstancesSql(instance, vector, params); + if (query == null) { + continue; + } + for (EntityInstance item : queryEntityRows(query.entity, query.sql, query.parameters)) { + related.put(item.getInternalId(), item); } - - return new EntityInstanceListSorter(params). - sort(new ArrayList<>(related.values())); - } catch (SqliteRegexToLikeConverter.RegexToLikeConversionException e) { - LOGGER.warning("SQLite regex relationship filter for " + relationshipName + - " could not be converted to LIKE; using Java compatibility filtering: " + - e.getMessage()); - return filterAndSort( - new ArrayList<>(getConnectedItems(instance, relationshipName)), - params); } + + return new EntityInstanceListSorter(params). + sort(new ArrayList<>(related.values())); } @Override @@ -419,9 +404,23 @@ private void openConnection() { } private void configureConnection() { + registerRegexpFunction(); executeSql("PRAGMA case_sensitive_like = ON"); } + private void registerRegexpFunction() { + try { + Function.create( + connection, + "regexp", + new JavaRegexpFunction(), + 2, + Function.FLAG_DETERMINISTIC); + } catch (SQLException e) { + throw new IllegalStateException("Could not register SQLite REGEXP function", e); + } + } + private void ensureSchemaReady() { if (schema != null) { refreshSchema(schema); @@ -603,7 +602,7 @@ private SqlQuery selectInstancesSql( parameters.add(sqlLikeWildcard(filterBy.fieldValue)); break; case "~=": - whereClauses.add(regexLikeSqlCondition(column, filterBy.fieldValue, parameters)); + whereClauses.add(regexSqlCondition(column, filterBy.fieldValue, parameters)); break; default: break; @@ -681,7 +680,7 @@ private SqlQuery selectRelatedInstancesSql( break; case "~=": sql.append(" AND "). - append(regexLikeSqlCondition(column, filterBy.fieldValue, parameters)); + append(regexSqlCondition(column, filterBy.fieldValue, parameters)); break; default: break; @@ -711,17 +710,25 @@ private void appendOrderBy( } } - private String regexLikeSqlCondition( + private String regexSqlCondition( final String column, final String regex, final List parameters) { - SqliteRegexToLikeConverter.Conversion conversion = - SqliteRegexToLikeConverter.convert(regex); - parameters.add(conversion.sqlValue()); - if (conversion.isEquality()) { - return column + " = ?"; + try { + SqliteRegexToLikeConverter.Conversion conversion = + SqliteRegexToLikeConverter.convert(regex); + parameters.add(conversion.sqlValue()); + if (conversion.isEquality()) { + return column + " = ?"; + } + return column + " LIKE ? ESCAPE '\\'"; + } catch (SqliteRegexToLikeConverter.RegexToLikeConversionException e) { + SqliteRegexFilterPolicy.compileSupported(regex); + LOGGER.warning("SQLite regex filter could not be converted to LIKE; using REGEXP: " + + e.getMessage()); + parameters.add(regex); + return column + " REGEXP ?"; } - return column + " LIKE ? ESCAPE '\\'"; } private List queryEntityRows( @@ -742,18 +749,6 @@ private List queryEntityRows( return instances; } - private List listInstancesUsingJavaFilter( - final EntityDefinition entity, - final QueryFilterParams params) { - List instances = new ArrayList<>(super.listInstances(entity)); - if (!legacySnapshotLoaded) { - instances = new ArrayList<>(getInstanceData(). - getInstanceCollectionForEntityNamed(entity.getName()). - getInstances()); - } - return filterAndSort(instances, params); - } - private EntityInstance instanceFromRow( final EntityDefinition entity, final ResultSet resultSet) throws SQLException { String internalId = resultSet.getString(INTERNAL_ID_COLUMN); @@ -1022,13 +1017,6 @@ private List relationshipVectors() { return vectors; } - private List filterAndSort( - final List instances, - final QueryFilterParams params) { - List filtered = new EntityInstanceListFilter(params).filter(instances); - return new EntityInstanceListSorter(params).sort(filtered); - } - private Set columnNames(final String tableName) { Set names = new HashSet<>(); String sql = "PRAGMA table_info(" + identifier(tableName) + ")"; @@ -1121,6 +1109,41 @@ private String identifier(final String rawIdentifier) { return "\"" + rawIdentifier.replace("\"", "\"\"") + "\""; } + private static class JavaRegexpFunction extends Function { + private String cachedRegex; + private Pattern cachedPattern; + + @Override + protected void xFunc() throws SQLException { + if (args() != 2) { + result(0); + return; + } + + String regex = value_text(0); + String value = value_text(1); + if (regex == null || value == null) { + result(0); + return; + } + + Pattern pattern = patternFor(regex); + result(pattern.matcher(value).matches() ? 1 : 0); + } + + private Pattern patternFor(final String regex) throws SQLException { + if (!regex.equals(cachedRegex)) { + try { + cachedPattern = SqliteRegexFilterPolicy.compileSupported(regex); + cachedRegex = regex; + } catch (SqliteRegexFilterPolicy.UnsupportedRegexFilterException e) { + throw new SQLException("Unsupported SQLite regex filter: " + e.getMessage(), e); + } + } + return cachedPattern; + } + } + private static class SqlQuery { private final EntityDefinition entity; private final String sql; diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java index ea83e113..1ee5966d 100644 --- a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingRepositoryContractTest.java @@ -200,6 +200,17 @@ public void sqliteRelationshipReadsDoNotHydrateCompatibilitySnapshot() { filteredTasks.get(0).getFieldValue("title").asString()); Assertions.assertFalse(reopened.hasLoadedCompatibilitySnapshot()); + QueryFilterParams regexParams = new QueryFilterParams(); + regexParams.put("title", "~=Wire [Rr]epository"); + + List regexFilteredTasks = + reopened.listRelatedInstances(project, "tasks", regexParams); + + Assertions.assertEquals(1, regexFilteredTasks.size()); + Assertions.assertEquals("Wire repository", + regexFilteredTasks.get(0).getFieldValue("title").asString()); + Assertions.assertFalse(reopened.hasLoadedCompatibilitySnapshot()); + RepositoryBackedSimpleQuery query = new RepositoryBackedSimpleQuery(schema, reopened, "project/1/tasks"). performQuery(new QueryFilterParams()); @@ -248,10 +259,10 @@ public void sqliteConvertibleRegexFilterDoesNotHydrateCompatibilitySnapshot() { } @Test - public void sqliteUnsupportedRegexFilterUsesCompatibilityFallback() { + public void sqliteRegexpFilterDoesNotHydrateCompatibilitySnapshot() { ERSchema schema = todoSchema(); EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); - Path databasePath = tempDir.resolve("regex-java-fallback.sqlite"); + Path databasePath = tempDir.resolve("regex-regexp-lazy.sqlite"); try (ThingRepository repository = SqliteThingRepository.fileBacked(EntityRelModel.DEFAULT_DATABASE_NAME, databasePath)) { @@ -276,7 +287,78 @@ public void sqliteUnsupportedRegexFilterUsesCompatibilityFallback() { Assertions.assertEquals(1, filtered.size()); Assertions.assertEquals("Repository project", filtered.get(0).getFieldValue("title").asString()); - Assertions.assertTrue(reopened.hasLoadedCompatibilitySnapshot()); + Assertions.assertFalse(reopened.hasLoadedCompatibilitySnapshot()); + Assertions.assertEquals(1, + reopened.getInstanceCollectionForEntityNamed("project").countInstances()); + } + } + + @Test + public void sqliteInMemoryRegexpFilterDoesNotHydrateCompatibilitySnapshot() { + ERSchema schema = todoSchema(); + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + + try (ThingRepository repository = + SqliteThingRepository.inMemory(EntityRelModel.DEFAULT_DATABASE_NAME)) { + repository.initializeFrom(schema); + + repository.addInstance(new EntityInstance(projectDefinition). + setValue("title", "Repository project")); + repository.addInstance(new EntityInstance(projectDefinition). + setValue("title", "Another project")); + + QueryFilterParams regexParams = new QueryFilterParams(); + regexParams.put("title", "~=Repository [Pp]roject"); + + List filtered = + repository.listInstances(projectDefinition, regexParams); + + Assertions.assertEquals(1, filtered.size()); + Assertions.assertEquals("Repository project", + filtered.get(0).getFieldValue("title").asString()); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + } + } + + @Test + public void sqliteInvalidRegexFilterFailsWithoutHydratingCompatibilitySnapshot() { + ERSchema schema = todoSchema(); + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + + try (ThingRepository repository = + SqliteThingRepository.inMemory(EntityRelModel.DEFAULT_DATABASE_NAME)) { + repository.initializeFrom(schema); + repository.addInstance(new EntityInstance(projectDefinition). + setValue("title", "Repository project")); + + QueryFilterParams regexParams = new QueryFilterParams(); + regexParams.put("title", "~=["); + + Assertions.assertThrows( + SqliteRegexFilterPolicy.UnsupportedRegexFilterException.class, + () -> repository.listInstances(projectDefinition, regexParams)); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); + } + } + + @Test + public void sqliteTooComplexRegexFilterFailsWithoutHydratingCompatibilitySnapshot() { + ERSchema schema = todoSchema(); + EntityDefinition projectDefinition = schema.getEntityDefinitionNamed("project"); + + try (ThingRepository repository = + SqliteThingRepository.inMemory(EntityRelModel.DEFAULT_DATABASE_NAME)) { + repository.initializeFrom(schema); + repository.addInstance(new EntityInstance(projectDefinition). + setValue("title", "Repository project")); + + QueryFilterParams regexParams = new QueryFilterParams(); + regexParams.put("title", "~=(.+)+"); + + Assertions.assertThrows( + SqliteRegexFilterPolicy.UnsupportedRegexFilterException.class, + () -> repository.listInstances(projectDefinition, regexParams)); + Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot()); } } From fdce1827602e68efef2b325a9eebbcf4136a0293 Mon Sep 17 00:00:00 2001 From: Alan Richardson Date: Mon, 6 Jul 2026 18:21:06 +0100 Subject: [PATCH 11/11] Deprecate legacy SimpleQuery --- .../thingifier/core/query/SimpleQuery.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SimpleQuery.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SimpleQuery.java index 53be9fbc..c43b34d2 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SimpleQuery.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/SimpleQuery.java @@ -14,17 +14,24 @@ import static uk.co.compendiumdev.thingifier.core.query.SimpleQuery.LastMatchValue.*; -/* -Note this is not the same as a GET e.g. -- /item will always mark the query as not a collection -- /items will always mark the query as a collection - -This is a simple query to then build more complex or specific query -processing on top. - -Use the isResultACollection, wasIntentToMatchACollection, lastMatchWasInstance -in the calling method. +/** + * Legacy ERInstanceData-backed query engine. + * + * Note this is not the same as a GET e.g. + * - /item will always mark the query as not a collection + * - /items will always mark the query as a collection + * + * This is a simple query to then build more complex or specific query + * processing on top. + * + * Use the isResultACollection, wasIntentToMatchACollection, lastMatchWasInstance + * in the calling method. + * + * @deprecated Runtime API paths should use repository-backed query resolution so + * repositories such as SQLite can query without hydrating compatibility snapshots. + * Keep this class only for compatibility/fallback paths until replacement is complete. */ +@Deprecated public final class SimpleQuery implements QueryResult { private final ERInstanceData database;