Add repository interface experiment with SQLite regex filtering

challenger/pom.xml

@@ -19,6 +19,12 @@
             <version>${thingifier.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.xerial</groupId>
+            <artifactId>sqlite-jdbc</artifactId>
+            <version>${sqlite-jdbc-version}</version>
+        </dependency>
+
         <dependency>
             <groupId>io.swagger.core.v3</groupId>
             <artifactId>swagger-models</artifactId>
@@ -166,4 +172,4 @@ https://www.slf4j.org/manual.html
 
 - initial release
 
--->
+-->

challenger/src/main/java/uk/co/compendiumdev/challenge/ChallengeMain.java

@@ -7,6 +7,7 @@
 import uk.co.compendiumdev.thingifier.Thingifier;
 import uk.co.compendiumdev.thingifier.application.MainImplementation;
 import uk.co.compendiumdev.thingifier.application.httprouting.ThingifierHttpApiRoutings;
+import uk.co.compendiumdev.thingifier.core.repository.ThingRepositoryProviderConfig;
 
 public class ChallengeMain {
 
@@ -19,7 +20,9 @@ public static void main(String[] args) {
         logger.info("Starting Challenger");
 
         MainImplementation app = new MainImplementation();
-        Thingifier thingifier = new ChallengeApiModel().get();
+        ThingRepositoryProviderConfig repositoryConfig = ThingRepositoryProviderConfig.fromArgs(args);
+        logger.info("Using Thingifier repository {}", repositoryConfig.describe());
+        Thingifier thingifier = new ChallengeApiModel().get(repositoryConfig.createProvider());
         app.registerModel("challengeapi", thingifier);
 
         // add any additional thingifier configurations here if more needed than model has defined
@@ -115,6 +118,9 @@ public static ChallengeRouteHandler getChallenger(){
     }
 
     public static void stop(){
+        if (challenger != null) {
+            challenger.getThingifier().close();
+        }
         challenger = null;
     }
 }

challenger/src/main/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModel.java

@@ -6,12 +6,21 @@
 import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType;
 import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field;
 import uk.co.compendiumdev.thingifier.core.domain.definitions.validation.VRule;
+import uk.co.compendiumdev.thingifier.core.repository.ThingRepositoryProvider;
 
 import static uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType.STRING;
 
 public class ChallengeApiModel {
     public Thingifier get() {
-        Thingifier todoList = new Thingifier();
+        return get(new EntityRelModel());
+    }
+
+    public Thingifier get(final ThingRepositoryProvider repositoryProvider) {
+        return get(new EntityRelModel(repositoryProvider));
+    }
+
+    public Thingifier get(final EntityRelModel entityRelModel) {
+        Thingifier todoList = new Thingifier(entityRelModel);
 
         StringBuilder para = new StringBuilder();
 

challenger/src/test/java/uk/co/compendiumdev/challenge/apimodel/ChallengeApiModelRepositoryTest.java

@@ -0,0 +1,215 @@
+package uk.co.compendiumdev.challenge.apimodel;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import uk.co.compendiumdev.thingifier.Thingifier;
+import uk.co.compendiumdev.thingifier.api.http.HttpApiRequest;
+import uk.co.compendiumdev.thingifier.api.http.bodyparser.BodyParser;
+import uk.co.compendiumdev.thingifier.api.http.headers.HttpHeadersBlock;
+import uk.co.compendiumdev.thingifier.api.response.ApiResponse;
+import uk.co.compendiumdev.thingifier.core.domain.definitions.Cardinality;
+import uk.co.compendiumdev.thingifier.core.EntityRelModel;
+import uk.co.compendiumdev.thingifier.core.domain.definitions.EntityDefinition;
+import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field;
+import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType;
+import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance;
+import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams;
+import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepository;
+import uk.co.compendiumdev.thingifier.core.repository.SqliteThingRepositoryProvider;
+import uk.co.compendiumdev.thingifier.core.repository.ThingRepository;
+
+public class ChallengeApiModelRepositoryTest {
+
+    @Test
+    public void challengeApiCanUseSqliteInMemoryRepository() {
+        try (SqliteThingRepositoryProvider provider = SqliteThingRepositoryProvider.inMemory()) {
+            Thingifier thingifier = new ChallengeApiModel().get(provider);
+
+            Assertions.assertTrue(
+                    thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME)
+                            instanceof SqliteThingRepository);
+            Assertions.assertEquals(
+                    10,
+                    thingifier.getThingInstancesNamed("todo", EntityRelModel.DEFAULT_DATABASE_NAME).
+                            countInstances());
+        }
+    }
+
+    @Test
+    public void challengeApiGetUsesSqliteRepositoryReadsForSimpleTodoQueries() {
+        try (SqliteThingRepositoryProvider provider = SqliteThingRepositoryProvider.inMemory()) {
+            Thingifier thingifier = new ChallengeApiModel().get(provider);
+            ThingRepository repository = thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME);
+            EntityDefinition todo = thingifier.getDefinitionNamed("todo");
+
+            EntityInstance firstTodo = repository.findInstanceByQueryIdentifier(todo, "1");
+            firstTodo.setValue("doneStatus", "true");
+            repository.updateInstance(firstTodo);
+
+            ApiResponse allTodos = thingifier.api().get(
+                    "/todos", new QueryFilterParams(), new HttpHeadersBlock());
+            Assertions.assertEquals(200, allTodos.getStatusCode());
+            Assertions.assertEquals(10, allTodos.getReturnedInstanceCollection().size());
+
+            ApiResponse todoById = thingifier.api().get(
+                    "/todos/1", new QueryFilterParams(), new HttpHeadersBlock());
+            Assertions.assertEquals(200, todoById.getStatusCode());
+            Assertions.assertEquals(1, todoById.getReturnedInstanceCollection().size());
+            Assertions.assertEquals(
+                    "1",
+                    todoById.getReturnedInstanceCollection().get(0).getPrimaryKeyValue());
+
+            QueryFilterParams doneStatusFilter = new QueryFilterParams();
+            doneStatusFilter.put("doneStatus", "false");
+            ApiResponse notDoneTodos = thingifier.api().get(
+                    "/todos", doneStatusFilter, new HttpHeadersBlock());
+            Assertions.assertEquals(200, notDoneTodos.getStatusCode());
+            Assertions.assertEquals(9, notDoneTodos.getReturnedInstanceCollection().size());
+
+            QueryFilterParams idRangeFilter = new QueryFilterParams();
+            idRangeFilter.put("id", ">=2");
+            ApiResponse idRangeTodos = thingifier.api().get(
+                    "/todos", idRangeFilter, new HttpHeadersBlock());
+            Assertions.assertEquals(200, idRangeTodos.getStatusCode());
+            Assertions.assertEquals(9, idRangeTodos.getReturnedInstanceCollection().size());
+
+            QueryFilterParams sortByDescendingId = new QueryFilterParams();
+            sortByDescendingId.put("sortBy", "-id");
+            ApiResponse sortedTodos = thingifier.api().get(
+                    "/todos", sortByDescendingId, new HttpHeadersBlock());
+            Assertions.assertEquals(200, sortedTodos.getStatusCode());
+            Assertions.assertEquals(
+                    "10",
+                    sortedTodos.getReturnedInstanceCollection().get(0).getPrimaryKeyValue());
+        }
+    }
+
+    @Test
+    public void relationshipGetRoutesStillFallBackToLegacyQueryTraversal() {
+        try (SqliteThingRepositoryProvider provider = SqliteThingRepositoryProvider.inMemory()) {
+            Thingifier thingifier = new Thingifier(new EntityRelModel(provider));
+
+            EntityDefinition project = thingifier.defineThing("project", "projects");
+            project.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT));
+            project.addField(Field.is("title", FieldType.STRING));
+
+            EntityDefinition task = thingifier.defineThing("task", "tasks");
+            task.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT));
+            task.addField(Field.is("title", FieldType.STRING));
+
+            thingifier.defineRelationship(project, task, "tasks", Cardinality.ONE_TO_MANY()).
+                    whenReversed(Cardinality.ONE_TO_ONE(), "task-of");
+
+            ThingRepository repository = thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME);
+            EntityInstance projectInstance = new EntityInstance(project).setValue("title", "Office");
+            EntityInstance taskInstance = new EntityInstance(task).setValue("title", "File paperwork");
+            repository.addInstance(projectInstance);
+            repository.addInstance(taskInstance);
+            repository.connectRelationship(projectInstance, "tasks", taskInstance);
+
+            ApiResponse response = thingifier.api().get(
+                    "/project/1/tasks", new QueryFilterParams(), new HttpHeadersBlock());
+
+            Assertions.assertEquals(200, response.getStatusCode());
+            Assertions.assertEquals(1, response.getReturnedInstanceCollection().size());
+            Assertions.assertEquals(
+                    "File paperwork",
+                    response.getReturnedInstanceCollection().get(0).getFieldValue("title").asString());
+        }
+    }
+
+    @Test
+    public void simpleEntityWritesUseSqliteRepositoryWithoutLoadingCompatibilitySnapshot() {
+        try (SqliteThingRepositoryProvider provider = SqliteThingRepositoryProvider.inMemory()) {
+            Thingifier thingifier = new Thingifier(new EntityRelModel(provider));
+
+            EntityDefinition note = thingifier.defineThing("note", "notes");
+            note.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT));
+            note.addField(Field.is("title", FieldType.STRING));
+
+            ThingRepository repository = thingifier.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME);
+
+            ApiResponse create = thingifier.api().post(
+                    "/notes",
+                    bodyParser(thingifier, "{\"title\":\"created\"}"),
+                    new HttpHeadersBlock());
+
+            Assertions.assertEquals(201, create.getStatusCode());
+            Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot());
+            Assertions.assertEquals(1, repository.listInstances(note).size());
+
+            ApiResponse postAmend = thingifier.api().post(
+                    "/notes/1",
+                    bodyParser(thingifier, "{\"title\":\"posted\"}"),
+                    new HttpHeadersBlock());
+
+            Assertions.assertEquals(200, postAmend.getStatusCode());
+            Assertions.assertEquals(
+                    "posted",
+                    repository.findInstanceByQueryIdentifier(note, "1").
+                            getFieldValue("title").asString());
+            Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot());
+
+            ApiResponse putAmend = thingifier.api().put(
+                    "/notes/1",
+                    bodyParser(thingifier, "{\"title\":\"put\"}"),
+                    new HttpHeadersBlock());
+
+            Assertions.assertEquals(200, putAmend.getStatusCode());
+            Assertions.assertEquals(
+                    "put",
+                    repository.findInstanceByQueryIdentifier(note, "1").
+                            getFieldValue("title").asString());
+            Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot());
+
+            ApiResponse delete = thingifier.api().delete(
+                    "/notes/1", new HttpHeadersBlock());
+
+            Assertions.assertEquals(200, delete.getStatusCode());
+            Assertions.assertEquals(0, repository.listInstances(note).size());
+            Assertions.assertFalse(repository.hasLoadedCompatibilitySnapshot());
+        }
+    }
+
+    @Test
+    public void sqliteMemoryProvidersHaveSeparateNamedDatabaseLifetimes() {
+        EntityRelModel firstModel = null;
+        EntityRelModel secondModel = null;
+        try (SqliteThingRepositoryProvider firstProvider = SqliteThingRepositoryProvider.inMemory();
+             SqliteThingRepositoryProvider secondProvider = SqliteThingRepositoryProvider.inMemory()) {
+            firstModel = new EntityRelModel(firstProvider);
+            secondModel = new EntityRelModel(secondProvider);
+
+            EntityDefinition firstNote = firstModel.createEntityDefinition("note", "notes");
+            firstNote.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT));
+            firstNote.addField(Field.is("title", FieldType.STRING));
+
+            EntityDefinition secondNote = secondModel.createEntityDefinition("note", "notes");
+            secondNote.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT));
+            secondNote.addField(Field.is("title", FieldType.STRING));
+
+            firstModel.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME).
+                    addInstance(new EntityInstance(firstNote).setValue("title", "first"));
+
+            Assertions.assertEquals(
+                    1,
+                    firstModel.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME).
+                            listInstances(firstNote).size());
+            Assertions.assertEquals(
+                    0,
+                    secondModel.getRepository(EntityRelModel.DEFAULT_DATABASE_NAME).
+                            listInstances(secondNote).size());
+        } finally {
+            if (firstModel != null) {
+                firstModel.close();
+            }
+            if (secondModel != null) {
+                secondModel.close();
+            }
+        }
+    }
+
+    private BodyParser bodyParser(final Thingifier thingifier, final String json) {
+        return new BodyParser(new HttpApiRequest("/path").setBody(json), thingifier.getThingNames());
+    }
+}

challengerAuto/pom.xml

@@ -24,6 +24,13 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.xerial</groupId>
+            <artifactId>sqlite-jdbc</artifactId>
+            <version>${sqlite-jdbc-version}</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
@@ -100,4 +107,4 @@
         </plugins>
     </build>
 
-</project>
+</project>

challengerAuto/src/test/java/uk/co/compendiumdev/sparkstart/Environment.java

@@ -37,10 +37,10 @@ public static String getBaseUri() {
             String [] args;
 
             if(SINGLE_PLAYER_MODE){
-                args = "".split(",");
+                args = "-sqlite-memory".split(",");
             }else{
-                args = "-multiplayer".split(",");
-            };
+                args = "-sqlite-memory,-multiplayer".split(",");
+            }
 
 
             ChallengeMain.main(args);

docs/repository-sqlite-migration-audit.md

@@ -0,0 +1,44 @@
+# Repository SQLite Migration Audit
+
+This note classifies the current `getInstanceData()` and `getThingInstancesNamed()` usage during the SQLite repository experiment.
+
+## Normal API Request Paths
+
+Simple entity `GET` requests use `ThingRepository` for `/entity`, `/entities`, `/entity/{id}`, and `/entities/{id}`.
+
+Simple relationship `GET` traversal, such as `/projects/{id}/tasks`, now uses `ThingRepository.listRelatedInstances(...)`. SQLite executes the relationship lookup against relationship tables and materializes only the returned rows.
+
+Simple entity `POST`, `PUT`, and `DELETE` requests resolve entity definitions and entity instances through `ThingRepository` for root collection and root instance paths. These paths should not call `getInstanceData()` for SQLite unless a payload uses unsupported legacy behavior.
+
+Relationship create/delete URL paths still use the legacy `SimpleQuery` fallback. This includes write paths such as `POST /projects/{id}/tasks` and `DELETE /projects/{id}/tasks/{id}`.
+
+## Relationship Payload Helpers
+
+Relationship references in request bodies now resolve target instances through `ThingRepository` rather than `EntityInstanceCollection` scans. This keeps normal create/amend payloads from requiring a full SQLite compatibility snapshot merely to find a related target by id, guid, or other supported field.
+
+## Compatibility Snapshot Paths
+
+`EntityRelModel.getInstanceData()` and `ThingRepository.getInstanceData()` are compatibility/export paths. For SQLite, `getInstanceData()` explicitly hydrates the legacy in-memory snapshot.
+
+Known compatibility users:
+
+- data populator and JSON import paths;
+- admin/debug/export views and reporting;
+- legacy data populators that have not implemented `RepositoryDataPopulator`;
+- relationship write URL fallback paths.
+
+The default Todo API and Todo Manager seed populators now implement `RepositoryDataPopulator`, so SQLite-backed Todo apps can seed through repository writes without first hydrating `ERInstanceData`.
+
+The base GUI data explorer list/detail pages now read via `ThingRepository` and render relationship sections via `ThingRepository.listRelatedInstances(...)`.
+
+These remaining compatibility paths are acceptable for this phase, but they are the next places to reduce if memory efficiency needs to extend beyond simple entity reads and Todo-style population.
+
+## Test Helpers
+
+Most `getInstanceData()` and `getThingInstancesNamed()` usage is in `src/test`. These are test fixture and assertion helpers, not runtime leaks, unless a test is specifically exercising SQLite memory efficiency.
+
+## Runtime Lifecycle
+
+SQLite in-memory repositories use named database URLs scoped to a single `SqliteThingRepositoryProvider` lifetime. Each logical database key receives its own SQLite in-memory database name inside that provider namespace.
+
+The repository/provider owns the SQLite connection. `Thingifier.close()`, `EntityRelModel.close()`, `MainImplementation.close()`, and the shutdown route close repository providers so the in-memory database lifetime is explicit and connection-bound.

ercoremodel/pom.xml

@@ -46,6 +46,11 @@
             <version>${junit.jupiter.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.xerial</groupId>
+            <artifactId>sqlite-jdbc</artifactId>
+            <version>${sqlite-jdbc-version}</version>
+        </dependency>
 
     </dependencies>
 
@@ -100,4 +105,4 @@
             </plugin>
         </plugins>
     </build>
-</project>
+</project>