From c19e523a0aae4276ab6c93f2b51bd427241280ad Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 19:27:19 +0200 Subject: [PATCH 01/25] remove global pch --- Engine/CMakeLists.txt | 4 ++-- Engine/include/delta/core/pch.h | 3 +++ Engine/include/delta/pch.h | 24 ------------------------ Engine/include/delta/platform/pch.h | 3 +++ Engine/src/delta/pch.cpp | 19 ------------------- Engine/src/delta/pch.h | 25 ------------------------- 6 files changed, 8 insertions(+), 70 deletions(-) delete mode 100644 Engine/include/delta/pch.h delete mode 100644 Engine/src/delta/pch.cpp delete mode 100644 Engine/src/delta/pch.h diff --git a/Engine/CMakeLists.txt b/Engine/CMakeLists.txt index 5594dc1..e635b35 100644 --- a/Engine/CMakeLists.txt +++ b/Engine/CMakeLists.txt @@ -69,8 +69,8 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug") endif() target_precompile_headers(ProjectDelta - PUBLIC "include/delta/pch.h" "include/delta/core/pch.h" "include/delta/platform/pch.h" "include/delta/utils/pch.h" - PRIVATE "src/delta/pch.h" "src/delta/core/pch.h" "src/delta/platform/pch.h" + PUBLIC "include/delta/core/pch.h" "include/delta/platform/pch.h" "include/delta/utils/pch.h" + PRIVATE "src/delta/core/pch.h" "src/delta/platform/pch.h" ) target_include_directories(ProjectDelta diff --git a/Engine/include/delta/core/pch.h b/Engine/include/delta/core/pch.h index 05fd1bc..4fcb3ec 100644 --- a/Engine/include/delta/core/pch.h +++ b/Engine/include/delta/core/pch.h @@ -16,6 +16,9 @@ #pragma once +#include +#include + #include "core_types.h" #include "engine.h" #include "memory.h" diff --git a/Engine/include/delta/pch.h b/Engine/include/delta/pch.h deleted file mode 100644 index e9c7a16..0000000 --- a/Engine/include/delta/pch.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2026 Jakub Bączyk - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -#include -#include diff --git a/Engine/include/delta/platform/pch.h b/Engine/include/delta/platform/pch.h index ba3d202..5d5f58a 100644 --- a/Engine/include/delta/platform/pch.h +++ b/Engine/include/delta/platform/pch.h @@ -16,6 +16,9 @@ #pragma once +#include +#include + #include "compiler.h" #include "os.h" #include "os_types.h" diff --git a/Engine/src/delta/pch.cpp b/Engine/src/delta/pch.cpp deleted file mode 100644 index 4d901d7..0000000 --- a/Engine/src/delta/pch.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2026 Jakub Bączyk - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "pch.h" diff --git a/Engine/src/delta/pch.h b/Engine/src/delta/pch.h deleted file mode 100644 index f3d91f5..0000000 --- a/Engine/src/delta/pch.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2026 Jakub Bączyk - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include From 78ecfd0ee4601387c7affc35481997acd4222c7b Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 19:51:50 +0200 Subject: [PATCH 02/25] allow having additional primitives in the codegen definitions --- CodeGen/main.py | 4 ++-- CodeGen/model.py | 4 ++++ Engine/codegen/ThreadContext.json | 20 +++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CodeGen/main.py b/CodeGen/main.py index 9143fa6..e72c3a1 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -27,7 +27,7 @@ def main(): core_data = json.load(f) primitives_map = {} - for p in core_data.get("primitives", []) + core_data.get("opaque_engine_types", []): + for p in core_data.get("primitives", []): primitives_map[p["name"]] = p if not os.path.exists(input_schema_path): @@ -35,7 +35,7 @@ def main(): sys.exit(1) parsed_structures, target_namespace = load_schemas_tree([input_schema_path], primitives_map) - + emitter = EngineEmitter(parsed_structures, primitives_map, target_namespace) emitter.emit_all(output_filepath) diff --git a/CodeGen/model.py b/CodeGen/model.py index 23cf405..8b1f023 100644 --- a/CodeGen/model.py +++ b/CodeGen/model.py @@ -28,6 +28,10 @@ def load_schemas_tree(schema_files, primitives_registry): if "namespace" in data: target_namespace = data["namespace"] + if "primitives" in data: + for t in data.get("primitives", []): + primitives_registry[t["name"]] = t + for s in data["structures"]: custom_types.add(s["name"]) raw_structs.append(s) diff --git a/Engine/codegen/ThreadContext.json b/Engine/codegen/ThreadContext.json index 7cc0cf7..1f79a30 100644 --- a/Engine/codegen/ThreadContext.json +++ b/Engine/codegen/ThreadContext.json @@ -1,5 +1,14 @@ { - "namespace": "delta::core", + "namespace": "delta::core", + "primitives": [ + { + "name": "std::atomic", + "alignment": 1, + "aos": "std::atomic", + "soa": "std::atomic*", + "view": "std::atomic&" + } + ], "structures": [ { "name": "PageCoordinator", @@ -19,6 +28,15 @@ } ] }, + { + "name": "ThreadState", + "fields": [ + { + "type": "std::atomic", + "name": "state" + } + ] + }, { "name": "ArenaAllocator", "alignment": 8, From 3ecae849d3c6172c4e43e82de9601268db6e7a6d Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 19:56:56 +0200 Subject: [PATCH 03/25] allow adding headers to include in the codegen --- CodeGen/emitters/engine_emitters.py | 5 ++++- CodeGen/main.py | 4 ++-- CodeGen/model.py | 6 +++++- Engine/codegen/ThreadContext.json | 3 +++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CodeGen/emitters/engine_emitters.py b/CodeGen/emitters/engine_emitters.py index f4f22b4..622220c 100644 --- a/CodeGen/emitters/engine_emitters.py +++ b/CodeGen/emitters/engine_emitters.py @@ -1,16 +1,19 @@ from .buffer import CodeBuffer class EngineEmitter: - def __init__(self, structures, primitives_registry, namespace=None): + def __init__(self, structures, primitives_registry, includes, namespace=None): self.structures = structures self.primitives = primitives_registry self.namespace = namespace + self.includes = includes self.buf = CodeBuffer() def emit_all(self, output_filepath): self.buf.write_line("// AUTOMATICALLY GENERATED FILE - DO NOT MODIFY") self.buf.write_line("#pragma once") self.buf.write_line("#include ") + for include in self.includes: + self.buf.write_line(include) self.buf.write_line() if self.namespace: diff --git a/CodeGen/main.py b/CodeGen/main.py index e72c3a1..e61756b 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -34,9 +34,9 @@ def main(): print(f"Error: Structural schema missing at {input_schema_path}", file=sys.stderr) sys.exit(1) - parsed_structures, target_namespace = load_schemas_tree([input_schema_path], primitives_map) + parsed_structures, target_namespace, includes = load_schemas_tree([input_schema_path], primitives_map) - emitter = EngineEmitter(parsed_structures, primitives_map, target_namespace) + emitter = EngineEmitter(parsed_structures, primitives_map, includes, target_namespace) emitter.emit_all(output_filepath) if __name__ == "__main__": diff --git a/CodeGen/model.py b/CodeGen/model.py index 8b1f023..a30332d 100644 --- a/CodeGen/model.py +++ b/CodeGen/model.py @@ -20,6 +20,7 @@ def load_schemas_tree(schema_files, primitives_registry): custom_types = set() raw_structs = [] target_namespace = None + includes = [] for file_path in schema_files: with open(file_path, "r") as f: @@ -32,6 +33,9 @@ def load_schemas_tree(schema_files, primitives_registry): for t in data.get("primitives", []): primitives_registry[t["name"]] = t + if "includes" in data: + includes = [include for include in data["includes"]] + for s in data["structures"]: custom_types.add(s["name"]) raw_structs.append(s) @@ -41,4 +45,4 @@ def load_schemas_tree(schema_files, primitives_registry): for s in raw_structs ] - return parsed_structures, target_namespace + return parsed_structures, target_namespace, includes diff --git a/Engine/codegen/ThreadContext.json b/Engine/codegen/ThreadContext.json index 1f79a30..9728f7d 100644 --- a/Engine/codegen/ThreadContext.json +++ b/Engine/codegen/ThreadContext.json @@ -1,5 +1,8 @@ { "namespace": "delta::core", + "includes": [ + "#include " + ], "primitives": [ { "name": "std::atomic", From f191f084271266cdaf1e009ab13330032b2719dd Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 20:53:02 +0200 Subject: [PATCH 04/25] redefine some types to allow adding other emitters in the future --- CodeGen/components.py | 39 +++++++++ CodeGen/emitters/buffer.py | 30 ------- CodeGen/emitters/code_buffer.py | 28 ++++++ CodeGen/emitters/emitter.py | 17 ++++ CodeGen/emitters/engine_emitters.py | 127 ---------------------------- CodeGen/emitters/soa_emitter.py | 50 +++++++++++ CodeGen/type_registry.py | 37 ++++++++ 7 files changed, 171 insertions(+), 157 deletions(-) create mode 100644 CodeGen/components.py delete mode 100644 CodeGen/emitters/buffer.py create mode 100644 CodeGen/emitters/code_buffer.py create mode 100644 CodeGen/emitters/emitter.py delete mode 100644 CodeGen/emitters/engine_emitters.py create mode 100644 CodeGen/emitters/soa_emitter.py create mode 100644 CodeGen/type_registry.py diff --git a/CodeGen/components.py b/CodeGen/components.py new file mode 100644 index 0000000..8282c24 --- /dev/null +++ b/CodeGen/components.py @@ -0,0 +1,39 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import List, Dict + +from type_registry import TypeRegistry + +class ISchemaComponent(ABC): + @property + @abstractmethod + def name(self) -> str: pass + + @abstractmethod + def calculate_bytes(self, registry: TypeRegistry) -> int: pass + +@dataclass +class EnumComponent(ISchemaComponent): + _name: str + underlying_type: str + values: List[str] = field(default_factory=list) + + @property + def name(self) -> str: return self.name + + def calculate_bytes(self, registry): + return registry.get_byte_size(self.underlying_type) + +@dataclass +class StructureComponent(ISchemaComponent): + _name: str + fields: Dict[str, str] = field(default_factory=dict) + + @property + def name(self) -> str: return self.name + + def calculate_bytes(self, registry): + total = 0 + for field_name, field_type in self.fields.items(): + total += registry.get_byte_size(field_type) + return total \ No newline at end of file diff --git a/CodeGen/emitters/buffer.py b/CodeGen/emitters/buffer.py deleted file mode 100644 index 4c7155c..0000000 --- a/CodeGen/emitters/buffer.py +++ /dev/null @@ -1,30 +0,0 @@ -class CodeBuffer: - def __init__(self, indent_string=" "): - self.lines = [] - self._indent_string = indent_string - self._current_indent = 0 - - def indent(self): - self._current_indent += 1 - - def dedent(self): - self._current_indent = max(0, self._current_indent - 1) - - def write_line(self, text=""): - if (text.strip() == ""): - self.lines.append("") - else: - spacing = self._indent_string * self._current_indent - self.lines.append(f"{spacing}{text}") - - def open_block(self, prefix_text=""): - if prefix_text: - self.write_line(prefix_text) - self.indent() - - def close_block(self, suffix_text="};"): - self.dedent() - self.write_line(suffix_text) - - def render(self) -> str: - return "\n".join(self.lines) \ No newline at end of file diff --git a/CodeGen/emitters/code_buffer.py b/CodeGen/emitters/code_buffer.py new file mode 100644 index 0000000..4b8b84a --- /dev/null +++ b/CodeGen/emitters/code_buffer.py @@ -0,0 +1,28 @@ +class CodeBuffer: + def __init__(self, indent_char: str = " "): + self._lines: list[str] = [] + self._indent_level: int = 0 + self._indent_char: str = indent_char + + def write_line(self, text: str = "") -> None: + if text.strip() == "": + self._lines.append("") + else: + indent = self._indent_char * self._indent_level + self._lines.append(f"{indent}{text}") + + def open_block(self, prefix_text: str = "") -> None: + if prefix_text: + self.write_line(prefix_text) + self.write_line("{") + self._indent_level += 1 + + def close_block(self, suffix_text: str = "") -> None: + self._indent_level = max(0, self._indent_level - 1) + self.write_line(f"}}{suffix_text}") + + def comment(self, text: str) -> None: + self.write_line(f"// {text}") + + def get_content(self) -> str: + return "\n".join(self._lines) diff --git a/CodeGen/emitters/emitter.py b/CodeGen/emitters/emitter.py new file mode 100644 index 0000000..ea3c970 --- /dev/null +++ b/CodeGen/emitters/emitter.py @@ -0,0 +1,17 @@ +from abc import ABC, abstractmethod +from type_registry import TypeRegistry +from code_buffer import CodeBuffer +from components import EnumComponent, StructureComponent + +class ICodeEmitter(ABC): + @abstractmethod + def emit_source(self, registry: 'TypeRegistry', buffer: CodeBuffer) -> None: + pass + + @abstractmethod + def _emit_enum(self, enum: EnumComponent, buffer: CodeBuffer) -> None: + pass + + @abstractmethod + def _emit_struct(self, struct: StructureComponent, buffer: CodeBuffer) -> None: + pass \ No newline at end of file diff --git a/CodeGen/emitters/engine_emitters.py b/CodeGen/emitters/engine_emitters.py deleted file mode 100644 index 622220c..0000000 --- a/CodeGen/emitters/engine_emitters.py +++ /dev/null @@ -1,127 +0,0 @@ -from .buffer import CodeBuffer - -class EngineEmitter: - def __init__(self, structures, primitives_registry, includes, namespace=None): - self.structures = structures - self.primitives = primitives_registry - self.namespace = namespace - self.includes = includes - self.buf = CodeBuffer() - - def emit_all(self, output_filepath): - self.buf.write_line("// AUTOMATICALLY GENERATED FILE - DO NOT MODIFY") - self.buf.write_line("#pragma once") - self.buf.write_line("#include ") - for include in self.includes: - self.buf.write_line(include) - self.buf.write_line() - - if self.namespace: - self.buf.open_block(f"namespace {self.namespace} {{") - self.buf.write_line() - - for struct in self.structures: - self._emit_aos(struct) - self.buf.write_line() - - for struct in self.structures: - self._emit_soa(struct) - self.buf.write_line() - - for struct in self.structures: - self._emit_view(struct) - self.buf.write_line() - - if self.namespace: - self.buf.close_block(f"}}; // namespace {self.namespace}") - - new_content = self.buf.render() - self._write_if_changed(output_filepath, new_content) - - def _get_type_name(self, field, target_track): - if target_track == "aos": - suffix = "AoS" - elif target_track == "soa": - suffix = "SoA" - else: - suffix = "View" - - return f"{field.type_name}{suffix}" - - def _get_field_type(self, field, target_track): - if field.is_primitive: - return self.primitives[field.type_name][target_track] - else: - if target_track == "aos": - suffix = "AoS" - elif target_track == "soa": - suffix = "SoA" - else: - suffix = "View" - - return f"{field.type_name}{suffix}" - - def _emit_aos(self, struct): - align_prefix = f"alignas({struct.alignment}) " if struct.alignment else "" - - self.buf.open_block(f"struct {align_prefix}{struct.name}AoS {{") - for field in struct.fields: - ftype = self._get_field_type(field, "aos") - self.buf.write_line(f"{ftype} {field.name};") - self.buf.close_block() - - def _emit_soa(self, struct): - align_prefix = f"alignas({struct.alignment}) " if struct.alignment else "" - - self.buf.open_block(f"struct {align_prefix}{struct.name}SoA {{") - for field in struct.fields: - ftype = self._get_field_type(field, "soa") - self.buf.write_line(f"{ftype} {field.name};") - self.buf.close_block() - - def _emit_view(self, struct): - self.buf.open_block(f"struct {struct.name}View {{") - - # Write references - for field in struct.fields: - ftype = self._get_field_type(field, "view") - self.buf.write_line(f"{ftype} {field.name};") - self.buf.write_line() - - # FromAoS factory initialization - self.buf.open_block(f"[[nodiscard]] static inline {struct.name}View FromAoS({struct.name}AoS& instance) noexcept {{") - self.buf.open_block(f"return {struct.name}View {{") - - for field in struct.fields: - if field.is_primitive: - self.buf.write_line(f".{field.name} = instance.{field.name},") - else: - self.buf.write_line(f".{field.name} = {field.type_name}View::FromAoS(instance.{field.name}),") - - self.buf.close_block("};") - self.buf.close_block("}") - - # FromSoA factory initialization - self.buf.open_block(f"[[nodiscard]] static inline {struct.name}View FromSoA({struct.name}SoA& instance, uint32_t ix) noexcept {{") - self.buf.open_block(f"return {struct.name}View {{") - - for field in struct.fields: - if field.is_primitive: - self.buf.write_line(f".{field.name} = instance.{field.name}[ix],") - else: - self.buf.write_line(f".{field.name} = {field.type_name}View::FromSoA(instance.{field.name}, ix),") - - self.buf.close_block("};") - self.buf.close_block("}") - - # end structure - self.buf.close_block() - - def _write_if_changed(self, filepath, content): - import os - if os.path.exists(filepath): - with open(filepath, "r") as f: - if f.read() == content: - return - with open(filepath, "w") as f: - f.write(content) diff --git a/CodeGen/emitters/soa_emitter.py b/CodeGen/emitters/soa_emitter.py new file mode 100644 index 0000000..e6103a7 --- /dev/null +++ b/CodeGen/emitters/soa_emitter.py @@ -0,0 +1,50 @@ +from emitter import ICodeEmitter +from components import EnumComponent, StructureComponent + +class SoALayoutEmitter(ICodeEmitter): + def emit_source(self, registry, buffer): + buffer.comment("="*80) + buffer.comment("CODE GENERATED BY DELTA ENGINE CODEGEN UTILITY") + buffer.comment("="*80) + buffer.write_line("#pragma once") + buffer.write_line("#include ") + buffer.write_line("#include ") + buffer.write_line() + + for comp in registry.custom_components.values(): + if isinstance(comp, EnumComponent): + self._emit_enum(comp, buffer) + + for comp in registry.custom_components.values(): + if isinstance(comp, StructureComponent): + self._emit_struct(comp, buffer) + + def _emit_enum(self, enum, buffer): + buffer.open_block(f"enum class {enum.name} : {enum.underlying_type}") + + for idx, val in enumerate(enum.values): + buffer.write_line(f"{val:<24} = {idx},") + + buffer.write_line(f"{'COUNT,':<24} = {idx}") + buffer.write_line(f"{'INVALID':<24} = 0xFF") + buffer.close_block(";\n") + + # string reflection helper + buffer.open_block(f"[[nodiscard]] constexpr const char* ToString({enum.name} value) noexcept") # open function + buffer.open_block("switch (value)") # open switch + + for val in enum.values: + buffer.write_line(f"case {enum.name}::{val}: return \"{val}\";") + buffer.write_line("default: return \"UNKNOWN\";") + + buffer.close_block() # close switch + buffer.close_block() # close function + buffer.write_line() + + def _emit_struct(self, struct, buffer): + buffer.write_line(f"struct {struct.name}SoA") + + for field_name, field_type in struct.fields.items(): + buffer.write_line(f"{field_type:<24}* {field_name};") + + buffer.close_block(";\n") \ No newline at end of file diff --git a/CodeGen/type_registry.py b/CodeGen/type_registry.py new file mode 100644 index 0000000..b9d2623 --- /dev/null +++ b/CodeGen/type_registry.py @@ -0,0 +1,37 @@ +from typing import Dict +from dataclasses import dataclass + +from components import ISchemaComponent + +class TypeRegistry: + def __init__(self): + self._types: Dict[str, int] = { + "int8_t": 1, "uint8_t": 1, + "int16_t": 2, "uint16_t": 2, + "int32_t": 4, "uint32_t": 4, + "int64_t": 8, "uint64_t": 8, + + "size_t": 8, + "uintptr_t": 8, + "ptrdiff_t": 8, + + "float": 4, + "double": 8, + "bool": 1 + } + + self.custom_components: Dict[str, 'ISchemaComponent'] = {} + + def register_custom(self, component: 'ISchemaComponent') -> None: + if component.name in self._types or component.name in self.custom_components: + raise ValueError(f"Type collision error: '{component.name}' is already registered.") + self.custom_components[component.name] = component + + def get_byte_size(self, type_name: str) -> int: + if type_name in self._types: + return self._types[type_name] + + if type_name in self.custom_components: + return self._types[type_name].calculate_bytes(self) + + raise TypeError(f"Compilation link error: Unrecognized type name schema referenced: '{type_name}'") \ No newline at end of file From 30659f7922ece6d3dba2d41af64c0d00625e6fd0 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 21:42:53 +0200 Subject: [PATCH 05/25] type registry and schema decoder --- CodeGen/main.py | 41 +++++++------------------ CodeGen/model.py | 48 ----------------------------- CodeGen/schema_decoder.py | 53 +++++++++++++++++++++++++++++++++ CodeGen/schemas/core_types.json | 46 ---------------------------- CodeGen/type_registry.py | 8 ++++- 5 files changed, 70 insertions(+), 126 deletions(-) delete mode 100644 CodeGen/model.py create mode 100644 CodeGen/schema_decoder.py delete mode 100644 CodeGen/schemas/core_types.json diff --git a/CodeGen/main.py b/CodeGen/main.py index e61756b..4c12213 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -1,43 +1,22 @@ -import os import sys -import json -from model import load_schemas_tree -from emitters.engine_emitters import EngineEmitter +from type_registry import TypeRegistry +from schema_decoder import SchemaDecorder def main(): if len(sys.argv) < 3: print("Error: Missing output file path argument from CMake.", file=sys.stderr) sys.exit(1) - - input_schema_path = sys.argv[1] - output_filepath = sys.argv[2] - script_dir = os.path.dirname(os.path.abspath(__file__)) - core_types_path = os.path.join(script_dir, "schemas", "core_types.json") - output_directory, _ = os.path.split(output_filepath) + schema_path: str = sys.argv[1] + output_header: str = sys.argv[2] - print(output_directory) - os.makedirs(output_directory, exist_ok=True) + registry = TypeRegistry() + decoder = SchemaDecorder(registry) - if not os.path.exists(core_types_path): - print(f"Error: Missing core types registry at {core_types_path}", file=sys.stderr) - sys.exit(1) - - with open(core_types_path, "r") as f: - core_data = json.load(f) - - primitives_map = {} - for p in core_data.get("primitives", []): - primitives_map[p["name"]] = p - - if not os.path.exists(input_schema_path): - print(f"Error: Structural schema missing at {input_schema_path}", file=sys.stderr) - sys.exit(1) - - parsed_structures, target_namespace, includes = load_schemas_tree([input_schema_path], primitives_map) - - emitter = EngineEmitter(parsed_structures, primitives_map, includes, target_namespace) - emitter.emit_all(output_filepath) + try: + decoder.decode_file(schema_path) + except Exception as e: + print(f"CODEGEN PIPELINE FAILURE:\n{e}") if __name__ == "__main__": main() diff --git a/CodeGen/model.py b/CodeGen/model.py deleted file mode 100644 index a30332d..0000000 --- a/CodeGen/model.py +++ /dev/null @@ -1,48 +0,0 @@ -import json - -class Field: - def __init__(self, type_name, name, is_primitive): - self.type_name = type_name - self.name = name - self.is_primitive = is_primitive - -class Structure: - def __init__(self, name, raw_fields, primitives_registry, custom_types, alignment = None): - self.name = name - self.fields = [] - self.alignment = int(alignment) if alignment is not None else None - - for f in raw_fields: - is_prim = f["type"] in primitives_registry - self.fields.append(Field(f["type"], f["name"], is_prim)) - -def load_schemas_tree(schema_files, primitives_registry): - custom_types = set() - raw_structs = [] - target_namespace = None - includes = [] - - for file_path in schema_files: - with open(file_path, "r") as f: - data = json.load(f) - - if "namespace" in data: - target_namespace = data["namespace"] - - if "primitives" in data: - for t in data.get("primitives", []): - primitives_registry[t["name"]] = t - - if "includes" in data: - includes = [include for include in data["includes"]] - - for s in data["structures"]: - custom_types.add(s["name"]) - raw_structs.append(s) - - parsed_structures = [ - Structure(s["name"], s["fields"], primitives_registry, custom_types, s.get("alignment")) - for s in raw_structs - ] - - return parsed_structures, target_namespace, includes diff --git a/CodeGen/schema_decoder.py b/CodeGen/schema_decoder.py new file mode 100644 index 0000000..fb3082e --- /dev/null +++ b/CodeGen/schema_decoder.py @@ -0,0 +1,53 @@ +import json +from type_registry import TypeRegistry +from components import EnumComponent, StructureComponent +from typing import Dict, Any + +class SchemaDecorder: + def __init__(self, registry: TypeRegistry): + self.registry = registry + + def decode_file(self, file_path: str): + with open(file_path, 'r') as f: + try: + data = json.load(f) + except json.JSONDecodeError as e: + raise ValueError(f"Failed to parse target JSON schema: {e}") + + if "enums" in data: + self._decode_enums(data["enums"]) + + if "structures" in data: + self._decode_structs(data["structures"]) + + def _decode_enums(self, enum_data: Dict[str, Any]) -> None: + for name, properties in enum_data.items(): + if "underlying_type" not in properties: + raise KeyError(f"Enum schema entry '{name}' must define 'underlying_type'") + elif "values" not in properties: + raise KeyError(f"Enum schema entry '{name}' must define 'values'") + + enum_comp = EnumComponent( + _name=name, + underlying_type=properties["underlying_type"], + values=list(properties["values"]) + ) + + self.registry.register_custom(enum_comp) + + def _decode_structures(self, struct_data: Dict[str, Any]) -> None: + for name, fields in struct_data.items(): + if not isinstance(fields, dict) or not fields: + raise ValueError(f"Structure '{name}' must contain a valid field definition dictionary.") + + struct_comp = StructureComponent( + _name=name, + fields=dict(fields) + ) + + try: + struct_comp.calculate_bytes(self.registry) + except TypeError as e: + raise TypeError(f"Validation for structure template '{name}': {e}") + + self.registry.register_custom(struct_comp) \ No newline at end of file diff --git a/CodeGen/schemas/core_types.json b/CodeGen/schemas/core_types.json deleted file mode 100644 index 7193a62..0000000 --- a/CodeGen/schemas/core_types.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "primitives": [ - { - "name": "uint8_t", - "alignment": 1, - "aos": "uint8_t", - "soa": "uint8_t*", - "view": "uint8_t&" - }, - { - "name": "uint16_t", - "alignment": 2, - "aos": "uint16_t", - "soa": "uint16_t*", - "view": "uint16_t&" - }, - { - "name": "uint32_t", - "alignment": 4, - "aos": "uint32_t", - "soa": "uint32_t*", - "view": "uint32_t&" - }, - { - "name": "uint64_t", - "alignment": 8, - "aos": "uint64_t", - "soa": "uint64_t*", - "view": "uint64_t&" - }, - { - "name": "size_t", - "alignment": 8, - "aos": "size_t", - "soa": "size_t*", - "view": "size_t&" - }, - { - "name": "uintptr_t", - "alignment": 8, - "aos": "uintptr_t", - "soa": "uintptr_t*", - "view": "uintptr_t&" - } - ] -} \ No newline at end of file diff --git a/CodeGen/type_registry.py b/CodeGen/type_registry.py index b9d2623..b4d96bb 100644 --- a/CodeGen/type_registry.py +++ b/CodeGen/type_registry.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, Set from dataclasses import dataclass from components import ISchemaComponent @@ -21,12 +21,18 @@ def __init__(self): } self.custom_components: Dict[str, 'ISchemaComponent'] = {} + self._imports: Set[str] = {} def register_custom(self, component: 'ISchemaComponent') -> None: if component.name in self._types or component.name in self.custom_components: raise ValueError(f"Type collision error: '{component.name}' is already registered.") self.custom_components[component.name] = component + def register_import(self, header: str) -> None: + if header in self._imports: + return + self._imports.add(header) + def get_byte_size(self, type_name: str) -> int: if type_name in self._types: return self._types[type_name] From bf8e63334ab28f525cbdcaec0dcfdee457c25acf Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 22:19:44 +0200 Subject: [PATCH 06/25] emitter selection for each structure and schema --- CodeGen/components.py | 1 + CodeGen/emitters/emitter.py | 2 +- CodeGen/main.py | 33 ++++++++++++++++++++++++++++++++- CodeGen/schema_decoder.py | 18 +++++++++++++----- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/CodeGen/components.py b/CodeGen/components.py index 8282c24..cfb53ab 100644 --- a/CodeGen/components.py +++ b/CodeGen/components.py @@ -28,6 +28,7 @@ def calculate_bytes(self, registry): class StructureComponent(ISchemaComponent): _name: str fields: Dict[str, str] = field(default_factory=dict) + target_emitters: List[str] = field(default_factory=list) @property def name(self) -> str: return self.name diff --git a/CodeGen/emitters/emitter.py b/CodeGen/emitters/emitter.py index ea3c970..05b03d9 100644 --- a/CodeGen/emitters/emitter.py +++ b/CodeGen/emitters/emitter.py @@ -5,7 +5,7 @@ class ICodeEmitter(ABC): @abstractmethod - def emit_source(self, registry: 'TypeRegistry', buffer: CodeBuffer) -> None: + def emit_source(self, registry: TypeRegistry, buffer: CodeBuffer) -> None: pass @abstractmethod diff --git a/CodeGen/main.py b/CodeGen/main.py index 4c12213..736c806 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -1,6 +1,20 @@ import sys +import os +from typing import Dict from type_registry import TypeRegistry from schema_decoder import SchemaDecorder +from emitters.emitter import ICodeEmitter +from emitters.soa_emitter import SoALayoutEmitter +from components import StructureComponent +from emitters.code_buffer import CodeBuffer + +AVAILABLE_EMITTERS: Dict[str, ICodeEmitter] = { + "SoAEmitter": SoALayoutEmitter() +} + +def prepare_file(filepath: str) -> None: + root, _ = os.path.split(filepath) + os.makedirs(root, exist_ok=True) def main(): if len(sys.argv) < 3: @@ -8,9 +22,13 @@ def main(): sys.exit(1) schema_path: str = sys.argv[1] - output_header: str = sys.argv[2] + output_header: str = sys.argv[2] + + # create directories if needed + prepare_file(output_header) registry = TypeRegistry() + buffer = CodeBuffer(" "*4) decoder = SchemaDecorder(registry) try: @@ -18,5 +36,18 @@ def main(): except Exception as e: print(f"CODEGEN PIPELINE FAILURE:\n{e}") + for comp in registry.custom_components.values(): + if isinstance(comp, StructureComponent): + for emitter_name in comp.target_emitters: + if emitter_name not in AVAILABLE_EMITTERS: + raise KeyError(f"ERROR: Struct '{comp.name}' requested an emitter that doesn't exist") + + emitter: ICodeEmitter = AVAILABLE_EMITTERS[emitter_name] + emitter.emit_source(registry, buffer) + + print(f"WRITING TARGET: {output_header}") + with open(output_header, "w") as f: + f.write(buffer.get_content()) + if __name__ == "__main__": main() diff --git a/CodeGen/schema_decoder.py b/CodeGen/schema_decoder.py index fb3082e..10ae63e 100644 --- a/CodeGen/schema_decoder.py +++ b/CodeGen/schema_decoder.py @@ -1,11 +1,12 @@ import json from type_registry import TypeRegistry from components import EnumComponent, StructureComponent -from typing import Dict, Any +from typing import Dict, List, Any class SchemaDecorder: def __init__(self, registry: TypeRegistry): self.registry = registry + self.global_emitters: List[str] = [] def decode_file(self, file_path: str): with open(file_path, 'r') as f: @@ -13,6 +14,8 @@ def decode_file(self, file_path: str): data = json.load(f) except json.JSONDecodeError as e: raise ValueError(f"Failed to parse target JSON schema: {e}") + + self.global_emitters = data.get("global_emitters", []) if "enums" in data: self._decode_enums(data["enums"]) @@ -36,13 +39,18 @@ def _decode_enums(self, enum_data: Dict[str, Any]) -> None: self.registry.register_custom(enum_comp) def _decode_structures(self, struct_data: Dict[str, Any]) -> None: - for name, fields in struct_data.items(): - if not isinstance(fields, dict) or not fields: - raise ValueError(f"Structure '{name}' must contain a valid field definition dictionary.") + for name, properties in struct_data.items(): + if "fields" in properties: + field_map = properties["fields"] + local_emitters = properties.get("emitters", self.global_emitters) + else: + field_map = properties + local_emitters = self.global_emitters struct_comp = StructureComponent( _name=name, - fields=dict(fields) + fields=dict(field_map), + target_emitters=list(local_emitters) ) try: From 4a04ba98354022e6bb9bbcbcd2887352f3816592 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 22:28:04 +0200 Subject: [PATCH 07/25] fix circular dependencies --- CodeGen/components.py | 4 +--- CodeGen/emitters/emitter.py | 2 +- CodeGen/emitters/soa_emitter.py | 2 +- CodeGen/type_registry.py | 4 ++-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/CodeGen/components.py b/CodeGen/components.py index cfb53ab..6ea1395 100644 --- a/CodeGen/components.py +++ b/CodeGen/components.py @@ -2,15 +2,13 @@ from dataclasses import dataclass, field from typing import List, Dict -from type_registry import TypeRegistry - class ISchemaComponent(ABC): @property @abstractmethod def name(self) -> str: pass @abstractmethod - def calculate_bytes(self, registry: TypeRegistry) -> int: pass + def calculate_bytes(self, registry: "TypeRegistry") -> int: pass @dataclass class EnumComponent(ISchemaComponent): diff --git a/CodeGen/emitters/emitter.py b/CodeGen/emitters/emitter.py index 05b03d9..ce9c18a 100644 --- a/CodeGen/emitters/emitter.py +++ b/CodeGen/emitters/emitter.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from type_registry import TypeRegistry -from code_buffer import CodeBuffer +from .code_buffer import CodeBuffer from components import EnumComponent, StructureComponent class ICodeEmitter(ABC): diff --git a/CodeGen/emitters/soa_emitter.py b/CodeGen/emitters/soa_emitter.py index e6103a7..27c2ee2 100644 --- a/CodeGen/emitters/soa_emitter.py +++ b/CodeGen/emitters/soa_emitter.py @@ -1,4 +1,4 @@ -from emitter import ICodeEmitter +from .emitter import ICodeEmitter from components import EnumComponent, StructureComponent class SoALayoutEmitter(ICodeEmitter): diff --git a/CodeGen/type_registry.py b/CodeGen/type_registry.py index b4d96bb..2b23f06 100644 --- a/CodeGen/type_registry.py +++ b/CodeGen/type_registry.py @@ -20,10 +20,10 @@ def __init__(self): "bool": 1 } - self.custom_components: Dict[str, 'ISchemaComponent'] = {} + self.custom_components: Dict[str, ISchemaComponent] = {} self._imports: Set[str] = {} - def register_custom(self, component: 'ISchemaComponent') -> None: + def register_custom(self, component: ISchemaComponent) -> None: if component.name in self._types or component.name in self.custom_components: raise ValueError(f"Type collision error: '{component.name}' is already registered.") self.custom_components[component.name] = component From e12507ff8a594bb5f1fba61796a681c9984c3d2b Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 23:13:33 +0200 Subject: [PATCH 08/25] enum emitter --- CodeGen/components.py | 4 +-- CodeGen/emitters/emitter.py | 12 ++------ CodeGen/emitters/enum_emitter.py | 27 +++++++++++++++++ CodeGen/emitters/soa_emitter.py | 50 -------------------------------- CodeGen/main.py | 24 +++++++++------ CodeGen/schema_decoder.py | 2 +- 6 files changed, 47 insertions(+), 72 deletions(-) create mode 100644 CodeGen/emitters/enum_emitter.py delete mode 100644 CodeGen/emitters/soa_emitter.py diff --git a/CodeGen/components.py b/CodeGen/components.py index 6ea1395..113b171 100644 --- a/CodeGen/components.py +++ b/CodeGen/components.py @@ -17,7 +17,7 @@ class EnumComponent(ISchemaComponent): values: List[str] = field(default_factory=list) @property - def name(self) -> str: return self.name + def name(self) -> str: return self._name def calculate_bytes(self, registry): return registry.get_byte_size(self.underlying_type) @@ -29,7 +29,7 @@ class StructureComponent(ISchemaComponent): target_emitters: List[str] = field(default_factory=list) @property - def name(self) -> str: return self.name + def name(self) -> str: return self._name def calculate_bytes(self, registry): total = 0 diff --git a/CodeGen/emitters/emitter.py b/CodeGen/emitters/emitter.py index ce9c18a..d8f5eda 100644 --- a/CodeGen/emitters/emitter.py +++ b/CodeGen/emitters/emitter.py @@ -1,17 +1,9 @@ from abc import ABC, abstractmethod from type_registry import TypeRegistry from .code_buffer import CodeBuffer -from components import EnumComponent, StructureComponent +from components import ISchemaComponent class ICodeEmitter(ABC): @abstractmethod - def emit_source(self, registry: TypeRegistry, buffer: CodeBuffer) -> None: - pass - - @abstractmethod - def _emit_enum(self, enum: EnumComponent, buffer: CodeBuffer) -> None: - pass - - @abstractmethod - def _emit_struct(self, struct: StructureComponent, buffer: CodeBuffer) -> None: + def emit_component(self, buffer: CodeBuffer, component: ISchemaComponent): pass \ No newline at end of file diff --git a/CodeGen/emitters/enum_emitter.py b/CodeGen/emitters/enum_emitter.py new file mode 100644 index 0000000..c5c3073 --- /dev/null +++ b/CodeGen/emitters/enum_emitter.py @@ -0,0 +1,27 @@ +import components +from .emitter import ICodeEmitter + +class EnumEmitter(ICodeEmitter): + def emit_component(self, buffer, component): + if not isinstance(component, components.EnumComponent): + raise TypeError("EnumEmitter cannot process components other than enum!") + + enum: components.EnumComponent = component + buffer.open_block(f"enum class {enum.name} : {enum.underlying_type}") # open enum + + for idx, val in enumerate(enum.values): + buffer.write_line(f"{val:<24} = {idx},") + + buffer.write_line(f"{'INVALID':<24} = 0xff") + buffer.close_block(";\n") # close enum + + buffer.open_block(f"[[nodiscard]] inline constexpr const char* ToString({enum.name} value) noexcept") # open function + buffer.open_block("switch (value)") # open switch + + for val in enum.values: + buffer.write_line(f"case {enum.name}::{val}: return \"{val}\";") + buffer.write_line("default: return \"UNKNOWN\";") + + buffer.close_block() # close switch + buffer.close_block() # close function + buffer.write_line() \ No newline at end of file diff --git a/CodeGen/emitters/soa_emitter.py b/CodeGen/emitters/soa_emitter.py deleted file mode 100644 index 27c2ee2..0000000 --- a/CodeGen/emitters/soa_emitter.py +++ /dev/null @@ -1,50 +0,0 @@ -from .emitter import ICodeEmitter -from components import EnumComponent, StructureComponent - -class SoALayoutEmitter(ICodeEmitter): - def emit_source(self, registry, buffer): - buffer.comment("="*80) - buffer.comment("CODE GENERATED BY DELTA ENGINE CODEGEN UTILITY") - buffer.comment("="*80) - buffer.write_line("#pragma once") - buffer.write_line("#include ") - buffer.write_line("#include ") - buffer.write_line() - - for comp in registry.custom_components.values(): - if isinstance(comp, EnumComponent): - self._emit_enum(comp, buffer) - - for comp in registry.custom_components.values(): - if isinstance(comp, StructureComponent): - self._emit_struct(comp, buffer) - - def _emit_enum(self, enum, buffer): - buffer.open_block(f"enum class {enum.name} : {enum.underlying_type}") - - for idx, val in enumerate(enum.values): - buffer.write_line(f"{val:<24} = {idx},") - - buffer.write_line(f"{'COUNT,':<24} = {idx}") - buffer.write_line(f"{'INVALID':<24} = 0xFF") - buffer.close_block(";\n") - - # string reflection helper - buffer.open_block(f"[[nodiscard]] constexpr const char* ToString({enum.name} value) noexcept") # open function - buffer.open_block("switch (value)") # open switch - - for val in enum.values: - buffer.write_line(f"case {enum.name}::{val}: return \"{val}\";") - buffer.write_line("default: return \"UNKNOWN\";") - - buffer.close_block() # close switch - buffer.close_block() # close function - buffer.write_line() - - def _emit_struct(self, struct, buffer): - buffer.write_line(f"struct {struct.name}SoA") - - for field_name, field_type in struct.fields.items(): - buffer.write_line(f"{field_type:<24}* {field_name};") - - buffer.close_block(";\n") \ No newline at end of file diff --git a/CodeGen/main.py b/CodeGen/main.py index 736c806..391ae0c 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -1,15 +1,16 @@ import sys import os +import components from typing import Dict from type_registry import TypeRegistry from schema_decoder import SchemaDecorder from emitters.emitter import ICodeEmitter -from emitters.soa_emitter import SoALayoutEmitter -from components import StructureComponent from emitters.code_buffer import CodeBuffer +from emitters.enum_emitter import EnumEmitter AVAILABLE_EMITTERS: Dict[str, ICodeEmitter] = { - "SoAEmitter": SoALayoutEmitter() + # "SoAEmitter": SoALayoutEmitter(), + "EnumEmitter": EnumEmitter() } def prepare_file(filepath: str) -> None: @@ -35,15 +36,20 @@ def main(): decoder.decode_file(schema_path) except Exception as e: print(f"CODEGEN PIPELINE FAILURE:\n{e}") + sys.exit(1) + # emit enums for comp in registry.custom_components.values(): - if isinstance(comp, StructureComponent): - for emitter_name in comp.target_emitters: - if emitter_name not in AVAILABLE_EMITTERS: - raise KeyError(f"ERROR: Struct '{comp.name}' requested an emitter that doesn't exist") + # if isinstance(comp, components.StructureComponent): + # for emitter_name in comp.target_emitters: + # if emitter_name not in AVAILABLE_EMITTERS: + # raise KeyError(f"ERROR: Struct '{comp.name}' requested an emitter that doesn't exist") - emitter: ICodeEmitter = AVAILABLE_EMITTERS[emitter_name] - emitter.emit_source(registry, buffer) + # emitter: ICodeEmitter = AVAILABLE_EMITTERS[emitter_name] + # emitter.emit_source(registry, buffer) + if isinstance(comp, components.EnumComponent): + emitter = AVAILABLE_EMITTERS["EnumEmitter"] + emitter.emit_component(buffer, comp) print(f"WRITING TARGET: {output_header}") with open(output_header, "w") as f: diff --git a/CodeGen/schema_decoder.py b/CodeGen/schema_decoder.py index 10ae63e..af9ca47 100644 --- a/CodeGen/schema_decoder.py +++ b/CodeGen/schema_decoder.py @@ -29,7 +29,7 @@ def _decode_enums(self, enum_data: Dict[str, Any]) -> None: raise KeyError(f"Enum schema entry '{name}' must define 'underlying_type'") elif "values" not in properties: raise KeyError(f"Enum schema entry '{name}' must define 'values'") - + enum_comp = EnumComponent( _name=name, underlying_type=properties["underlying_type"], From 224b858df25a37fc936a0fec2c08c5f07981ead2 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 23:24:07 +0200 Subject: [PATCH 09/25] group enums and structures in separate groups --- CodeGen/main.py | 23 +++++++++++------------ CodeGen/type_registry.py | 24 ++++++++++++++++-------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/CodeGen/main.py b/CodeGen/main.py index 391ae0c..a83b184 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -8,9 +8,9 @@ from emitters.code_buffer import CodeBuffer from emitters.enum_emitter import EnumEmitter +enumEmitter = EnumEmitter() AVAILABLE_EMITTERS: Dict[str, ICodeEmitter] = { # "SoAEmitter": SoALayoutEmitter(), - "EnumEmitter": EnumEmitter() } def prepare_file(filepath: str) -> None: @@ -39,17 +39,16 @@ def main(): sys.exit(1) # emit enums - for comp in registry.custom_components.values(): - # if isinstance(comp, components.StructureComponent): - # for emitter_name in comp.target_emitters: - # if emitter_name not in AVAILABLE_EMITTERS: - # raise KeyError(f"ERROR: Struct '{comp.name}' requested an emitter that doesn't exist") - - # emitter: ICodeEmitter = AVAILABLE_EMITTERS[emitter_name] - # emitter.emit_source(registry, buffer) - if isinstance(comp, components.EnumComponent): - emitter = AVAILABLE_EMITTERS["EnumEmitter"] - emitter.emit_component(buffer, comp) + for enum in registry.custom_enums.values(): + enumEmitter.emit_component(buffer, enum) + + for struct in registry.custom_structures.values(): + for emitter_name in struct.target_emitters: + if emitter_name not in AVAILABLE_EMITTERS: + raise KeyError(f"ERROR: Struct '{struct.name}' requested an emitter that doesn't exist") + + emitter: ICodeEmitter = AVAILABLE_EMITTERS[emitter_name] + emitter.emit_component(buffer, struct) print(f"WRITING TARGET: {output_header}") with open(output_header, "w") as f: diff --git a/CodeGen/type_registry.py b/CodeGen/type_registry.py index 2b23f06..514d6e2 100644 --- a/CodeGen/type_registry.py +++ b/CodeGen/type_registry.py @@ -1,8 +1,8 @@ +import components + from typing import Dict, Set from dataclasses import dataclass -from components import ISchemaComponent - class TypeRegistry: def __init__(self): self._types: Dict[str, int] = { @@ -20,13 +20,21 @@ def __init__(self): "bool": 1 } - self.custom_components: Dict[str, ISchemaComponent] = {} + self.custom_structures: Dict[str, components.StructureComponent] = {} + self.custom_enums: Dict[str, components.EnumComponent] = {} self._imports: Set[str] = {} - - def register_custom(self, component: ISchemaComponent) -> None: - if component.name in self._types or component.name in self.custom_components: - raise ValueError(f"Type collision error: '{component.name}' is already registered.") - self.custom_components[component.name] = component + + def register_custom(self, component: components.ISchemaComponent) -> None: + if isinstance(component, components.EnumComponent): + if component.name in self._types or component.name in self.custom_enums: + raise ValueError(f"Type collision error: '{component.name}' is already registered.") + self.custom_enums[component.name] = component + elif isinstance(component, components.StructureComponent): + if component.name in self._types or component.name in self.custom_structures: + raise ValueError(f"Type collision error: '{component.name}' is already registered.") + self.custom_structures[component.name] = component + else: + raise ValueError("Trying to register an unknown component type") def register_import(self, header: str) -> None: if header in self._imports: From 7162dbff7a2d84a87b438c5e173fb066ec481b48 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 23:32:36 +0200 Subject: [PATCH 10/25] write a notice at the top of the file --- CodeGen/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CodeGen/main.py b/CodeGen/main.py index a83b184..1db0adc 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -1,6 +1,5 @@ import sys import os -import components from typing import Dict from type_registry import TypeRegistry from schema_decoder import SchemaDecorder @@ -13,6 +12,9 @@ # "SoAEmitter": SoALayoutEmitter(), } +FILE_BANNER_DECORATOR = "="*100 +FILE_BANNER_TEXT = "FILE GENERATED BY DELTA ENGINE CODEGEN UTILITY - DO NOT MODIFY".center(100, " ") + def prepare_file(filepath: str) -> None: root, _ = os.path.split(filepath) os.makedirs(root, exist_ok=True) @@ -37,6 +39,12 @@ def main(): except Exception as e: print(f"CODEGEN PIPELINE FAILURE:\n{e}") sys.exit(1) + + # write decorator + buffer.comment(FILE_BANNER_DECORATOR) + buffer.comment(FILE_BANNER_TEXT) + buffer.comment(FILE_BANNER_DECORATOR) + buffer.write_line() # emit enums for enum in registry.custom_enums.values(): From 1c8189d69325b239bf360dfff5d7f820dd8e9e7d Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 23:34:49 +0200 Subject: [PATCH 11/25] use absolute path to the header --- CodeGen/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CodeGen/main.py b/CodeGen/main.py index 1db0adc..986c846 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -27,8 +27,9 @@ def main(): schema_path: str = sys.argv[1] output_header: str = sys.argv[2] - # create directories if needed - prepare_file(output_header) + # create directories if needed + output_header_absolute = os.path.abspath(output_header) + prepare_file(output_header_absolute) registry = TypeRegistry() buffer = CodeBuffer(" "*4) @@ -58,8 +59,8 @@ def main(): emitter: ICodeEmitter = AVAILABLE_EMITTERS[emitter_name] emitter.emit_component(buffer, struct) - print(f"WRITING TARGET: {output_header}") - with open(output_header, "w") as f: + print(f"WRITING CODEGEN TARGET: {output_header_absolute}") + with open(output_header_absolute, "w") as f: f.write(buffer.get_content()) if __name__ == "__main__": From 480124dabd40ef0167b3cc58eb175dbae9d63d5a Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 23:39:01 +0200 Subject: [PATCH 12/25] namespace support --- CodeGen/main.py | 7 +++++++ CodeGen/schema_decoder.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/CodeGen/main.py b/CodeGen/main.py index 986c846..b9c202b 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -47,6 +47,10 @@ def main(): buffer.comment(FILE_BANNER_DECORATOR) buffer.write_line() + # write namespace if present + if decoder.namespace: + buffer.open_block(f"namespace {decoder.namespace}") + # emit enums for enum in registry.custom_enums.values(): enumEmitter.emit_component(buffer, enum) @@ -59,6 +63,9 @@ def main(): emitter: ICodeEmitter = AVAILABLE_EMITTERS[emitter_name] emitter.emit_component(buffer, struct) + if decoder.namespace: + buffer.close_block(f" // namespace {decoder.namespace}") + print(f"WRITING CODEGEN TARGET: {output_header_absolute}") with open(output_header_absolute, "w") as f: f.write(buffer.get_content()) diff --git a/CodeGen/schema_decoder.py b/CodeGen/schema_decoder.py index af9ca47..27146ec 100644 --- a/CodeGen/schema_decoder.py +++ b/CodeGen/schema_decoder.py @@ -6,6 +6,7 @@ class SchemaDecorder: def __init__(self, registry: TypeRegistry): self.registry = registry + self.namespace: str = "" self.global_emitters: List[str] = [] def decode_file(self, file_path: str): @@ -23,6 +24,9 @@ def decode_file(self, file_path: str): if "structures" in data: self._decode_structs(data["structures"]) + if "namespace" in data: + self.namespace = data["namespace"] + def _decode_enums(self, enum_data: Dict[str, Any]) -> None: for name, properties in enum_data.items(): if "underlying_type" not in properties: From 8537d51eeb9495a2d3a106830edd8bd4e6510139 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 12 Jul 2026 23:54:17 +0200 Subject: [PATCH 13/25] aos emitter --- CodeGen/components.py | 2 +- CodeGen/emitters/aos_emitter.py | 14 ++++++++++++++ CodeGen/main.py | 4 +++- CodeGen/schema_decoder.py | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 CodeGen/emitters/aos_emitter.py diff --git a/CodeGen/components.py b/CodeGen/components.py index 113b171..e505831 100644 --- a/CodeGen/components.py +++ b/CodeGen/components.py @@ -33,6 +33,6 @@ def name(self) -> str: return self._name def calculate_bytes(self, registry): total = 0 - for field_name, field_type in self.fields.items(): + for _, field_type in self.fields.items(): total += registry.get_byte_size(field_type) return total \ No newline at end of file diff --git a/CodeGen/emitters/aos_emitter.py b/CodeGen/emitters/aos_emitter.py new file mode 100644 index 0000000..e17d588 --- /dev/null +++ b/CodeGen/emitters/aos_emitter.py @@ -0,0 +1,14 @@ +import components +from .emitter import ICodeEmitter + +class AosEmitter(ICodeEmitter): + def emit_component(self, buffer, component): + if not isinstance(component, components.StructureComponent): + raise TypeError("AosEmitter cannot emit components other than structures") + + buffer.open_block(f"struct {component.name}AoS") + for name, type in component.fields.items(): + buffer.write_line(f"{type} {name};") + + buffer.close_block(";") + buffer.write_line() \ No newline at end of file diff --git a/CodeGen/main.py b/CodeGen/main.py index b9c202b..13ab36d 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -6,10 +6,11 @@ from emitters.emitter import ICodeEmitter from emitters.code_buffer import CodeBuffer from emitters.enum_emitter import EnumEmitter +from emitters.aos_emitter import AosEmitter enumEmitter = EnumEmitter() AVAILABLE_EMITTERS: Dict[str, ICodeEmitter] = { - # "SoAEmitter": SoALayoutEmitter(), + "AoSEmitter": AosEmitter() } FILE_BANNER_DECORATOR = "="*100 @@ -55,6 +56,7 @@ def main(): for enum in registry.custom_enums.values(): enumEmitter.emit_component(buffer, enum) + # emit structs for struct in registry.custom_structures.values(): for emitter_name in struct.target_emitters: if emitter_name not in AVAILABLE_EMITTERS: diff --git a/CodeGen/schema_decoder.py b/CodeGen/schema_decoder.py index 27146ec..3628d71 100644 --- a/CodeGen/schema_decoder.py +++ b/CodeGen/schema_decoder.py @@ -22,7 +22,7 @@ def decode_file(self, file_path: str): self._decode_enums(data["enums"]) if "structures" in data: - self._decode_structs(data["structures"]) + self._decode_structures(data["structures"]) if "namespace" in data: self.namespace = data["namespace"] From 89c04b61447a2f565a6a8f7b6484a91ad7db31a7 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Mon, 13 Jul 2026 00:05:24 +0200 Subject: [PATCH 14/25] soa emitter and some type safety --- CodeGen/emitters/aos_emitter.py | 14 +++++--------- CodeGen/emitters/soa_emitter.py | 10 ++++++++++ CodeGen/emitters/structure_emitter.py | 15 +++++++++++++++ CodeGen/main.py | 7 +++++-- 4 files changed, 35 insertions(+), 11 deletions(-) create mode 100644 CodeGen/emitters/soa_emitter.py create mode 100644 CodeGen/emitters/structure_emitter.py diff --git a/CodeGen/emitters/aos_emitter.py b/CodeGen/emitters/aos_emitter.py index e17d588..9e973f3 100644 --- a/CodeGen/emitters/aos_emitter.py +++ b/CodeGen/emitters/aos_emitter.py @@ -1,13 +1,9 @@ -import components -from .emitter import ICodeEmitter +from .structure_emitter import BaseStructureEmitter -class AosEmitter(ICodeEmitter): - def emit_component(self, buffer, component): - if not isinstance(component, components.StructureComponent): - raise TypeError("AosEmitter cannot emit components other than structures") - - buffer.open_block(f"struct {component.name}AoS") - for name, type in component.fields.items(): +class AosEmitter(BaseStructureEmitter): + def _emit_structure(self, buffer, struct): + buffer.open_block(f"struct {struct.name}AoS") + for name, type in struct.fields.items(): buffer.write_line(f"{type} {name};") buffer.close_block(";") diff --git a/CodeGen/emitters/soa_emitter.py b/CodeGen/emitters/soa_emitter.py new file mode 100644 index 0000000..c3ebae7 --- /dev/null +++ b/CodeGen/emitters/soa_emitter.py @@ -0,0 +1,10 @@ +from .structure_emitter import BaseStructureEmitter + +class SoAEmitter(BaseStructureEmitter): + def _emit_structure(self, buffer, struct): + buffer.open_block(f"struct {struct.name}SoA") + for name, type in struct.fields.items(): + buffer.write_line(f"{type}* {name};") + + buffer.close_block(";") + buffer.write_line() \ No newline at end of file diff --git a/CodeGen/emitters/structure_emitter.py b/CodeGen/emitters/structure_emitter.py new file mode 100644 index 0000000..c3407e9 --- /dev/null +++ b/CodeGen/emitters/structure_emitter.py @@ -0,0 +1,15 @@ +from components import StructureComponent +from .emitter import ICodeEmitter +from .code_buffer import CodeBuffer +from abc import abstractmethod + +class BaseStructureEmitter(ICodeEmitter): + def emit_component(self, buffer, component): + if not isinstance(component, StructureComponent): + raise TypeError("AosEmitter cannot emit components other than structures") + + self._emit_structure(buffer, component) + + @abstractmethod + def _emit_structure(self, buffer: CodeBuffer, struct: StructureComponent): + pass \ No newline at end of file diff --git a/CodeGen/main.py b/CodeGen/main.py index 13ab36d..1d37259 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -7,10 +7,13 @@ from emitters.code_buffer import CodeBuffer from emitters.enum_emitter import EnumEmitter from emitters.aos_emitter import AosEmitter +from emitters.soa_emitter import SoAEmitter +from emitters.structure_emitter import BaseStructureEmitter enumEmitter = EnumEmitter() -AVAILABLE_EMITTERS: Dict[str, ICodeEmitter] = { - "AoSEmitter": AosEmitter() +AVAILABLE_EMITTERS: Dict[str, BaseStructureEmitter] = { + "AoSEmitter": AosEmitter(), + "SoAEmitter": SoAEmitter() } FILE_BANNER_DECORATOR = "="*100 From ee223dd353e381e84e3129c8199bbca1223a836b Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Mon, 13 Jul 2026 00:18:18 +0200 Subject: [PATCH 15/25] basic view emitter --- CodeGen/emitters/view_emitter.py | 16 ++++++++++++++++ CodeGen/main.py | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 CodeGen/emitters/view_emitter.py diff --git a/CodeGen/emitters/view_emitter.py b/CodeGen/emitters/view_emitter.py new file mode 100644 index 0000000..7faa161 --- /dev/null +++ b/CodeGen/emitters/view_emitter.py @@ -0,0 +1,16 @@ +from .structure_emitter import BaseStructureEmitter + +class ViewEmitter(BaseStructureEmitter): + def _emit_structure(self, buffer, struct): + struct_type_name = f"{struct.name}View" + + buffer.open_block(f"struct {struct_type_name}") + buffer.comment("delete default constructor") + buffer.write_line(f"{struct_type_name}() = delete;") + buffer.write_line() + + for name, type in struct.fields.items(): + buffer.write_line(f"{type}& {name};") + + buffer.close_block(";") + buffer.write_line() \ No newline at end of file diff --git a/CodeGen/main.py b/CodeGen/main.py index 1d37259..f1b2132 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -8,12 +8,14 @@ from emitters.enum_emitter import EnumEmitter from emitters.aos_emitter import AosEmitter from emitters.soa_emitter import SoAEmitter +from emitters.view_emitter import ViewEmitter from emitters.structure_emitter import BaseStructureEmitter enumEmitter = EnumEmitter() AVAILABLE_EMITTERS: Dict[str, BaseStructureEmitter] = { "AoSEmitter": AosEmitter(), - "SoAEmitter": SoAEmitter() + "SoAEmitter": SoAEmitter(), + "ViewEmitter": ViewEmitter() } FILE_BANNER_DECORATOR = "="*100 From 1000893be1cd98845de7efaa590644e3ae3182f5 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Mon, 13 Jul 2026 00:21:17 +0200 Subject: [PATCH 16/25] rename AoS emitter to default --- CodeGen/emitters/{aos_emitter.py => default_emitter.py} | 4 ++-- CodeGen/main.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename CodeGen/emitters/{aos_emitter.py => default_emitter.py} (73%) diff --git a/CodeGen/emitters/aos_emitter.py b/CodeGen/emitters/default_emitter.py similarity index 73% rename from CodeGen/emitters/aos_emitter.py rename to CodeGen/emitters/default_emitter.py index 9e973f3..14922af 100644 --- a/CodeGen/emitters/aos_emitter.py +++ b/CodeGen/emitters/default_emitter.py @@ -1,8 +1,8 @@ from .structure_emitter import BaseStructureEmitter -class AosEmitter(BaseStructureEmitter): +class DefaultEmitter(BaseStructureEmitter): def _emit_structure(self, buffer, struct): - buffer.open_block(f"struct {struct.name}AoS") + buffer.open_block(f"struct {struct.name}") for name, type in struct.fields.items(): buffer.write_line(f"{type} {name};") diff --git a/CodeGen/main.py b/CodeGen/main.py index f1b2132..367d414 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -6,14 +6,14 @@ from emitters.emitter import ICodeEmitter from emitters.code_buffer import CodeBuffer from emitters.enum_emitter import EnumEmitter -from emitters.aos_emitter import AosEmitter +from emitters.default_emitter import DefaultEmitter from emitters.soa_emitter import SoAEmitter from emitters.view_emitter import ViewEmitter from emitters.structure_emitter import BaseStructureEmitter enumEmitter = EnumEmitter() AVAILABLE_EMITTERS: Dict[str, BaseStructureEmitter] = { - "AoSEmitter": AosEmitter(), + "default": DefaultEmitter(), "SoAEmitter": SoAEmitter(), "ViewEmitter": ViewEmitter() } From af98f5acd6d3d462ef7650bde8004859d1c66388 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Mon, 13 Jul 2026 23:40:19 +0200 Subject: [PATCH 17/25] better INVALID value assignment --- CodeGen/emitters/enum_emitter.py | 5 +++-- CodeGen/main.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CodeGen/emitters/enum_emitter.py b/CodeGen/emitters/enum_emitter.py index c5c3073..da62b81 100644 --- a/CodeGen/emitters/enum_emitter.py +++ b/CodeGen/emitters/enum_emitter.py @@ -12,7 +12,7 @@ def emit_component(self, buffer, component): for idx, val in enumerate(enum.values): buffer.write_line(f"{val:<24} = {idx},") - buffer.write_line(f"{'INVALID':<24} = 0xff") + buffer.write_line(f"{'INVALID':<24} = std::numeric_limits<{enum.underlying_type}>::max()") buffer.close_block(";\n") # close enum buffer.open_block(f"[[nodiscard]] inline constexpr const char* ToString({enum.name} value) noexcept") # open function @@ -20,8 +20,9 @@ def emit_component(self, buffer, component): for val in enum.values: buffer.write_line(f"case {enum.name}::{val}: return \"{val}\";") + buffer.write_line(f"case {enum.name}::INVALID: return \"INVALID\"") buffer.write_line("default: return \"UNKNOWN\";") buffer.close_block() # close switch buffer.close_block() # close function - buffer.write_line() \ No newline at end of file + buffer.write_line() diff --git a/CodeGen/main.py b/CodeGen/main.py index 367d414..8dd8e08 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -53,6 +53,12 @@ def main(): buffer.comment(FILE_BANNER_DECORATOR) buffer.write_line() + # include basic headers + buffer.write_line("#include ") + buffer.write_line("#include ") + buffer.write_line("#include ") + buffer.write_line() + # write namespace if present if decoder.namespace: buffer.open_block(f"namespace {decoder.namespace}") From 1b01c2970893561451d0e1d3d8cc1e0a533d58cf Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Tue, 14 Jul 2026 01:27:52 +0200 Subject: [PATCH 18/25] structurize the generator more for flexibility --- CodeGen/components.py | 38 +++------------- CodeGen/emitters/emitter.py | 9 ---- CodeGen/generators.py | 30 +++++++++++++ CodeGen/interfaces/component.py | 18 ++++++++ CodeGen/interfaces/file_generator.py | 41 ++++++++++++++++++ CodeGen/interfaces/type_registry.py | 23 ++++++++++ CodeGen/main.py | 2 - CodeGen/schema.py | 54 +++++++++++++++++++++++ CodeGen/schema_decoder.py | 65 ---------------------------- CodeGen/type_registries.py | 32 ++++++++++++++ 10 files changed, 203 insertions(+), 109 deletions(-) delete mode 100644 CodeGen/emitters/emitter.py create mode 100644 CodeGen/generators.py create mode 100644 CodeGen/interfaces/component.py create mode 100644 CodeGen/interfaces/file_generator.py create mode 100644 CodeGen/interfaces/type_registry.py create mode 100644 CodeGen/schema.py delete mode 100644 CodeGen/schema_decoder.py create mode 100644 CodeGen/type_registries.py diff --git a/CodeGen/components.py b/CodeGen/components.py index e505831..c2849d5 100644 --- a/CodeGen/components.py +++ b/CodeGen/components.py @@ -1,38 +1,10 @@ -from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import List, Dict - -class ISchemaComponent(ABC): - @property - @abstractmethod - def name(self) -> str: pass - - @abstractmethod - def calculate_bytes(self, registry: "TypeRegistry") -> int: pass +from interfaces.component import IComponent @dataclass -class EnumComponent(ISchemaComponent): - _name: str - underlying_type: str - values: List[str] = field(default_factory=list) - - @property - def name(self) -> str: return self._name - - def calculate_bytes(self, registry): - return registry.get_byte_size(self.underlying_type) - -@dataclass -class StructureComponent(ISchemaComponent): - _name: str - fields: Dict[str, str] = field(default_factory=dict) - target_emitters: List[str] = field(default_factory=list) - - @property - def name(self) -> str: return self._name +class EnumComponent(IComponent): + underlying_type: str = "u32" + values: list[str] = field(default_factory=dict) def calculate_bytes(self, registry): - total = 0 - for _, field_type in self.fields.items(): - total += registry.get_byte_size(field_type) - return total \ No newline at end of file + return registry.get_byte_size(self.underlying_type) \ No newline at end of file diff --git a/CodeGen/emitters/emitter.py b/CodeGen/emitters/emitter.py deleted file mode 100644 index d8f5eda..0000000 --- a/CodeGen/emitters/emitter.py +++ /dev/null @@ -1,9 +0,0 @@ -from abc import ABC, abstractmethod -from type_registry import TypeRegistry -from .code_buffer import CodeBuffer -from components import ISchemaComponent - -class ICodeEmitter(ABC): - @abstractmethod - def emit_component(self, buffer: CodeBuffer, component: ISchemaComponent): - pass \ No newline at end of file diff --git a/CodeGen/generators.py b/CodeGen/generators.py new file mode 100644 index 0000000..1ca51c9 --- /dev/null +++ b/CodeGen/generators.py @@ -0,0 +1,30 @@ +from interfaces.file_generator import IFileGenerator + +class CppFileGenerator(IFileGenerator): + def write_comment(self, text) -> None: + self.write_line(f"// {text}") + + def open_block(self, prefix: str = "") -> None: + if prefix: + self.write_line(prefix) + + self.write_line("{") + self._indent_level += 1 + + def close_block(self, suffix = ""): + self._indent_level = max(0, self._indent_level - 1) + self.write_line(f"}}{suffix}") + + def generate(self, schema_data, registry): + self.write_comment("="*100) + self.write_comment("FILE GENERATED BY DELTA CODEGEN UTILITY".center(100)) + self.write_comment("="*100) + self.write_line() + + self.write_line("#pragma once") + self.write_line("#include ") + self.write_line("#include ") + self.write_line() + + # TODO: Namespace, structures, enums and anything else + diff --git a/CodeGen/interfaces/component.py b/CodeGen/interfaces/component.py new file mode 100644 index 0000000..82b110b --- /dev/null +++ b/CodeGen/interfaces/component.py @@ -0,0 +1,18 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Dict, Any +from type_registries import ITypeRegistry + +@dataclass +class IComponent(ABC): + _name: str + metadata: Dict[str, Any] = field(default_factory=dict) + + @property + def name(self) -> str: + return self._name + + @abstractmethod + def calculate_bytes(self, registry: ITypeRegistry) -> int: + """Calculates the byte footprint of the component.""" + pass \ No newline at end of file diff --git a/CodeGen/interfaces/file_generator.py b/CodeGen/interfaces/file_generator.py new file mode 100644 index 0000000..128ff21 --- /dev/null +++ b/CodeGen/interfaces/file_generator.py @@ -0,0 +1,41 @@ +from abc import ABC, abstractmethod +from typing import List, Dict, Any +from .type_registry import ITypeRegistry +from .component import IComponent + +class IFileGenerator(ABC): + def __init__(self, filepath: str): + self.filepath = filepath + self._lines: List[str] = [] + self._indent_level: int = 0 + self._indent_char: str = " " + + def write_line(self, text: str = "") -> None: + if text.strip() == "": + self._lines.append("") + else: + indent = self._indent_char * self._indent_level + self._lines.append(f"{indent}{text}") + + @abstractmethod + def write_comment(self, text: str) -> None: + pass + + @abstractmethod + def open_block(self, prefix: str = "") -> None: + pass + + @abstractmethod + def close_block(self, suffix: str = "") -> None: + pass + + @abstractmethod + def generate_component(self, component: IComponent): + pass + + def save_to_disk(self) -> None: + content = "\n".join(self._lines) + print(f"--- WRITING FILE: {self.filepath} ---") + + with open(self.filepath, 'w') as f: + f.write(content) \ No newline at end of file diff --git a/CodeGen/interfaces/type_registry.py b/CodeGen/interfaces/type_registry.py new file mode 100644 index 0000000..3cfb070 --- /dev/null +++ b/CodeGen/interfaces/type_registry.py @@ -0,0 +1,23 @@ +from abc import ABC, abstractmethod +from typing import Dict +from .component import IComponent + +class ITypeRegistry(ABC): + def __init__(self): + self.custom_components: Dict[str, IComponent] + pass + + def register_custom(self, component: IComponent) -> None: + if component.name in self.custom_components: + raise KeyError(f"Type Registry Error: component '{component.name}' already exists in the registry.") + self.custom_components[component.name] = component + + @abstractmethod + def resolve_type_string(self, logical_type: str) -> str: + """Translates a logical type (e.g. u32) to a target-language type (e.g. uint32_t)""" + pass + + @abstractmethod + def get_byte_size(self, logical_type: str) -> int: + """Resolved the byte size of a primitive or custom structural layout.""" + pass \ No newline at end of file diff --git a/CodeGen/main.py b/CodeGen/main.py index 8dd8e08..41c8115 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -2,8 +2,6 @@ import os from typing import Dict from type_registry import TypeRegistry -from schema_decoder import SchemaDecorder -from emitters.emitter import ICodeEmitter from emitters.code_buffer import CodeBuffer from emitters.enum_emitter import EnumEmitter from emitters.default_emitter import DefaultEmitter diff --git a/CodeGen/schema.py b/CodeGen/schema.py new file mode 100644 index 0000000..57e8031 --- /dev/null +++ b/CodeGen/schema.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass, field +from abc import ABC, abstractmethod +from typing import Dict, Any, List +from interfaces.component import IComponent + +import json +import components + +@dataclass +class Schema: + generator_type: str + config: Dict[str, Any] = field(default_factory=dict) + components: List[IComponent] = field(default_factory=list) + +class ISchemaDecoder(ABC): + @abstractmethod + def decode(self, file_path: str) -> Schema: + """Reads a file and builds a standarized schema IR wrapper""" + pass + +class JsonSchemaDecoder(ISchemaDecoder): + def decode(self, file_path): + with open(file_path, "r") as f: + raw_data = json.load(f) + + components = self._parse_components(raw_data["components"]) + + return Schema( + generator_type=raw_data["generator"], + config=raw_data["config"], + components=components + ) + + def _parse_components(self, components_data: Dict[str, Any]) -> List[IComponent]: + parsed_components = [] + + for name, body in components_data.items(): + if "type" not in body: + raise KeyError(f"Component '{name}' is missing the required 'type' field.") + + comp_type = body["type"].lower() + + if comp_type == "enum": + parsed_components.append(components.EnumComponent( + _name = name, + underlying_type=body.get("underlying_type", "u32"), + values=list(body.get("values", [])) + )) + elif comp_type == "struct": + pass # TODO: implement + else: + raise ValueError(f"Unknown component type '{comp_type}' for component '{name}'") + + return parsed_components \ No newline at end of file diff --git a/CodeGen/schema_decoder.py b/CodeGen/schema_decoder.py deleted file mode 100644 index 3628d71..0000000 --- a/CodeGen/schema_decoder.py +++ /dev/null @@ -1,65 +0,0 @@ -import json -from type_registry import TypeRegistry -from components import EnumComponent, StructureComponent -from typing import Dict, List, Any - -class SchemaDecorder: - def __init__(self, registry: TypeRegistry): - self.registry = registry - self.namespace: str = "" - self.global_emitters: List[str] = [] - - def decode_file(self, file_path: str): - with open(file_path, 'r') as f: - try: - data = json.load(f) - except json.JSONDecodeError as e: - raise ValueError(f"Failed to parse target JSON schema: {e}") - - self.global_emitters = data.get("global_emitters", []) - - if "enums" in data: - self._decode_enums(data["enums"]) - - if "structures" in data: - self._decode_structures(data["structures"]) - - if "namespace" in data: - self.namespace = data["namespace"] - - def _decode_enums(self, enum_data: Dict[str, Any]) -> None: - for name, properties in enum_data.items(): - if "underlying_type" not in properties: - raise KeyError(f"Enum schema entry '{name}' must define 'underlying_type'") - elif "values" not in properties: - raise KeyError(f"Enum schema entry '{name}' must define 'values'") - - enum_comp = EnumComponent( - _name=name, - underlying_type=properties["underlying_type"], - values=list(properties["values"]) - ) - - self.registry.register_custom(enum_comp) - - def _decode_structures(self, struct_data: Dict[str, Any]) -> None: - for name, properties in struct_data.items(): - if "fields" in properties: - field_map = properties["fields"] - local_emitters = properties.get("emitters", self.global_emitters) - else: - field_map = properties - local_emitters = self.global_emitters - - struct_comp = StructureComponent( - _name=name, - fields=dict(field_map), - target_emitters=list(local_emitters) - ) - - try: - struct_comp.calculate_bytes(self.registry) - except TypeError as e: - raise TypeError(f"Validation for structure template '{name}': {e}") - - self.registry.register_custom(struct_comp) \ No newline at end of file diff --git a/CodeGen/type_registries.py b/CodeGen/type_registries.py new file mode 100644 index 0000000..168bb55 --- /dev/null +++ b/CodeGen/type_registries.py @@ -0,0 +1,32 @@ +from interfaces.type_registry import ITypeRegistry +from typing import Dict, Tuple + +class CppTypeRegistry(ITypeRegistry): + PRIMITIVE_MAP: Dict[str, Tuple[str, int]] = { + "u8": ("uint8_t", 1), + "u16": ("uint16_t", 2), + "u32": ("uint32_t", 4), + "u64": ("uint64_t", 8), + + "i8": ("int8_t", 1), + "i16": ("int16_t", 2), + "i32": ("int32_t", 4), + "i64": ("int64_t", 8), + + "f32": ("float", 4), + "f64": ("double", 8), + + "bool": ("bool", 1), + "usize": ("size_t", 8), + "uintptr": ("uintptr_t", 8) + } + + def resolve_type_string(self, logical_type): + if logical_type in self.PRIMITIVE_MAP: + return self.PRIMITIVE_MAP[logical_type][0] + raise TypeError(f"C++ Type Registry Error: Undefined type reference '{logical_type}'") + + def get_byte_size(self, logical_type): + if logical_type in self.PRIMITIVE_MAP: + return self.PRIMITIVE_MAP[logical_type][1] + raise TypeError(f"C++ Type Registry Error: Undefined type reference '{logical_type}'") \ No newline at end of file From 4c5d08dc5f3104626df754088f0270d0b8076f00 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Tue, 14 Jul 2026 01:28:22 +0200 Subject: [PATCH 19/25] remove old type registry --- CodeGen/type_registry.py | 51 ---------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 CodeGen/type_registry.py diff --git a/CodeGen/type_registry.py b/CodeGen/type_registry.py deleted file mode 100644 index 514d6e2..0000000 --- a/CodeGen/type_registry.py +++ /dev/null @@ -1,51 +0,0 @@ -import components - -from typing import Dict, Set -from dataclasses import dataclass - -class TypeRegistry: - def __init__(self): - self._types: Dict[str, int] = { - "int8_t": 1, "uint8_t": 1, - "int16_t": 2, "uint16_t": 2, - "int32_t": 4, "uint32_t": 4, - "int64_t": 8, "uint64_t": 8, - - "size_t": 8, - "uintptr_t": 8, - "ptrdiff_t": 8, - - "float": 4, - "double": 8, - "bool": 1 - } - - self.custom_structures: Dict[str, components.StructureComponent] = {} - self.custom_enums: Dict[str, components.EnumComponent] = {} - self._imports: Set[str] = {} - - def register_custom(self, component: components.ISchemaComponent) -> None: - if isinstance(component, components.EnumComponent): - if component.name in self._types or component.name in self.custom_enums: - raise ValueError(f"Type collision error: '{component.name}' is already registered.") - self.custom_enums[component.name] = component - elif isinstance(component, components.StructureComponent): - if component.name in self._types or component.name in self.custom_structures: - raise ValueError(f"Type collision error: '{component.name}' is already registered.") - self.custom_structures[component.name] = component - else: - raise ValueError("Trying to register an unknown component type") - - def register_import(self, header: str) -> None: - if header in self._imports: - return - self._imports.add(header) - - def get_byte_size(self, type_name: str) -> int: - if type_name in self._types: - return self._types[type_name] - - if type_name in self.custom_components: - return self._types[type_name].calculate_bytes(self) - - raise TypeError(f"Compilation link error: Unrecognized type name schema referenced: '{type_name}'") \ No newline at end of file From a9e6d1fa92bb371536b2ecb43a1ffc5436a108f0 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Tue, 14 Jul 2026 01:43:30 +0200 Subject: [PATCH 20/25] adjust main.py to the new api --- CodeGen/main.py | 80 +++++++------------------------------------------ 1 file changed, 11 insertions(+), 69 deletions(-) diff --git a/CodeGen/main.py b/CodeGen/main.py index 41c8115..fd7f4d7 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -1,85 +1,27 @@ import sys import os -from typing import Dict -from type_registry import TypeRegistry -from emitters.code_buffer import CodeBuffer -from emitters.enum_emitter import EnumEmitter -from emitters.default_emitter import DefaultEmitter -from emitters.soa_emitter import SoAEmitter -from emitters.view_emitter import ViewEmitter -from emitters.structure_emitter import BaseStructureEmitter -enumEmitter = EnumEmitter() -AVAILABLE_EMITTERS: Dict[str, BaseStructureEmitter] = { - "default": DefaultEmitter(), - "SoAEmitter": SoAEmitter(), - "ViewEmitter": ViewEmitter() -} - -FILE_BANNER_DECORATOR = "="*100 -FILE_BANNER_TEXT = "FILE GENERATED BY DELTA ENGINE CODEGEN UTILITY - DO NOT MODIFY".center(100, " ") - -def prepare_file(filepath: str) -> None: - root, _ = os.path.split(filepath) - os.makedirs(root, exist_ok=True) +import schema def main(): if len(sys.argv) < 3: print("Error: Missing output file path argument from CMake.", file=sys.stderr) sys.exit(1) - schema_path: str = sys.argv[1] - output_header: str = sys.argv[2] + schema_path = sys.argv[1] + output_file = sys.argv[2] - # create directories if needed - output_header_absolute = os.path.abspath(output_header) - prepare_file(output_header_absolute) + schema_path_absolute = os.path.abspath(schema_path) + _, schema_ext = os.path.splitext(schema_path_absolute) + output_file_absolute = os.path.abspath(output_file) - registry = TypeRegistry() - buffer = CodeBuffer(" "*4) - decoder = SchemaDecorder(registry) - - try: - decoder.decode_file(schema_path) - except Exception as e: - print(f"CODEGEN PIPELINE FAILURE:\n{e}") + if schema_ext.lower() == ".json": + s = schema.JsonSchemaDecoder().decode(schema_path_absolute) + else: + print("Schema Parser Error: Unrecognized file type") sys.exit(1) - - # write decorator - buffer.comment(FILE_BANNER_DECORATOR) - buffer.comment(FILE_BANNER_TEXT) - buffer.comment(FILE_BANNER_DECORATOR) - buffer.write_line() - - # include basic headers - buffer.write_line("#include ") - buffer.write_line("#include ") - buffer.write_line("#include ") - buffer.write_line() - - # write namespace if present - if decoder.namespace: - buffer.open_block(f"namespace {decoder.namespace}") - - # emit enums - for enum in registry.custom_enums.values(): - enumEmitter.emit_component(buffer, enum) - - # emit structs - for struct in registry.custom_structures.values(): - for emitter_name in struct.target_emitters: - if emitter_name not in AVAILABLE_EMITTERS: - raise KeyError(f"ERROR: Struct '{struct.name}' requested an emitter that doesn't exist") - - emitter: ICodeEmitter = AVAILABLE_EMITTERS[emitter_name] - emitter.emit_component(buffer, struct) - - if decoder.namespace: - buffer.close_block(f" // namespace {decoder.namespace}") - print(f"WRITING CODEGEN TARGET: {output_header_absolute}") - with open(output_header_absolute, "w") as f: - f.write(buffer.get_content()) + print(schema) if __name__ == "__main__": main() From dc15ce21f7fcbc4760f379fe33282366ac436934 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Tue, 14 Jul 2026 01:43:49 +0200 Subject: [PATCH 21/25] adjust file generator --- CodeGen/interfaces/file_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CodeGen/interfaces/file_generator.py b/CodeGen/interfaces/file_generator.py index 128ff21..09f34a2 100644 --- a/CodeGen/interfaces/file_generator.py +++ b/CodeGen/interfaces/file_generator.py @@ -2,6 +2,7 @@ from typing import List, Dict, Any from .type_registry import ITypeRegistry from .component import IComponent +from schema import Schema class IFileGenerator(ABC): def __init__(self, filepath: str): @@ -30,7 +31,7 @@ def close_block(self, suffix: str = "") -> None: pass @abstractmethod - def generate_component(self, component: IComponent): + def generate(self, schema: Schema, ) -> None: pass def save_to_disk(self) -> None: From fa569ea2fcfbc5ac017d57cf4b278feb0cbcaf1e Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Tue, 14 Jul 2026 01:44:07 +0200 Subject: [PATCH 22/25] fix circular dependencies --- CodeGen/interfaces/component.py | 8 +++++--- CodeGen/interfaces/type_registry.py | 9 ++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CodeGen/interfaces/component.py b/CodeGen/interfaces/component.py index 82b110b..2a3b5f1 100644 --- a/CodeGen/interfaces/component.py +++ b/CodeGen/interfaces/component.py @@ -1,7 +1,9 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Dict, Any -from type_registries import ITypeRegistry +from typing import Dict, Any, TYPE_CHECKING + +if TYPE_CHECKING: + from type_registries import ITypeRegistry @dataclass class IComponent(ABC): @@ -13,6 +15,6 @@ def name(self) -> str: return self._name @abstractmethod - def calculate_bytes(self, registry: ITypeRegistry) -> int: + def calculate_bytes(self, registry: "ITypeRegistry") -> int: """Calculates the byte footprint of the component.""" pass \ No newline at end of file diff --git a/CodeGen/interfaces/type_registry.py b/CodeGen/interfaces/type_registry.py index 3cfb070..2c3556e 100644 --- a/CodeGen/interfaces/type_registry.py +++ b/CodeGen/interfaces/type_registry.py @@ -1,13 +1,16 @@ from abc import ABC, abstractmethod -from typing import Dict +from typing import Dict, TYPE_CHECKING from .component import IComponent +if TYPE_CHECKING: + from .component import IComponent + class ITypeRegistry(ABC): def __init__(self): - self.custom_components: Dict[str, IComponent] + self.custom_components: Dict[str, "IComponent"] pass - def register_custom(self, component: IComponent) -> None: + def register_custom(self, component: "IComponent") -> None: if component.name in self.custom_components: raise KeyError(f"Type Registry Error: component '{component.name}' already exists in the registry.") self.custom_components[component.name] = component From 59273396052096723960e65c985d71aeedefe0c2 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Tue, 14 Jul 2026 01:53:02 +0200 Subject: [PATCH 23/25] exclude python virtual environment and .vscode --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 98591ad..277afb9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .vs +.vscode build out +CodeGen/venv __pycache__ From 1e250f6bd1c85562d662bf8bda166c4b5c769b81 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Tue, 14 Jul 2026 02:13:14 +0200 Subject: [PATCH 24/25] factory functions --- CodeGen/generators.py | 10 ++++++++++ CodeGen/interfaces/file_generator.py | 2 +- CodeGen/main.py | 15 ++++++++------- CodeGen/schema.py | 11 ++++++++++- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/CodeGen/generators.py b/CodeGen/generators.py index 1ca51c9..ab9af53 100644 --- a/CodeGen/generators.py +++ b/CodeGen/generators.py @@ -1,4 +1,8 @@ from interfaces.file_generator import IFileGenerator +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from schema import Schema class CppFileGenerator(IFileGenerator): def write_comment(self, text) -> None: @@ -28,3 +32,9 @@ def generate(self, schema_data, registry): # TODO: Namespace, structures, enums and anything else +def select_generator(schema: "Schema", outfile: str) -> IFileGenerator: + type = schema.generator_type.lower() + if type == "cpp": + return CppFileGenerator(outfile) + else: + raise RuntimeError(f"Unrecognized generator type: '{schema.generator_type}'") diff --git a/CodeGen/interfaces/file_generator.py b/CodeGen/interfaces/file_generator.py index 09f34a2..a72d330 100644 --- a/CodeGen/interfaces/file_generator.py +++ b/CodeGen/interfaces/file_generator.py @@ -31,7 +31,7 @@ def close_block(self, suffix: str = "") -> None: pass @abstractmethod - def generate(self, schema: Schema, ) -> None: + def generate(self, schema: Schema) -> None: pass def save_to_disk(self) -> None: diff --git a/CodeGen/main.py b/CodeGen/main.py index fd7f4d7..d52fa0d 100644 --- a/CodeGen/main.py +++ b/CodeGen/main.py @@ -2,6 +2,7 @@ import os import schema +import generators def main(): if len(sys.argv) < 3: @@ -12,16 +13,16 @@ def main(): output_file = sys.argv[2] schema_path_absolute = os.path.abspath(schema_path) - _, schema_ext = os.path.splitext(schema_path_absolute) output_file_absolute = os.path.abspath(output_file) - if schema_ext.lower() == ".json": - s = schema.JsonSchemaDecoder().decode(schema_path_absolute) - else: - print("Schema Parser Error: Unrecognized file type") + try: + schema_decoder = schema.select_decoder(schema_path_absolute) + s = schema_decoder.decode(schema_path_absolute) + generator = generators.select_generator(s, output_file_absolute) + print(generator) + except RuntimeError as e: + print(e) sys.exit(1) - print(schema) - if __name__ == "__main__": main() diff --git a/CodeGen/schema.py b/CodeGen/schema.py index 57e8031..3681be3 100644 --- a/CodeGen/schema.py +++ b/CodeGen/schema.py @@ -4,6 +4,7 @@ from interfaces.component import IComponent import json +import os import components @dataclass @@ -51,4 +52,12 @@ def _parse_components(self, components_data: Dict[str, Any]) -> List[IComponent] else: raise ValueError(f"Unknown component type '{comp_type}' for component '{name}'") - return parsed_components \ No newline at end of file + return parsed_components + +def select_decoder(schema_file: str) -> ISchemaDecoder: + _, ext = os.path.splitext(schema_file) + + if ext.lower() == ".json": + return JsonSchemaDecoder() + else: + raise RuntimeError(f"Unrecognized file format: '{ext}'") \ No newline at end of file From 02732858cdcc6b82d608dc383a99d29126c49865 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Tue, 14 Jul 2026 02:14:45 +0200 Subject: [PATCH 25/25] safety exclusions --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 277afb9..dcd876b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,8 @@ .vscode build out -CodeGen/venv +venv __pycache__ + +CodeGen/*.json +CodeGen/*.h