diff --git a/pineforge_codegen/analyzer/base.py b/pineforge_codegen/analyzer/base.py index e23355c..21b9323 100644 --- a/pineforge_codegen/analyzer/base.py +++ b/pineforge_codegen/analyzer/base.py @@ -488,28 +488,181 @@ def _propagate_call_site_counts(self) -> None: if isinstance(stmt, FuncDef): func_defs[stmt.name] = stmt + # UDT methods participate in the same stateful call graph as plain + # UDFs. Their analyzer identity is ``Type.method`` while FuncInfo.node + # carries the synthetic FuncDef used by codegen. + func_info_by_name = {fi.name: fi for fi in self._func_infos} + for fi in self._func_infos: + if getattr(fi, "is_udt_method", False) and fi.node is not None: + func_defs.setdefault(fi.name, fi.node) + # Preserve call-node identity as well as the callee name: late clone # materialization needs the textual call's argument mapping to resolve # parameterized TA constructor lengths. - def _find_calls(node, known_funcs: set[str]) -> list[tuple[str, FuncCall]]: + def _resolved_user_call_name(call: FuncCall, owner: str | None) -> str | None: + if isinstance(call.callee, Identifier): + return call.callee.name if call.callee.name in func_defs else None + if not isinstance(call.callee, MemberAccess): + return None + + recv = call.callee.object + method = call.callee.member + udt_name: str | None = None + if isinstance(recv, Identifier): + udt_name = self._udt_var_types.get(recv.name) + owner_info = func_info_by_name.get(owner or "") + if udt_name is None and owner_info is not None \ + and owner_info.node is not None: + if (getattr(owner_info, "is_udt_method", False) + and owner_info.node.params + and recv.name == owner_info.node.params[0]): + udt_name = owner_info.udt_type_name + elif recv.name in owner_info.node.params: + param_idx = owner_info.node.params.index(recv.name) + specs = getattr(owner_info, "param_type_specs", []) or [] + spec = specs[param_idx] if param_idx < len(specs) else None + if spec is not None and spec.kind == "udt": + udt_name = spec.name + if udt_name is None: + spec = self._type_spec_from_expr(recv) + if spec is not None and spec.kind == "udt": + udt_name = spec.name + key = f"{udt_name}.{method}" if udt_name else "" + return key if key in func_defs else None + + def _find_calls(node, known_funcs: set[str], + owner: str | None = None, + seen: set[int] | None = None) -> list[tuple[str, FuncCall]]: calls: list[tuple[str, FuncCall]] = [] - if isinstance(node, FuncCall) and isinstance(node.callee, Identifier): - if node.callee.name in known_funcs: - calls.append((node.callee.name, node)) + if node is None: + return calls + if seen is None: + seen = set() + if isinstance(node, (list, tuple, dict)) or hasattr(node, "__dict__"): + node_id = id(node) + if node_id in seen: + return calls + seen.add(node_id) + if isinstance(node, (list, tuple)): + for item in node: + calls.extend(_find_calls(item, known_funcs, owner, seen)) + return calls + if isinstance(node, dict): + for item in node.values(): + calls.extend(_find_calls(item, known_funcs, owner, seen)) + return calls + if not hasattr(node, "__dict__"): + return calls + if isinstance(node, FuncCall): + resolved = _resolved_user_call_name(node, owner) + if resolved in known_funcs: + calls.append((resolved, node)) for attr_val in vars(node).values(): - if isinstance(attr_val, list): - for item in attr_val: - if hasattr(item, '__dict__'): - calls.extend(_find_calls(item, known_funcs)) - elif hasattr(attr_val, '__dict__'): - calls.extend(_find_calls(attr_val, known_funcs)) + calls.extend(_find_calls(attr_val, known_funcs, owner, seen)) return calls known_func_names = set(func_defs.keys()) calls_by_parent = { - name: _find_calls(func_def, known_func_names) + name: _find_calls(func_def, known_func_names, name) for name, func_def in func_defs.items() } + calls_by_callee: dict[str, list[FuncCall]] = { + name: [] for name in known_func_names + } + # Preserve source order across definitions and top-level statements. + # Method bodies use their ``Type.method`` owner so ``self.sibling()`` + # resolves without relying on a now-exited symbol-table scope. + for stmt in self._ast.body: + if isinstance(stmt, FuncDef): + owner = stmt.name + elif isinstance(stmt, MethodDef): + owner = f"{stmt.type_name}.{stmt.name}" + else: + owner = None + for callee, call in _find_calls(stmt, known_func_names, owner): + calls_by_callee.setdefault(callee, []).append(call) + + # Codegen synthesizes a Series buffer for two expression shapes that + # do not appear in ``_func_series_vars`` themselves: + # + # * a call result read through history, e.g. ``f()[1]``; + # * a scalar expression bridged into a UDF series parameter, e.g. + # ``history(close + open)`` where ``history(src) => src[1]``. + # + # A buffer is mutable per-call-site state just like TA/fixnan. Mark + # its lexical owner stateful before the normal call-path closure so a + # function invoked from two source sites receives two emitted bodies + # (and therefore two independent generated buffer members). Without + # this, moving the old function-local static into a class member would + # accidentally merge both Pine call sites into one history stream. + def _actual_arg(call: FuncCall, param_name: str, param_idx: int): + if param_idx < len(call.args): + return call.args[param_idx] + return call.kwargs.get(param_name) + + def _needs_scalar_series_bridge(call: FuncCall) -> bool: + if not isinstance(call.callee, Identifier): + return False + callee = call.callee.name + fi = func_info_by_name.get(callee) + if fi is None or fi.node is None: + return False + series_params = self._func_series_vars.get(callee, set()) + if not series_params: + return False + direct_bar_series = { + "open", "high", "low", "close", "volume", + "hl2", "hlc3", "ohlc4", + } + for idx, param_name in enumerate(fi.node.params): + if param_name not in series_params: + continue + arg = _actual_arg(call, param_name, idx) + if arg is None: + continue + if isinstance(arg, Identifier) and ( + arg.name in direct_bar_series or arg.name in self._series_vars + ): + continue + return True + return False + + def _has_synthetic_history_state( + node, seen: set[int] | None = None) -> bool: + if node is None: + return False + if seen is None: + seen = set() + if isinstance(node, (list, tuple, dict)) or hasattr(node, "__dict__"): + node_id = id(node) + if node_id in seen: + return False + seen.add(node_id) + if isinstance(node, (list, tuple)): + return any( + _has_synthetic_history_state(item, seen) for item in node + ) + if isinstance(node, dict): + return any( + _has_synthetic_history_state(item, seen) + for item in node.values() + ) + if not hasattr(node, "__dict__"): + return False + if (isinstance(node, Subscript) + and isinstance(node.object, FuncCall)): + return True + if isinstance(node, FuncCall) and _needs_scalar_series_bridge(node): + return True + return any( + _has_synthetic_history_state(value, seen) + for value in vars(node).values() + ) + + synthetic_history_stateful = { + name for name, func_def in func_defs.items() + if _has_synthetic_history_state(func_def) + } # request.security owns a separate evaluator context and already # materializes/remaps its embedded TA state per SecurityCallInfo. Do @@ -524,7 +677,9 @@ def _find_calls(node, known_funcs: set[str]) -> list[tuple[str, FuncCall]]: expression = getattr(sec, "expression", None) if expression is not None: security_boundary_funcs.update( - sub for sub, _ in _find_calls(expression, known_func_names) + sub for sub, _ in _find_calls( + expression, known_func_names, containing or None + ) ) # Canonical direct-state predicate. TA-only and fixnan-only helpers @@ -534,6 +689,7 @@ def _find_calls(node, known_funcs: set[str]) -> list[tuple[str, FuncCall]]: | set(self._func_var_members) | set(self._func_ta_ranges) | set(self._func_fixnan_indices) + | synthetic_history_stateful ) # Close upward over the call graph so a pure A -> B -> stateful C chain @@ -566,7 +722,7 @@ def _find_calls(node, known_funcs: set[str]) -> list[tuple[str, FuncCall]]: # disturbing any indices already assigned by the visitor or security # monomorphization. for fname in sorted(stateful): - calls = list(self._iter_user_func_calls(fname)) + calls = calls_by_callee.get(fname, []) current = self._func_call_site_count.get(fname, 0) next_idx = current for call in calls: diff --git a/pineforge_codegen/codegen/base.py b/pineforge_codegen/codegen/base.py index c71120a..4e70dff 100644 --- a/pineforge_codegen/codegen/base.py +++ b/pineforge_codegen/codegen/base.py @@ -381,6 +381,14 @@ def __init__(self, ctx: AnalyzerContext) -> None: self._security_inline_counter = 0 self._random_call_counter = 0 self._for_counter = 0 + # Synthetic history buffers used by inline call-history and by scalar + # expressions passed to UDF series parameters. They are pre-registered + # at generate() time so declarations precede method emission, then + # addressed by (source node, emitted UDF variant). Each record is a + # real class-member Series and therefore joins _PFScriptState through + # the declaration-derived checkpoint inventory. + self._inline_history_members: list[dict] = [] + self._inline_history_member_by_key: dict[tuple, str] = {} # Unique lambda-local names used when an array lowering references its # receiver more than once. The binding keeps temporary-producing or # side-effectful receivers single-evaluation (see TypeInferer). @@ -796,7 +804,8 @@ def _build_func_instances(self) -> None: stateful = (set(ctx.func_ta_ranges.keys()) | set(ctx.func_var_members.keys()) | set(ctx.func_series_vars.keys()) - | set(ctx.func_fixnan_indices.keys())) + | set(ctx.func_fixnan_indices.keys()) + | set(ctx.func_security_clone_only)) if not stateful: return @@ -816,7 +825,7 @@ def fixnan_originals(fname: str) -> list[str]: return list(self._func_cs_fixnan_remap.get((fname, 0), {}).keys()) def natural_name(fname: str, cs_idx: int) -> str: - return f"{self._func_safe_name(fname)}_cs{cs_idx}" + return f"{self._func_cpp_base_name(fname)}_cs{cs_idx}" interned: dict[tuple, dict] = {} worklist: list[dict] = [] @@ -824,7 +833,7 @@ def natural_name(fname: str, cs_idx: int) -> str: fresh_counter = 0 # Seed with the natural clones the flat emission loop produces. - for fname in stateful: + for fname in sorted(stateful): if fname not in func_bodies: continue total_cs = ctx.func_call_site_counts.get(fname, 0) @@ -840,7 +849,7 @@ def natural_name(fname: str, cs_idx: int) -> str: else: worklist.append({ "fname": fname, - "name": self._func_safe_name(fname), + "name": self._func_cpp_base_name(fname), "ta_remap": {}, "var_remap": {}, "fixnan_remap": {}, @@ -883,7 +892,7 @@ def natural_name(fname: str, cs_idx: int) -> str: ginst = interned.get(key) if ginst is None: fresh_counter += 1 - inst_name = f"{self._func_safe_name(g_name)}__ni{fresh_counter}" + inst_name = f"{self._func_cpp_base_name(g_name)}__ni{fresh_counter}" fvar_remap: dict[str, str] = {} for v in var_originals(g_name): fresh_member = f"{v}__ni{fresh_counter}" @@ -1742,11 +1751,163 @@ def walk(node): walk(v) walk(self.ctx.ast) + def _func_cpp_base_name(self, fname: str) -> str: + """Return the actual emitted C++ base name for a UDF or UDT method.""" + fi = self._func_info_map.get(fname) + if fi is not None and getattr(fi, "is_udt_method", False): + return self._emit_udt_method_cpp_name(fi) + return self._func_safe_name(fname) + + def _inline_history_contexts_for_owner(self, owner: str | None) -> list[str | None]: + """Return every method-emission context that owns one source AST site. + + ``None`` denotes top-level/on_bar or a function emitted exactly once. + Stateful UDF clones and fresh nested-helper instances use the same names + assigned to ``_current_instance_name`` by ``_emit_func_def`` so lookup + while visiting a body is deterministic and cannot collapse call sites. + """ + if owner is None: + return [None] + if owner in self._dead_func_names: + return [] + + total_cs = self.ctx.func_call_site_counts.get(owner, 0) + cloned = ( + owner in self.ctx.func_ta_ranges + or owner in self.ctx.func_series_vars + or owner in self.ctx.func_var_members + or owner in self.ctx.func_security_clone_only + ) and total_cs > 0 + if cloned: + contexts: list[str | None] = [ + f"{self._func_cpp_base_name(owner)}_cs{idx}" + for idx in range(total_cs) + ] + else: + contexts = [None] + + for inst in self._fresh_instances: + if inst["fname"] == owner and inst["name"] not in contexts: + contexts.append(inst["name"]) + return contexts + + def _prepare_inline_history_members(self) -> None: + """Pre-register every generated temporary-Series class member. + + Member declarations and the declaration-derived rollback aggregate are + emitted before function/on_bar bodies. A source-order AST pass therefore + reserves stable names up front. The key includes an emitted UDF context + because the same body AST is rendered once per stateful call-site clone. + """ + self._inline_history_members = [] + self._inline_history_member_by_key = {} + counters = {"hist_call": 0, "series_arg": 0} + + def walk_nodes(value): + """Yield AST nodes in stable field order, including tuple elements. + + NamingHelper._walk_ast predates several AST containers and is + intentionally a best-effort utility. Member pre-registration must + be exhaustive because a missed node becomes an undeclared C++ + member, so use the dataclass field graph directly here. + """ + if isinstance(value, ASTNode): + yield value + for child in vars(value).values(): + yield from walk_nodes(child) + return + if isinstance(value, (list, tuple)): + for child in value: + yield from walk_nodes(child) + return + if isinstance(value, dict): + for child in value.values(): + yield from walk_nodes(child) + return + if isinstance(value, TypeField) and value.default is not None: + yield from walk_nodes(value.default) + + owner_by_node: dict[int, str] = {} + for fi in self.ctx.func_infos: + if fi.node is None: + continue + for child in walk_nodes(fi.node): + owner_by_node[id(child)] = fi.name + + def register(kind: str, source_key: tuple, cpp_type: str, + owner: str | None) -> None: + if cpp_type not in ("double", "int", "bool"): + cpp_type = "double" + for context in self._inline_history_contexts_for_owner(owner): + key = (kind, *source_key, context) + if key in self._inline_history_member_by_key: + continue + counters[kind] += 1 + member_name = f"_{kind}_{counters[kind]}" + self._inline_history_member_by_key[key] = member_name + self._inline_history_members.append({ + "kind": kind, + "member_name": member_name, + "cpp_type": cpp_type, + "context": context, + }) + + def actual_args_for(call: FuncCall, params: list[str]) -> list: + if call.kwargs: + return _merge_kwargs(call.args, call.kwargs, params, lambda arg: arg) + return list(call.args) + + for node in walk_nodes(self.ctx.ast): + owner = owner_by_node.get(id(node)) + if isinstance(node, Subscript) and isinstance(node.object, FuncCall): + register( + "hist_call", (id(node),), self._infer_type(node.object), owner + ) + + if not isinstance(node, FuncCall): + continue + func_name, _ = self._resolve_callee(node.callee) + fi = self._func_info_map.get(func_name) + if fi is None or fi.node is None: + continue + func_sv = self.ctx.func_series_vars.get(fi.name, set()) + series_param_indices = { + idx for idx, name in enumerate(fi.node.params) if name in func_sv + } + if not series_param_indices: + continue + args = actual_args_for(node, list(fi.node.params)) + for idx, arg in enumerate(args): + if idx not in series_param_indices: + continue + if isinstance(arg, Identifier): + if arg.name in BAR_FIELDS or arg.name in BAR_SERIES_PUSH: + continue + if arg.name in self.ctx.series_vars: + continue + register( + "series_arg", (id(node), idx), self._infer_type(arg), owner + ) + + def _inline_history_member(self, kind: str, node: ASTNode, + arg_idx: int | None = None) -> str: + source_key = (id(node),) if arg_idx is None else (id(node), arg_idx) + key = (kind, *source_key, self._current_instance_name) + member = self._inline_history_member_by_key.get(key) + if member is None: + raise AssertionError( + "missing pre-registered inline history member for " + f"{kind} at {getattr(node, 'loc', None)} in context " + f"{self._current_instance_name!r}" + ) + return member + def generate(self) -> str: """Generate C++ source from the AnalyzerContext.""" # Context-sensitive instance pre-pass (needs the naming helpers populated # in __init__). Computes nested stateful-helper dispatch + fresh instances. self._build_func_instances() + self._prepare_inline_history_members() # Pre-scan for strategy series vars self._prescan_strategy_series() self._security_ohlc_hist_fields_by_sec: dict[int, set[str]] = {} @@ -1820,6 +1981,7 @@ def generate(self) -> str: # 2. Open class lines.append("class GeneratedStrategy : public BacktestEngine {") lines.append("public:") + _script_state_decl_start = len(lines) # request.security state for item in self._security_calls: @@ -2051,6 +2213,14 @@ def generate(self) -> str: else: lines.append(f" Series {svar}{_mbb};") + # 8a. Synthetic temporary history. Unlike the legacy function-local + # static buffers, these members are value-copyable rollback state and + # have one identity per source site / emitted UDF variant. + for info in self._inline_history_members: + lines.append( + f" Series<{info['cpp_type']}> {info['member_name']}{_mbb};" + ) + # 8b. Global-scope non-var declarations as class members # (so user-defined functions can reference them) seen_global = set() @@ -2173,6 +2343,16 @@ def generate(self) -> str: lines.append("") + # 9d. Historical execution rollback checkpoint. Derive the member + # inventory from the declarations above so every future generated + # state category is captured automatically (or generation fails loudly + # if it introduces an unfamiliar declaration form). + _script_state_members = self._collect_script_state_members( + lines[_script_state_decl_start:-1] + ) + self._emit_script_state_hooks(lines, _script_state_members) + lines.append("") + # 9. Constructor with TA initializer list self._emit_constructor(lines) lines.append("") diff --git a/pineforge_codegen/codegen/emit_top.py b/pineforge_codegen/codegen/emit_top.py index 9f6c3ee..4f33b36 100644 --- a/pineforge_codegen/codegen/emit_top.py +++ b/pineforge_codegen/codegen/emit_top.py @@ -118,6 +118,8 @@ def _emit_includes(self, lines: list[str]) -> None: lines.append("#include ") lines.append("#include ") lines.append("#include ") + lines.append("#include ") + lines.append("#include ") lines.append("#include ") lines.append("#include ") lines.append("#include ") @@ -192,6 +194,139 @@ def _script_has_input_source(self) -> bool: return True return False + @staticmethod + def _script_state_member_name(decl_line: str) -> str | None: + """Extract a generated class-member name from one declaration line. + + ``CodeGen.generate`` emits the complete persistent script state as a + contiguous block of one-line declarations before the constructor. The + rollback checkpoint is derived from that block rather than from a + second, hand-maintained inventory: adding a new TA/helper/collection + member therefore automatically makes it part of Pine's historical + execution rollback. + + The supported declaration shapes are the only shapes emitted in that + block today:: + + Type name; + Type name = value; + Type name(args); + Type name{args}; + + A declaration that does not match fails generation loudly. Silently + omitting an unfamiliar member would be materially worse: it would let + state leak between ``calc_on_order_fills`` executions. + """ + text = decl_line.strip() + if not text: + return None + if not text.endswith(";"): + raise AssertionError( + f"unexpected generated script-state declaration: {decl_line!r}" + ) + text = text[:-1].rstrip() + if " = " in text: + text = text.split(" = ", 1)[0].rstrip() + else: + # Series members can carry a max_bars_back ctor suffix and + # drawing arenas use brace initialization. Both suffixes start in + # the final declarator token; C++ types emitted here never contain + # parentheses or braces. + last_space = text.rfind(" ") + if last_space < 0: + raise AssertionError( + f"missing type in generated script-state declaration: {decl_line!r}" + ) + declarator = text[last_space + 1:] + cut = len(declarator) + for marker in ("(", "{"): + pos = declarator.find(marker) + if pos >= 0: + cut = min(cut, pos) + text = text[:last_space + 1] + declarator[:cut] + + name = text.rsplit(" ", 1)[-1].strip() + if not name or not (name[0].isalpha() or name[0] == "_") \ + or not all(ch.isalnum() or ch == "_" for ch in name): + raise AssertionError( + f"cannot identify generated script-state member: {decl_line!r}" + ) + return name + + def _collect_script_state_members(self, declaration_lines: list[str]) -> list[str]: + """Return every rollback-relevant generated member in declaration order. + + Precalculated TA result vectors and their mode flag are immutable once + the engine starts its broker walk. Copying an O(number-of-bars) cache + before every COOF execution would be both unnecessary and catastrophic, + so those implementation caches are the sole exclusions. The live TA + objects themselves remain captured because dynamic/magnifier runs call + ``compute`` and mutate them. + """ + members: list[str] = [] + seen: set[str] = set() + for line in declaration_lines: + name = self._script_state_member_name(line) + if name is None: + continue + if name == "_use_precalc" or name.startswith("_precalc_"): + continue + if name in seen: + raise AssertionError(f"duplicate generated script-state member: {name}") + seen.add(name) + members.append(name) + return members + + def _emit_script_state_hooks(self, lines: list[str], members: list[str]) -> None: + """Emit the engine's Pine rollback checkpoint hook implementation. + + The checkpoint owns value copies of all generated mutable state. Every + runtime container used by generated code (Series, std::vector/map, + PineMatrix/generic matrices, UDTs and drawing arenas) has value + semantics, so copying recursively preserves data without retaining + pointers into live state. Drawing handles themselves are stable ids; + their arenas are captured in the same checkpoint. + + The static assertions deliberately turn any future non-copyable member + into a compile failure instead of a nominal, shallow rollback. Engine + broker/order state lives in the base class and is intentionally absent: + fills must survive while Pine script variables roll back. + """ + lines.append(" struct _PFScriptState {") + for idx, name in enumerate(members): + lines.append( + f" decltype(GeneratedStrategy::{name}) _pf_value_{idx};" + ) + lines.append(" };") + lines.append( + " static_assert(std::is_copy_constructible_v<_PFScriptState>, " + '"generated Pine state must be deep-copy constructible");' + ) + lines.append( + " static_assert(std::is_copy_assignable_v<_PFScriptState>, " + '"generated Pine state must be deep-copy assignable");' + ) + lines.append(" std::optional<_PFScriptState> _pf_script_state_checkpoint_;") + lines.append("") + lines.append(" void snapshot_script_state() override {") + lines.append(" _pf_script_state_checkpoint_.emplace(_PFScriptState{") + for name in members: + lines.append(f" {name},") + lines.append(" });") + lines.append(" }") + lines.append("") + lines.append(" void restore_script_state() override {") + lines.append(" if (!_pf_script_state_checkpoint_) return;") + for idx, name in enumerate(members): + lines.append( + f" this->{name} = _pf_script_state_checkpoint_->_pf_value_{idx};" + ) + lines.append(" }") + lines.append("") + lines.append(" void commit_script_state() override {") + lines.append(" snapshot_script_state();") + lines.append(" }") + def _typed_na_init(self, cpp_val: str, name: str, ptype) -> str: """Re-type a bare ``na()`` initializer to match a non-double member's C++ type. A ``var int x = na`` resolves its RHS to @@ -300,6 +435,9 @@ def _emit_constructor(self, lines: list[str]) -> None: if sp.get("process_orders_on_close") is True: ctor_body.append(" process_orders_on_close_ = true;") + if sp.get("calc_on_order_fills") is True: + ctor_body.append(" calc_on_order_fills_ = true;") + if "initial_capital" in sp and isinstance(sp["initial_capital"], (int, float)): ctor_body.append(f" initial_capital_ = {float(sp['initial_capital'])};") @@ -384,6 +522,7 @@ def _emit_constructor(self, lines: list[str]) -> None: lines.append(' if (key == "pyramiding") { pyramiding_ = std::stoi(value); return; }') lines.append(' if (key == "slippage") { slippage_ = std::stoi(value); return; }') lines.append(' if (key == "process_orders_on_close") { process_orders_on_close_ = (value == "true" || value == "1"); return; }') + lines.append(' if (key == "calc_on_order_fills") { calc_on_order_fills_ = (value == "true" || value == "1"); return; }') lines.append(' if (key == "close_entries_rule") { close_entries_rule_any_ = (value == "ANY" || value == "any" || value == "1"); return; }') lines.append(' if (key == "default_qty_type") {') lines.append(' if (value == "fixed" || value == "strategy.fixed" || value == "0") default_qty_type_ = QtyType::FIXED;') @@ -452,14 +591,45 @@ def _emit_constructor(self, lines: list[str]) -> None: "initial_capital": "initial_capital_", } + @staticmethod + def _emit_history_series_write( + lines: list[str], pad: str, member: str, value: str) -> None: + """Emit one Pine-series write without conflating history with isnew. + + Historical fill recalculations keep ``barstate.isnew`` true, but a + post-close recalculation restored from the completed ordinary-close + checkpoint must replace that bar's current history slot rather than + append a duplicate slot. The engine exposes those independent facts + as ``is_first_tick_`` and ``history_advances_new_bar()`` respectively. + """ + lines.append( + f"{pad}if (history_advances_new_bar()) {member}.push({value});" + ) + lines.append(f"{pad}else {member}.update({value});") + def _emit_on_bar(self, lines: list[str]) -> None: lines.append(" void on_bar(const Bar& bar) override {") + # reset_run_state() owns engine/broker state, while these generated + # Series members belong to the strategy object. Clear all of them on + # the first genuine history slot of bar zero, unconditionally: a site + # may live behind a branch that does not execute on bar zero. This is a + # narrow synthetic-buffer reset, not a promise that every generated + # member/init latch supports full same-handle reruns. The post-C rollback + # execution has history_advances_new_bar()==false, so it preserves the + # committed slot. + for info in self._inline_history_members: + lines.append( + " if (history_advances_new_bar() && bar_index_ == 0) " + f"{info['member_name']}.clear();" + ) + # a. Push bar field series (with bar magnifier support) for field_name in sorted(self.ctx.series_bar_fields): push_expr = BAR_SERIES_PUSH.get(field_name, f"current_bar_.{field_name}") - lines.append(f" if (is_first_tick_) _s_{field_name}.push({push_expr});") - lines.append(f" else _s_{field_name}.update({push_expr});") + self._emit_history_series_write( + lines, " ", f"_s_{field_name}", push_expr + ) # a1. Push history-referenced scalar bar builtins (time[n], bar_index[n], # hl2[n], …). They land in ``series_vars`` and are declared as Series @@ -477,14 +647,13 @@ def _emit_on_bar(self, lines: list[str]) -> None: if _bexpr is None or _bexpr.strip().startswith(f"{_bname}("): continue _bsafe = self._safe_name(_bname) - lines.append(f" if (is_first_tick_) {_bsafe}.push({_bexpr});") - lines.append(f" else {_bsafe}.update({_bexpr});") + self._emit_history_series_write(lines, " ", _bsafe, _bexpr) # a2. Push strategy series for svar in sorted(self._strategy_series_vars): member = svar.replace("_strat_", "") push_expr = self._STRAT_SERIES_PUSH.get(member, "0") - lines.append(f" {svar}.push({push_expr});") + self._emit_history_series_write(lines, " ", svar, push_expr) # b. Var init / carry-forward if self.ctx.var_members: @@ -556,7 +725,9 @@ def _emit_on_bar(self, lines: list[str]) -> None: if name in self._array_vars: continue if name in self.ctx.series_vars: - lines.append(f" if (is_first_tick_) {safe}.push({safe}[0]);") + self._emit_history_series_write( + lines, " ", safe, f"{safe}[0]" + ) # Also carry-forward cloned copies for per-call-site function variants carry_emitted: set[str] = set() for (fname, cs_idx), remap in self._func_cs_var_remap.items(): @@ -566,7 +737,9 @@ def _emit_on_bar(self, lines: list[str]) -> None: cloned = remap[safe] if cloned not in carry_emitted: carry_emitted.add(cloned) - lines.append(f" if (is_first_tick_) {cloned}.push({cloned}[0]);") + self._emit_history_series_write( + lines, " ", cloned, f"{cloned}[0]" + ) lines.append(" }") # c. Push non-var series (they start fresh each bar with a push) @@ -875,7 +1048,11 @@ def _emit_func_def(self, fi: FuncInfo, lines: list[str], call_site_idx: int | No self._active_fixnan_remap = self._func_cs_fixnan_remap.get((fi.name, call_site_idx), {}) self._in_ta_func_variant = True self._active_call_site_idx = call_site_idx - self._current_instance_name = f"{self._func_safe_name(fi.name)}_cs{call_site_idx}" + # Use the actual emitted name. Plain UDFs are unchanged; UDT + # methods carry their `_udt_Type_method` prefix. This identity is + # shared with _build_func_instances and synthetic-history member + # registration, so method call paths can dispatch independently. + self._current_instance_name = func_name else: self._active_ta_remap = {} self._active_var_remap = {} diff --git a/pineforge_codegen/codegen/visit_call.py b/pineforge_codegen/codegen/visit_call.py index 5ec9af5..b94889e 100644 --- a/pineforge_codegen/codegen/visit_call.py +++ b/pineforge_codegen/codegen/visit_call.py @@ -224,6 +224,25 @@ class CallVisitor: # Function-call dispatch # ------------------------------------------------------------------ + def _udt_method_call_emit_name(self, fi, node: FuncCall) -> str: + """Resolve a UDT method call through the ordinary UDF clone graph.""" + base = self._emit_udt_method_cpp_name(fi) + dispatch = self._instance_dispatch.get( + (self._current_instance_name, id(node)) + ) + if dispatch is not None: + return dispatch + + cs_info = self.ctx.func_call_cs_map.get(id(node)) + if self._active_call_site_idx is not None and cs_info is not None: + return f"{base}_cs{self._active_call_site_idx}" + if cs_info is not None and cs_info[0] == fi.name: + return f"{base}_cs{cs_info[1]}" + if (self._active_call_site_idx is not None + and self.ctx.func_call_site_counts.get(fi.name, 0) > 1): + return f"{base}_cs{self._active_call_site_idx}" + return base + def _array_init_value_expr(self, elem_spec: TypeSpec | None, value_node) -> str: if isinstance(value_node, NaLiteral): if elem_spec is not None and elem_spec.kind == "udt": @@ -270,7 +289,7 @@ def _visit_func_call(self, node: FuncCall) -> str: mk = f"{recv_spec.name}.{callee.member}" fi_u = self._func_info_map.get(mk) if fi_u is not None and getattr(fi_u, "is_udt_method", False): - fn_cpp = self._emit_udt_method_cpp_name(fi_u) + fn_cpp = self._udt_method_call_emit_name(fi_u, node) recv_e = self._visit_expr(callee.object) param_names = list(fi_u.node.params[1:]) if fi_u.node else [] # Drop the leading ``self`` slot from param_defaults so the @@ -380,7 +399,7 @@ def _visit_func_call(self, node: FuncCall) -> str: mk = f"{udt_t}.{meth_raw}" fi_u = self._func_info_map.get(mk) if fi_u is not None and getattr(fi_u, "is_udt_method", False): - fn_cpp = self._emit_udt_method_cpp_name(fi_u) + fn_cpp = self._udt_method_call_emit_name(fi_u, node) recv_e = self._visit_expr(obj) param_names = list(fi_u.node.params[1:]) if fi_u.node else [] # Drop the leading ``self`` slot so param_defaults @@ -448,8 +467,15 @@ def _visit_func_call(self, node: FuncCall) -> str: if getattr(self, "_precalc_loop_active", False) and uses_precalc: return f"_precalc_{ta_mem}[i]" if uses_precalc: - return f"(_use_precalc ? _precalc_{ta_mem}[bar_index_] : (is_first_tick_ ? {ta_mem}.compute({compute_args}) : {ta_mem}.recompute({compute_args})))" - return f"(is_first_tick_ ? {ta_mem}.compute({compute_args}) : {ta_mem}.recompute({compute_args}))" + return ( + f"(_use_precalc ? _precalc_{ta_mem}[bar_index_] : " + f"(history_advances_new_bar() ? {ta_mem}.compute({compute_args}) " + f": {ta_mem}.recompute({compute_args})))" + ) + return ( + f"(history_advances_new_bar() ? {ta_mem}.compute({compute_args}) " + f": {ta_mem}.recompute({compute_args}))" + ) # math.* calls if namespace == "math": @@ -1151,14 +1177,15 @@ def _visit_arg_for_series(arg_node, arg_idx): cpp_t = self._infer_type(arg_node) if cpp_t not in ("double", "int", "bool"): cpp_t = "double" + member = self._inline_history_member( + "series_arg", node, arg_idx=arg_idx + ) return ( f"([&]() -> const Series<{cpp_t}>& {{ " - f"static thread_local Series<{cpp_t}> _series_arg; " - f"if (is_first_tick_ && bar_index_ == 0) _series_arg.clear(); " f"{cpp_t} _sv = ({expr_cpp}); " - f"if (is_first_tick_) _series_arg.push(_sv); " - f"else _series_arg.update(_sv); " - f"return _series_arg; }}())" + f"if (history_advances_new_bar()) {member}.push(_sv); " + f"else {member}.update(_sv); " + f"return {member}; }}())" ) return self._visit_expr(arg_node) diff --git a/pineforge_codegen/codegen/visit_expr.py b/pineforge_codegen/codegen/visit_expr.py index 5c2e75f..abe3c44 100644 --- a/pineforge_codegen/codegen/visit_expr.py +++ b/pineforge_codegen/codegen/visit_expr.py @@ -509,8 +509,18 @@ def _visit_member_access(self, node: MemberAccess) -> str: ta_short = site.class_name.split("::")[-1].lower() if site.member_name.startswith(f"_ta_{node.member}_"): if node.member == "vwap": - return f"(is_first_tick_ ? {site.member_name}.compute(current_bar_.close, current_bar_.volume, current_bar_.timestamp) : {site.member_name}.recompute(current_bar_.close, current_bar_.volume, current_bar_.timestamp))" - return f"(is_first_tick_ ? {site.member_name}.compute({TA_IMPLICIT_COMPUTE_FULL[node.member]}) : {site.member_name}.recompute({TA_IMPLICIT_COMPUTE_FULL[node.member]}))" + return ( + f"(history_advances_new_bar() ? {site.member_name}.compute(" + "current_bar_.close, current_bar_.volume, current_bar_.timestamp) " + f": {site.member_name}.recompute(current_bar_.close, " + "current_bar_.volume, current_bar_.timestamp))" + ) + return ( + f"(history_advances_new_bar() ? {site.member_name}.compute(" + f"{TA_IMPLICIT_COMPUTE_FULL[node.member]}) : " + f"{site.member_name}.recompute(" + f"{TA_IMPLICIT_COMPUTE_FULL[node.member]}))" + ) # No registered call site for this TA property read — # the old fallback emitted std::string(""), a # silent type mismatch. Reject loudly instead. @@ -822,26 +832,26 @@ def _visit_subscript(self, node: Subscript) -> str: # ``ta.highest(high, 10)[1]`` or ``f()[2]``. In Pine the call yields a # series, so ``[k]`` reads its value k bars ago — but the call lowers to # a freshly-computed C++ scalar, and ``scalar[k]`` is not subscriptable. - # Materialize the result into a self-contained history buffer: a static + # Materialize the result into a checkpoint-owned class-member # ``Series`` that pushes (new bar) / updates (intrabar) the value # exactly once per evaluation — same semantics as every other series in # the strategy — and read ``[k]`` off it. The inner call is emitted once - # so its own stateful indicator is not double-stepped, and the buffer - # clears itself on run-start (``is_first_tick_ && bar_index_ == 0``) so a - # reused strategy handle (parameter sweep) does not leak prior-run history. + # so its own stateful indicator is not double-stepped. The deterministic + # member identity includes the emitted UDF variant; separate Pine call + # sites never share history, and on_bar clears every synthetic member at + # run-start even when this particular expression is conditional. if isinstance(node.object, FuncCall): inner = self._visit_expr(node.object) cpp_t = self._infer_type(node.object) if cpp_t not in ("double", "int", "bool"): cpp_t = "double" + member = self._inline_history_member("hist_call", node) return ( f"([&]() -> {cpp_t} {{ " - f"static thread_local Series<{cpp_t}> _hist_call; " - f"if (is_first_tick_ && bar_index_ == 0) _hist_call.clear(); " f"{cpp_t} _hv = ({inner}); " - f"if (is_first_tick_) _hist_call.push(_hv); " - f"else _hist_call.update(_hv); " - f"return _hist_call[(int)({idx})]; }}())" + f"if (history_advances_new_bar()) {member}.push(_hv); " + f"else {member}.update(_hv); " + f"return {member}[(int)({idx})]; }}())" ) obj = self._visit_expr(node.object) # If subscripting a non-series variable (e.g., function parameter), diff --git a/pineforge_codegen/codegen/visit_stmt.py b/pineforge_codegen/codegen/visit_stmt.py index 3769bf9..fb4c84f 100644 --- a/pineforge_codegen/codegen/visit_stmt.py +++ b/pineforge_codegen/codegen/visit_stmt.py @@ -288,7 +288,7 @@ def remember_local_type(cpp_type: str | None) -> None: title = self._get_input_title(node.value, var_name=node.name) cpp_val = self._render_input_value(node.value, func_name_i, namespace_i, title) if node.name in self.ctx.series_vars: - lines.append(f"{pad}{safe}.push({cpp_val});") + self._emit_history_series_write(lines, pad, safe, cpp_val) elif is_global_member: lines.append(f"{pad}{safe} = {cpp_val};") else: @@ -385,9 +385,12 @@ def remember_local_type(cpp_type: str | None) -> None: compute_args = self._ta_compute_args_for_site(site) ret_type = "bool" if self._ta_name_from_site(site) in TA_RETURNS_BOOL else "double" ta_name = self._ta_member_name(site) - ta_expr = f"(is_first_tick_ ? {ta_name}.compute({compute_args}) : {ta_name}.recompute({compute_args}))" + ta_expr = ( + f"(history_advances_new_bar() ? {ta_name}.compute({compute_args}) " + f": {ta_name}.recompute({compute_args}))" + ) if node.name in self.ctx.series_vars: - lines.append(f"{pad}{safe}.push({ta_expr});") + self._emit_history_series_write(lines, pad, safe, ta_expr) elif is_global_member: lines.append(f"{pad}{safe} = {ta_expr};") else: @@ -397,7 +400,7 @@ def remember_local_type(cpp_type: str | None) -> None: # Non-var series variable — push instead of declare if node.name in self.ctx.series_vars: cpp_val = self._visit_expr(node.value) - lines.append(f"{pad}{safe}.push({cpp_val});") + self._emit_history_series_write(lines, pad, safe, cpp_val) return # If/switch expression: x = if cond ... else ... @@ -612,7 +615,11 @@ def _visit_tuple_assign(self, node: TupleAssign, lines: list[str], pad: str) -> compute_args = self._ta_compute_args_for_site(site) ta_mem = self._ta_member_name(site) result_var = f"_result_{ta_mem}" - lines.append(f"{pad}auto {result_var} = (is_first_tick_ ? {ta_mem}.compute({compute_args}) : {ta_mem}.recompute({compute_args}));") + lines.append( + f"{pad}auto {result_var} = (history_advances_new_bar() ? " + f"{ta_mem}.compute({compute_args}) : " + f"{ta_mem}.recompute({compute_args}));" + ) ta_name = self._ta_name_from_site(site) fields = TA_TUPLE_FIELDS.get(ta_name, [f"field{i}" for i in range(len(node.names))]) @@ -632,7 +639,9 @@ def _visit_tuple_assign(self, node: TupleAssign, lines: list[str], pad: str) -> # destructured names keep the plain scalar declaration. if name in self.ctx.series_vars: safe = self._safe_name(name) - lines.append(f"{pad}{safe}.push({field_expr});") + self._emit_history_series_write( + lines, pad, safe, field_expr + ) else: lines.append(f"{pad}double {name} = {field_expr};") return diff --git a/tests/golden/matrix_eigen_pca.cpp b/tests/golden/matrix_eigen_pca.cpp index 9b2de1f..81cf6d7 100644 --- a/tests/golden/matrix_eigen_pca.cpp +++ b/tests/golden/matrix_eigen_pca.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include #include @@ -121,6 +123,97 @@ class GeneratedStrategy : public BacktestEngine { bool _ta_initialized_ = false; bool _inputs_initialized_ = false; + struct _PFScriptState { + decltype(GeneratedStrategy::_ta_sma_1) _pf_value_0; + decltype(GeneratedStrategy::_ta_sma_2) _pf_value_1; + decltype(GeneratedStrategy::_ta_sma_3) _pf_value_2; + decltype(GeneratedStrategy::_ta_sma_4) _pf_value_3; + decltype(GeneratedStrategy::_ta_sma_5) _pf_value_4; + decltype(GeneratedStrategy::_ta_sma_6) _pf_value_5; + decltype(GeneratedStrategy::_ta_crossover_7) _pf_value_6; + decltype(GeneratedStrategy::_ta_crossunder_8) _pf_value_7; + decltype(GeneratedStrategy::m) _pf_value_8; + decltype(GeneratedStrategy::length) _pf_value_9; + decltype(GeneratedStrategy::v1) _pf_value_10; + decltype(GeneratedStrategy::v2) _pf_value_11; + decltype(GeneratedStrategy::v1_mean) _pf_value_12; + decltype(GeneratedStrategy::v2_mean) _pf_value_13; + decltype(GeneratedStrategy::cov11) _pf_value_14; + decltype(GeneratedStrategy::cov12) _pf_value_15; + decltype(GeneratedStrategy::cov21) _pf_value_16; + decltype(GeneratedStrategy::cov22) _pf_value_17; + decltype(GeneratedStrategy::covReady) _pf_value_18; + decltype(GeneratedStrategy::lam) _pf_value_19; + decltype(GeneratedStrategy::lamSma) _pf_value_20; + decltype(GeneratedStrategy::_var_initialized) _pf_value_21; + decltype(GeneratedStrategy::_ta_initialized_) _pf_value_22; + decltype(GeneratedStrategy::_inputs_initialized_) _pf_value_23; + }; + static_assert(std::is_copy_constructible_v<_PFScriptState>, "generated Pine state must be deep-copy constructible"); + static_assert(std::is_copy_assignable_v<_PFScriptState>, "generated Pine state must be deep-copy assignable"); + std::optional<_PFScriptState> _pf_script_state_checkpoint_; + + void snapshot_script_state() override { + _pf_script_state_checkpoint_.emplace(_PFScriptState{ + _ta_sma_1, + _ta_sma_2, + _ta_sma_3, + _ta_sma_4, + _ta_sma_5, + _ta_sma_6, + _ta_crossover_7, + _ta_crossunder_8, + m, + length, + v1, + v2, + v1_mean, + v2_mean, + cov11, + cov12, + cov21, + cov22, + covReady, + lam, + lamSma, + _var_initialized, + _ta_initialized_, + _inputs_initialized_, + }); + } + + void restore_script_state() override { + if (!_pf_script_state_checkpoint_) return; + this->_ta_sma_1 = _pf_script_state_checkpoint_->_pf_value_0; + this->_ta_sma_2 = _pf_script_state_checkpoint_->_pf_value_1; + this->_ta_sma_3 = _pf_script_state_checkpoint_->_pf_value_2; + this->_ta_sma_4 = _pf_script_state_checkpoint_->_pf_value_3; + this->_ta_sma_5 = _pf_script_state_checkpoint_->_pf_value_4; + this->_ta_sma_6 = _pf_script_state_checkpoint_->_pf_value_5; + this->_ta_crossover_7 = _pf_script_state_checkpoint_->_pf_value_6; + this->_ta_crossunder_8 = _pf_script_state_checkpoint_->_pf_value_7; + this->m = _pf_script_state_checkpoint_->_pf_value_8; + this->length = _pf_script_state_checkpoint_->_pf_value_9; + this->v1 = _pf_script_state_checkpoint_->_pf_value_10; + this->v2 = _pf_script_state_checkpoint_->_pf_value_11; + this->v1_mean = _pf_script_state_checkpoint_->_pf_value_12; + this->v2_mean = _pf_script_state_checkpoint_->_pf_value_13; + this->cov11 = _pf_script_state_checkpoint_->_pf_value_14; + this->cov12 = _pf_script_state_checkpoint_->_pf_value_15; + this->cov21 = _pf_script_state_checkpoint_->_pf_value_16; + this->cov22 = _pf_script_state_checkpoint_->_pf_value_17; + this->covReady = _pf_script_state_checkpoint_->_pf_value_18; + this->lam = _pf_script_state_checkpoint_->_pf_value_19; + this->lamSma = _pf_script_state_checkpoint_->_pf_value_20; + this->_var_initialized = _pf_script_state_checkpoint_->_pf_value_21; + this->_ta_initialized_ = _pf_script_state_checkpoint_->_pf_value_22; + this->_inputs_initialized_ = _pf_script_state_checkpoint_->_pf_value_23; + } + + void commit_script_state() override { + snapshot_script_state(); + } + explicit GeneratedStrategy() : _ta_sma_1(14), _ta_sma_2(14), _ta_sma_3(14), _ta_sma_4(14), _ta_sma_5(14), _ta_sma_6(14) { initial_capital_ = 1000000.0; default_qty_type_ = QtyType::FIXED; @@ -138,6 +231,7 @@ class GeneratedStrategy : public BacktestEngine { if (key == "pyramiding") { pyramiding_ = std::stoi(value); return; } if (key == "slippage") { slippage_ = std::stoi(value); return; } if (key == "process_orders_on_close") { process_orders_on_close_ = (value == "true" || value == "1"); return; } + if (key == "calc_on_order_fills") { calc_on_order_fills_ = (value == "true" || value == "1"); return; } if (key == "close_entries_rule") { close_entries_rule_any_ = (value == "ANY" || value == "any" || value == "1"); return; } if (key == "default_qty_type") { if (value == "fixed" || value == "strategy.fixed" || value == "0") default_qty_type_ = QtyType::FIXED; @@ -174,12 +268,12 @@ class GeneratedStrategy : public BacktestEngine { } v1 = (current_bar_.close - current_bar_.open); v2 = (current_bar_.high - current_bar_.low); - v1_mean = (is_first_tick_ ? _ta_sma_1.compute(v1) : _ta_sma_1.recompute(v1)); - v2_mean = (is_first_tick_ ? _ta_sma_2.compute(v2) : _ta_sma_2.recompute(v2)); - cov11 = (is_first_tick_ ? _ta_sma_3.compute(((v1 - v1_mean) * (v1 - v1_mean))) : _ta_sma_3.recompute(((v1 - v1_mean) * (v1 - v1_mean)))); - cov12 = (is_first_tick_ ? _ta_sma_4.compute(((v1 - v1_mean) * (v2 - v2_mean))) : _ta_sma_4.recompute(((v1 - v1_mean) * (v2 - v2_mean)))); + v1_mean = (history_advances_new_bar() ? _ta_sma_1.compute(v1) : _ta_sma_1.recompute(v1)); + v2_mean = (history_advances_new_bar() ? _ta_sma_2.compute(v2) : _ta_sma_2.recompute(v2)); + cov11 = (history_advances_new_bar() ? _ta_sma_3.compute(((v1 - v1_mean) * (v1 - v1_mean))) : _ta_sma_3.recompute(((v1 - v1_mean) * (v1 - v1_mean)))); + cov12 = (history_advances_new_bar() ? _ta_sma_4.compute(((v1 - v1_mean) * (v2 - v2_mean))) : _ta_sma_4.recompute(((v1 - v1_mean) * (v2 - v2_mean)))); cov21 = cov12; - cov22 = (is_first_tick_ ? _ta_sma_5.compute(((v2 - v2_mean) * (v2 - v2_mean))) : _ta_sma_5.recompute(((v2 - v2_mean) * (v2 - v2_mean)))); + cov22 = (history_advances_new_bar() ? _ta_sma_5.compute(((v2 - v2_mean) * (v2 - v2_mean))) : _ta_sma_5.recompute(((v2 - v2_mean) * (v2 - v2_mean)))); m.set((int)(0), (int)(0), cov11); m.set((int)(0), (int)(1), cov12); m.set((int)(1), (int)(0), cov21); @@ -189,11 +283,11 @@ class GeneratedStrategy : public BacktestEngine { if (covReady) { lam = ((((double)m.eigenvalues().size() > 0)) ? (m.eigenvalues()[(0)]) : (na())); } - lamSma = (is_first_tick_ ? _ta_sma_6.compute(lam) : _ta_sma_6.recompute(lam)); - if ((((covReady && !(is_na(lam))) && !(is_na(lamSma))) && (is_first_tick_ ? _ta_crossover_7.compute(lam, lamSma) : _ta_crossover_7.recompute(lam, lamSma)))) { + lamSma = (history_advances_new_bar() ? _ta_sma_6.compute(lam) : _ta_sma_6.recompute(lam)); + if ((((covReady && !(is_na(lam))) && !(is_na(lamSma))) && (history_advances_new_bar() ? _ta_crossover_7.compute(lam, lamSma) : _ta_crossover_7.recompute(lam, lamSma)))) { strategy_entry(std::string("Long"), true, na(), na(), na(), ""); } - if ((((covReady && !(is_na(lam))) && !(is_na(lamSma))) && (is_first_tick_ ? _ta_crossunder_8.compute(lam, lamSma) : _ta_crossunder_8.recompute(lam, lamSma)))) { + if ((((covReady && !(is_na(lam))) && !(is_na(lamSma))) && (history_advances_new_bar() ? _ta_crossunder_8.compute(lam, lamSma) : _ta_crossunder_8.recompute(lam, lamSma)))) { strategy_entry(std::string("Short"), false, na(), na(), na(), ""); } } diff --git a/tests/test_calc_on_order_fills_codegen.py b/tests/test_calc_on_order_fills_codegen.py new file mode 100644 index 0000000..e44c33a --- /dev/null +++ b/tests/test_calc_on_order_fills_codegen.py @@ -0,0 +1,435 @@ +"""Codegen contract for strategy ``calc_on_order_fills`` rollback support.""" + +from __future__ import annotations + +import re + +from pineforge_codegen import transpile + + +def _strategy(header: str, body: str = "") -> str: + return f'//@version=6\nstrategy("T"{header})\n{body}' + + +def test_calc_on_order_fills_declaration_and_runtime_override_plumbing(): + cpp = transpile(_strategy(", calc_on_order_fills=true")) + + assert "calc_on_order_fills_ = true;" in cpp + assert ( + 'if (key == "calc_on_order_fills") { calc_on_order_fills_ = ' + '(value == "true" || value == "1"); return; }' + ) in cpp + + +def test_calc_on_order_fills_false_and_calc_on_every_tick_are_independent(): + explicit_false = transpile(_strategy(", calc_on_order_fills=false")) + every_tick_only = transpile(_strategy(", calc_on_every_tick=true")) + + # The override setter is emitted for every strategy. Limit this check to + # the constructor body so that it cannot make either assertion pass. + false_ctor = explicit_false.split("explicit GeneratedStrategy()", 1)[1].split( + "void set_strategy_override", 1 + )[0] + every_tick_ctor = every_tick_only.split("explicit GeneratedStrategy()", 1)[ + 1 + ].split("void set_strategy_override", 1)[0] + assert "calc_on_order_fills_ = true;" not in false_ctor + assert "calc_on_order_fills_ = true;" not in every_tick_ctor + + +_ROLLBACK_PROBE = '''//@version=6 +strategy("COOF state", calc_on_order_fills=true) +type Pt + float x + float y +bump(float v) => + var float total = 0.0 + total += v + fixnan(total) +var float scalar = 0.0 +var float historical = 0.0 +var array xs = array.new() +var mp = map.new() +var mx = matrix.new(2, 2, 0.0) +var Pt point = Pt.new(1.0, 2.0) +var line ln = na +scalar += close +historical += close +previous = historical[1] +array.push(xs, scalar) +map.put(mp, "x", scalar) +matrix.set(mx, 0, 0, scalar) +point.x := scalar +ln := line.new(bar_index, close, bar_index + 1, close) +e = ta.ema(close, 3) +f = fixnan(close) +a = bump(close) +b = bump(open) +if e > 0 + strategy.entry("L", strategy.long) +''' + + +def test_script_state_checkpoint_covers_every_mutable_state_family(): + """Mutation guard: dropping any state family must break this inventory.""" + cpp = transpile(_ROLLBACK_PROBE) + fields = { + name: int(index) + for name, index in re.findall( + r"decltype\(GeneratedStrategy::(\w+)\) _pf_value_(\d+);", cpp + ) + } + + # Scalar + Series, TA/call-site helpers, all collection kinds, a UDT, + # drawing handle/arenas, function-local cloned state, and init latches. + expected = { + "scalar", + "historical", + "_ta_ema_1", + "_prev_fixnan_1", + "xs", + "mp", + "mx", + "point", + "ln", + "total", + "total_cs1", + "_prev_fixnan_2", + "_prev_fixnan_1_cs1", + "_pf_lines_", + "_pf_boxes_", + "_pf_labels_", + "_pf_linefills_", + "_var_initialized", + "_fvinit_bump_cs0", + "_fvinit_bump_cs1", + "_ta_initialized_", + "_inputs_initialized_", + } + assert expected <= fields.keys() + + # Every owned checkpoint field must participate in both directions. This + # catches mutations which leave the struct populated but omit snapshot or + # restore plumbing for one member. + for name, index in fields.items(): + assert re.search(rf"^\s+{re.escape(name)},$", cpp, re.MULTILINE) + assert ( + f"this->{name} = _pf_script_state_checkpoint_->_pf_value_{index};" in cpp + ) + + # Full-dataset precalc output is immutable during a broker walk. It must + # not be copied per bar; the live TA helper above remains checkpointed for + # dynamic and magnifier execution. + assert "GeneratedStrategy::_precalc_" not in cpp + assert "GeneratedStrategy::_use_precalc" not in cpp + assert "std::is_copy_constructible_v<_PFScriptState>" in cpp + assert "std::is_copy_assignable_v<_PFScriptState>" in cpp + + +def test_commit_replaces_checkpoint_with_current_live_state(): + cpp = transpile(_strategy("")) + commit = cpp.split("void commit_script_state() override", 1)[1].split( + "explicit GeneratedStrategy()", 1 + )[0] + + assert "snapshot_script_state();" in commit + assert "reset(" not in commit + + +_HISTORY_ADVANCE_PROBE = '''//@version=6 +strategy("COOF history", calc_on_order_fills=true, process_orders_on_close=true) +source_input = input.source(close, "Source") +source_prev = source_input[1] +var float carried = 0.0 +carried += close +carried_prev = carried[1] +rolling = close + open +rolling_prev = rolling[1] +chart_prev = close[1] +index_prev = bar_index[1] +position_prev = strategy.position_size[1] +ema = ta.ema(close, 2) +ema_prev = ema[1] +[basis, upper, lower] = ta.bb(close, 2, 2.0) +upper_prev = upper[1] +inline_prev = ta.highest(high, 2)[1] +history_arg(float src) => + src[1] +series_prev = history_arg(close + open) +if barstate.isnew + strategy.entry("L", strategy.long) +''' + + +def test_post_fill_recalc_updates_current_history_slot_but_barstate_stays_new(): + """Post-C fill recalcs must not turn Pine's current bar into ``[1]``. + + The engine deliberately keeps historical ``barstate.isnew`` true during a + fill recalculation. History advancement therefore has a separate runtime + predicate: every rolling member updates its existing current-bar slot when + the ordinary-close checkpoint is restored, while ``barstate.isnew`` keeps + lowering to ``is_first_tick_``. + """ + cpp = transpile(_HISTORY_ADVANCE_PROBE) + on_bar = cpp.split("void on_bar(const Bar& bar) override {", 1)[1].split( + "\n }", 1 + )[0] + + expected_writes = { + "_s_close": "current_bar_.close", + "bar_index": "pine_bar_index()", + "_strat_position_size": "signed_position_size()", + "rolling": "(current_bar_.close + current_bar_.open)", + } + for member, value in expected_writes.items(): + assert ( + f"if (history_advances_new_bar()) {member}.push({value});" in on_bar + ) + assert f"else {member}.update({value});" in on_bar + + # A persistent series carries its current value into a genuinely new bar, + # but updates that same slot during post-fill execution. + assert ( + "if (history_advances_new_bar()) carried.push(carried[0]);" in on_bar + ) + + # All VarDecl series families use the shared push/update lowering: an + # input.source history, a TA result, and a history-referenced tuple field. + for member in ("source_input", "ema", "upper"): + assert f"if (history_advances_new_bar()) {member}.push(" in on_bar + assert f"else {member}.update(" in on_bar + + # TA state and temporary history buffers obey the same history predicate. + # Temporary buffers are named class members: fill recalculation rollback + # cannot restore a function-local static, and two source sites must never + # share one history stream. + assert "history_advances_new_bar() ? _ta_ema_1.compute(" in on_bar + assert re.search( + r"history_advances_new_bar\(\) \? _ta_highest_\d+\.compute\(", on_bar + ) + assert "static thread_local Series" not in cpp + assert re.search( + r"if \(history_advances_new_bar\(\)\) _hist_call_\d+\.push\(_hv\);", + on_bar, + ) + assert re.search( + r"if \(history_advances_new_bar\(\)\) _series_arg_\d+\.push\(_sv\);", + on_bar, + ) + + # Mutation guard: coupling barstate.isnew to history advancement would make + # it false during historical fill recalcs, contrary to Pine semantics. + assert "if (is_first_tick_) {" in on_bar + assert "if (history_advances_new_bar()) {\n strategy_entry" not in on_bar + + +_INLINE_BUFFER_PROBE = '''//@version=6 +strategy("COOF inline members", calc_on_order_fills=true, process_orders_on_close=true) +history_arg(float src) => + src[1] +wrapped(float src) => + inline_prev = ta.highest(src, 2)[1] + bridged_prev = history_arg(src + 1.0) + inline_prev + bridged_prev +top_inline_a = ta.highest(high, 2)[1] +top_inline_b = ta.lowest(low, 2)[1] +top_bridge_a = history_arg(close + open) +top_bridge_b = history_arg(high - low) +wrapped_a = wrapped(close) +wrapped_b = wrapped(open) +if barstate.isnew + strategy.entry("L", strategy.long) +''' + + +def _checkpoint_fields(cpp: str) -> dict[str, int]: + return { + name: int(index) + for name, index in re.findall( + r"decltype\(GeneratedStrategy::(\w+)\) _pf_value_(\d+);", cpp + ) + } + + +def test_inline_history_buffers_are_owned_independent_and_clear_at_bar_zero(): + """Mutation guard for both synthesized temporary-Series families. + + This kills three tempting regressions: moving a buffer back into a local + static, collapsing separate source sites/UDF variants onto one stream, or + clearing only when a conditional site happens to execute on bar zero. + + The clear assertion is deliberately narrow. It preserves the legacy + synthetic-buffer run-start guard; it does not claim that every generated + member/init latch supports full same-handle reruns. + """ + cpp = transpile(_INLINE_BUFFER_PROBE) + fields = _checkpoint_fields(cpp) + hist_members = re.findall(r"^\s+Series (_hist_call_\d+);$", cpp, re.MULTILINE) + arg_members = re.findall(r"^\s+Series (_series_arg_\d+);$", cpp, re.MULTILINE) + + # Two top-level sites plus one site in each wrapped() call-site clone. + assert len(hist_members) == 4 + assert len(set(hist_members)) == 4 + assert len(arg_members) == 4 + assert len(set(arg_members)) == 4 + assert "static thread_local Series" not in cpp + + on_bar = cpp.split("void on_bar(const Bar& bar) override {", 1)[1].split( + "\n }", 1 + )[0] + for member in hist_members + arg_members: + index = fields[member] + assert f"if (history_advances_new_bar() && bar_index_ == 0) {member}.clear();" in on_bar + assert f"if (history_advances_new_bar()) {member}.push(" in cpp + assert f"else {member}.update(" in cpp + assert re.search(rf"^\s+{re.escape(member)},$", cpp, re.MULTILINE) + assert ( + f"this->{member} = _pf_script_state_checkpoint_->_pf_value_{index};" + in cpp + ) + + wrapped_cs0 = cpp.split("double wrapped_cs0(", 1)[1].split("\n }", 1)[0] + wrapped_cs1 = cpp.split("double wrapped_cs1(", 1)[1].split("\n }", 1)[0] + assert set(re.findall(r"_hist_call_\d+", wrapped_cs0)).isdisjoint( + re.findall(r"_hist_call_\d+", wrapped_cs1) + ) + assert set(re.findall(r"_series_arg_\d+", wrapped_cs0)).isdisjoint( + re.findall(r"_series_arg_\d+", wrapped_cs1) + ) + + +_NESTED_SYNTHETIC_ONLY_PROBE = '''//@version=6 +strategy("COOF nested synthetic", calc_on_order_fills=true) +history_arg(float src) => + src[1] +leaf(float src) => + history_arg(src + 1.0) +left(float src) => + leaf(src) +right(float src) => + leaf(src) +left_value = left(close) +right_value = right(open) +''' + + +def test_nested_synthetic_only_helpers_dispatch_to_distinct_leaf_instances(): + """Pure wrappers must preserve the synthetic leaf's call-path identity.""" + cpp = transpile(_NESTED_SYNTHETIC_ONLY_PROBE) + left = cpp.split("double left_cs0(", 1)[1].split("\n }", 1)[0] + right = cpp.split("double right_cs0(", 1)[1].split("\n }", 1)[0] + + assert "leaf_cs0(src)" in left + assert "leaf_cs1(src)" in right + assert "leaf_cs0(src)" not in right + + leaf0 = cpp.split("double leaf_cs0(", 1)[1].split("\n }", 1)[0] + leaf1 = cpp.split("double leaf_cs1(", 1)[1].split("\n }", 1)[0] + assert set(re.findall(r"_series_arg_\d+", leaf0)).isdisjoint( + re.findall(r"_series_arg_\d+", leaf1) + ) + + +_SWITCH_ARM_SYNTHETIC_PROBE = '''//@version=6 +strategy("COOF switch synthetic", calc_on_order_fills=true) +passthrough(float src) => + src +wrapped(float src, int mode) => + switch mode + 1 => passthrough(src)[1] + => src +first = wrapped(close, 1) +second = wrapped(open, 1) +''' + + +def test_switch_arm_tuple_history_marks_wrapper_stateful_and_isolates_calls(): + """SwitchStmt.cases stores arms as tuples; state discovery must enter them.""" + cpp = transpile(_SWITCH_ARM_SYNTHETIC_PROBE) + assert "double wrapped_cs0(" in cpp + assert "double wrapped_cs1(" in cpp + + members = re.findall(r"^\s+Series (_hist_call_\d+);$", cpp, re.MULTILINE) + assert len(members) == 2 + body0 = cpp.split("double wrapped_cs0(", 1)[1].split("\n }", 1)[0] + body1 = cpp.split("double wrapped_cs1(", 1)[1].split("\n }", 1)[0] + assert set(re.findall(r"_hist_call_\d+", body0)).isdisjoint( + re.findall(r"_hist_call_\d+", body1) + ) + + +_UDT_METHOD_SYNTHETIC_PROBE = '''//@version=6 +strategy("COOF UDT synthetic", calc_on_order_fills=true) +type Box + float bias +passthrough(float src) => + src +history_arg(float src) => + src[1] +method measure(Box self, float src) => + call_prev = passthrough(src + self.bias)[1] + arg_prev = history_arg(src - self.bias) + call_prev + arg_prev +var Box bx = Box.new(1.0) +first = bx.measure(close) +second = bx.measure(open) +''' + + +def test_udt_method_synthetic_history_isolated_per_source_call_site(): + """UDT methods need the same per-call-site state identity as plain UDFs.""" + cpp = transpile(_UDT_METHOD_SYNTHETIC_PROBE) + assert "double _udt_Box_measure_cs0(" in cpp + assert "double _udt_Box_measure_cs1(" in cpp + assert "first = _udt_Box_measure_cs0(" in cpp + assert "second = _udt_Box_measure_cs1(" in cpp + + hist_members = re.findall(r"^\s+Series (_hist_call_\d+);$", cpp, re.MULTILINE) + arg_members = re.findall(r"^\s+Series (_series_arg_\d+);$", cpp, re.MULTILINE) + assert len(hist_members) == 2 + assert len(arg_members) == 2 + fields = _checkpoint_fields(cpp) + assert set(hist_members + arg_members) <= fields.keys() + + body0 = cpp.split("double _udt_Box_measure_cs0(", 1)[1].split("\n }", 1)[0] + body1 = cpp.split("double _udt_Box_measure_cs1(", 1)[1].split("\n }", 1)[0] + assert set(re.findall(r"_(?:hist_call|series_arg)_\d+", body0)).isdisjoint( + re.findall(r"_(?:hist_call|series_arg)_\d+", body1) + ) + + +def test_every_strategy_history_member_pushes_or_updates(): + body = "\n".join( + f"v_{name} = strategy.{name}[1]" + for name in ( + "position_size", + "closedtrades", + "opentrades", + "wintrades", + "losstrades", + "equity", + "netprofit", + "openprofit", + "initial_capital", + ) + ) + cpp = transpile(_strategy(", calc_on_order_fills=true", body)) + on_bar = cpp.split("void on_bar(const Bar& bar) override {", 1)[1].split( + "\n }", 1 + )[0] + + for name in ( + "position_size", + "closedtrades", + "opentrades", + "wintrades", + "losstrades", + "equity", + "netprofit", + "openprofit", + "initial_capital", + ): + member = f"_strat_{name}" + assert f"if (history_advances_new_bar()) {member}.push(" in on_bar + assert f"else {member}.update(" in on_bar + assert not re.search(rf"^\s*{re.escape(member)}\.push\(", on_bar, re.MULTILINE) diff --git a/tests/test_codegen_determinism.py b/tests/test_codegen_determinism.py index 56bdd11..4ab1a99 100644 --- a/tests/test_codegen_determinism.py +++ b/tests/test_codegen_determinism.py @@ -95,7 +95,7 @@ _SEEDS = [0, 1, 2, 7, 12345] -def _transpile_under_seed(seed: int) -> str: +def _transpile_under_seed(seed: int, fixture: str = FIXTURE) -> str: """Transpile FIXTURE in a fresh interpreter pinned to ``PYTHONHASHSEED=seed``.""" env = { **os.environ, @@ -108,7 +108,7 @@ def _transpile_under_seed(seed: int) -> str: } proc = subprocess.run( [sys.executable, "-c", _CHILD], - input=FIXTURE, + input=fixture, env=env, capture_output=True, text=True, @@ -134,3 +134,38 @@ def test_transpile_byte_identical_across_hash_seeds() -> None: "(see fix(codegen): deterministic member-clone ordering). " "Likely a set is being iterated into emitted C++ again." ) + + +_SYNTHETIC_HISTORY_FIXTURE = """//@version=6 +strategy("Synthetic Determinism", calc_on_order_fills=true) +type Box + float bias +passthrough(float src) => src +history_arg(float src) => src[1] +leaf(float src) => history_arg(src + 1.0) +left(float src) => leaf(src) +right(float src) => leaf(src) +wrapped(float src, int mode) => + switch mode + 1 => passthrough(src)[1] + => src +method measure(Box self, float src) => + call_prev = passthrough(src + self.bias)[1] + arg_prev = history_arg(src - self.bias) + call_prev + arg_prev +var Box bx = Box.new(1.0) +a = left(close) +b = right(open) +c = wrapped(close, 1) +d = wrapped(open, 1) +e = bx.measure(close) +f = bx.measure(open) +""" + + +def test_synthetic_history_names_byte_identical_across_hash_seeds() -> None: + outputs = [ + _transpile_under_seed(seed, _SYNTHETIC_HISTORY_FIXTURE) + for seed in _SEEDS + ] + assert len(set(outputs)) == 1 diff --git a/tests/test_codegen_new.py b/tests/test_codegen_new.py index a2c5bf2..0c8b366 100644 --- a/tests/test_codegen_new.py +++ b/tests/test_codegen_new.py @@ -1375,7 +1375,7 @@ def test_magnifier_series_guard(): strategy("T") x = ta.sma(close, 14) """) - assert "is_first_tick_" in cpp + assert "history_advances_new_bar()" in cpp assert ".recompute(" in cpp diff --git a/tests/test_codegen_validation_fixes.py b/tests/test_codegen_validation_fixes.py index 7ab059d..4bd7423 100644 --- a/tests/test_codegen_validation_fixes.py +++ b/tests/test_codegen_validation_fixes.py @@ -174,7 +174,9 @@ def test_bar_index_history_series_is_pushed_from_offset_helper(): "plot(close)" ) assert "Series bar_index" in cpp - assert "if (is_first_tick_) bar_index.push(pine_bar_index());" in cpp + assert ( + "if (history_advances_new_bar()) bar_index.push(pine_bar_index());" in cpp + ) assert "else bar_index.update(pine_bar_index());" in cpp assert "(pine_bar_index() - past)" in cpp diff --git a/tests/test_compile_smoke.py b/tests/test_compile_smoke.py index 6ff6c79..2ce4f63 100644 --- a/tests/test_compile_smoke.py +++ b/tests/test_compile_smoke.py @@ -67,6 +67,81 @@ def test_minimal_strategy_compiles(): compile_cpp(cpp, label="minimal_strategy") +def test_calc_on_order_fills_mixed_script_state_checkpoint_compiles(): + """The rollback aggregate must remain copyable across every state family.""" + skip_if_no_compile_env() + cpp = transpile('''//@version=6 +strategy("COOF compile", calc_on_order_fills=true) +type Pt + float x + float y +bump(float v) => + var float total = 0.0 + total += v + fixnan(total) +var float historical = 0.0 +var array xs = array.new() +var mp = map.new() +var mx = matrix.new(2, 2, 0.0) +var Pt point = Pt.new(1.0, 2.0) +var line ln = na +historical += close +previous = historical[1] +previous_position = strategy.position_size[1] +array.push(xs, historical) +map.put(mp, "x", historical) +matrix.set(mx, 0, 0, historical) +point.x := historical +ln := line.new(bar_index, close, bar_index + 1, close) +e = ta.ema(close, 3) +a = bump(close) +b = bump(open) +history_arg(float src) => + src[1] +inline_previous = ta.highest(high, 3)[1] +bridged_previous = history_arg(close + open) +if barstate.isnew and e > 0 + strategy.entry("L", strategy.long) +''') + compile_cpp(cpp, label="calc_on_order_fills_mixed_state") + + +def test_calc_on_order_fills_synthetic_history_isolation_compiles(): + """Nested, switch-arm, and UDT-method buffer clones must all parse.""" + skip_if_no_compile_env() + cpp = transpile('''//@version=6 +strategy("COOF synthetic compile", calc_on_order_fills=true) +type Box + float bias +passthrough(float src) => + src +history_arg(float src) => + src[1] +leaf(float src) => + history_arg(src + 1.0) +left(float src) => + leaf(src) +right(float src) => + leaf(src) +switch_wrapped(float src, int mode) => + switch mode + 1 => passthrough(src)[1] + => src +method measure(Box self, float src) => + call_prev = passthrough(src + self.bias)[1] + arg_prev = history_arg(src - self.bias) + call_prev + arg_prev +var Box bx = Box.new(1.0) +nested_a = left(close) +nested_b = right(open) +switch_a = switch_wrapped(close, 1) +switch_b = switch_wrapped(open, 1) +method_a = bx.measure(close) +method_b = bx.measure(open) +''') + compile_cpp(cpp, label="calc_on_order_fills_synthetic_isolation") + + # --------------------------------------------------------------------------- # TA — covers single-value, tuple-returning, implicit-OHLC, and ta.* vars. # ---------------------------------------------------------------------------