From 447df6e0127f7584123db7ac99714a2594fa9556 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sat, 11 Jul 2026 03:41:21 +0800 Subject: [PATCH 1/5] Document realtime execution contract plan --- docs/design/realtime-execution-audit.md | 586 ++++++++++++++++++++++++ 1 file changed, 586 insertions(+) create mode 100644 docs/design/realtime-execution-audit.md diff --git a/docs/design/realtime-execution-audit.md b/docs/design/realtime-execution-audit.md new file mode 100644 index 0000000..f2d4361 --- /dev/null +++ b/docs/design/realtime-execution-audit.md @@ -0,0 +1,586 @@ +# Realtime execution contract: audit and development plan + +Status: design audit; implementation is not approved by this document alone + +Date: 2026-07-11 + +Worktree branch: `codex/realtime-execution-contract` + +## Decision summary + +PineForge should implement a deterministic historical-to-live simulator with a +documented execution contract. It should not claim to reproduce TradingView's +private realtime feed, broker emulator, or exchange execution. + +The first supported realtime calculation profile should be Pine's default +close-only strategy cadence: + +- historical confirmed OHLCV warms the same strategy and broker instance; +- positions, open entry lots, equity, pending orders, series, TA state, + `request.security()` state, and partial timeframe aggregation survive the + handoff; +- ordered executed-trade events update the forming bar and immediately evaluate + the broker's resting orders; +- strategy code executes only when a script-timeframe bar becomes confirmed; +- `calc_on_every_tick` rollback and realtime `calc_on_order_fills` re-execution + are separate calculation profiles and must not be silently approximated. + +There is no single source-of-truth ordering for every question: + +| Question | Authority | +|---|---| +| Event time, order, price, and quantity | Normalized input tape plus its versioned normalization manifest | +| Pine commands, IDs, pyramiding, exit reservation, and calculation semantics | TradingView documentation, followed by focused TradingView probes where documentation is ambiguous | +| Undocumented realtime simulation choices | PineForge's published contract and executable invariants | +| Historical confirmed-bar regression | Pinned TradingView exports | +| Alternative implementation ideas | Other engines as design evidence, never as an oracle | + +## Audit sources + +### TradingView + +Primary references: + +- [Strategies](https://www.tradingview.com/pine-script-docs/concepts/strategies/) +- [Execution model](https://www.tradingview.com/pine-script-docs/language/execution-model/) +- [Alerts FAQ](https://www.tradingview.com/pine-script-docs/faq/alerts/) +- [Webhook configuration](https://www.tradingview.com/support/solutions/43000529348-how-to-configure-webhook-alerts/) + +Relevant documented behavior: + +- An order is an instruction; a trade is a transaction produced by an order + fill. A net position can contain multiple open trades. +- By default, a strategy calculates at a bar's closing tick and newly created + market orders fill on the next available price tick. +- `pyramiding` limits same-direction open trades created by `strategy.entry()`; + it does not make an entry ID unique and does not constrain + `strategy.order()`. +- The same entry ID can identify multiple open trades. With `pyramiding = 2`, + two `strategy.entry("buy", ...)` fills can coexist, and one + `strategy.exit(..., from_entry = "buy")` call can create exit orders for + both entry trades. +- Price-based entries created on the same tick can exceed the declared + pyramiding limit when several eligible orders trigger. +- `calc_on_every_tick = true` can create and fill the same `"Buy"` entry ID + more than once on one realtime bar. +- `calc_on_order_fills = true` can re-run the script after each fill. The + official `strategy.entry("Buy", ...)` example can produce four same-ID + entries on one historical bar. +- A stop-limit order has durable activation: after the stop triggers, its limit + leg remains active until it fills, is replaced, or is cancelled. +- `process_orders_on_close` adds a closing-tick processing opportunity. It is + not a claim that a real venue will fill an order after the session closes. + +### PineTS + +Audited revision: +[`fdb7650`](https://github.com/QuantForgeOrg/PineTS/tree/fdb7650fde7b5cfff1851d03d1602cc826237c9e) +(reported as v0.9.28 by the checkout). + +Relevant design observations: + +- PineTS now contains a strategy simulator; PineForge's existing benchmark + prose that calls it indicator-only is stale relative to this revision. +- Its strategy state separates pending orders, open trades, closed trades, and + the net-position scalars. +- [`strategy.entry()`](https://github.com/QuantForgeOrg/PineTS/blob/fdb7650fde7b5cfff1851d03d1602cc826237c9e/src/namespaces/strategy/methods/entry.ts) + appends order objects to `pending_orders`. Its tests explicitly queue and + fill three entries on the same bar when pyramiding permits them. +- Its current fill engine evaluates pending orders from bar OHLC. The live + stream tests cover indicator/runtime streaming, but the audited strategy + tests do not establish a tick-by-tick live broker contract. +- Therefore PineTS is useful evidence for separating order, trade, and + position identity, but not a realtime fill oracle for PineForge. + +### PyneCore + +Audited revision: +[`ffeab9e`](https://github.com/PyneSys/pynecore/tree/ffeab9e5dfe6f063ed2728626df290dae8a0c5e6). + +Relevant design observations: + +- PyneCore's simulator keeps one pending entry/normal order per order ID and + replaces the still-unfilled object on reissue. Exit intents are keyed by + `(exit_id, from_entry)`, allowing one exit ID to fan out across entry scopes. +- Its open-trade list is distinct from the pending-order maps and net position. +- Its [live-mode contract](https://pynecore.org/docs/advanced/live-mode/) + deliberately suppresses strategy commands during historical warmup and + starts paper trading in the live phase. That is not PineForge's required + model because PineForge must preserve warmup positions, equity, and pending + orders. +- PyneCore separates its simulated `SimPosition` path from exchange-backed + broker plugins. That separation supports keeping PineForge's deterministic + simulator distinct from future real order routing. + +## The named-order hypothesis + +The proposed rule—"each named position triggers at most once for entry and +exit per bar, even with pyramiding"—is a reasonable operational safety instinct, +but the unit and scope need correction. + +An ID does not name a position in Pine: + +| Concept | Identity | Cardinality | +|---|---|---| +| Net position | symbol + strategy instance | One signed aggregate | +| Open trade / entry lot | immutable `entry_lot_id`, plus public entry ID | Many per net position; several can share a public ID | +| Command revision | immutable `command_revision_id` for one placement/reissue | One creation snapshot; may generate several executable legs | +| Executable order leg | immutable `order_leg_id`, parent command revision, entry-lot scope, and leg kind | Market, limit, stop, or trail state machine; one command can own many | +| Fill event | immutable `fill_id`, order leg, and market-event provenance | One terminal fill per executable leg | +| Closed-trade row | FIFO/ANY allocation of a fill to entry lots | One fill can create several rows | + +A global `(ID, bar) -> at most one fill` rule would conflict with documented +Pine behavior for pyramiding, `calc_on_every_tick`, and +`calc_on_order_fills`. It would also conflate a command with its executable +legs, one broker fill with the several trade rows that fill may close, and a +public display ID with an immutable broker identity. + +The recommended invariant is: + +> Every executable order leg has an immutable identity and creation snapshot, +> mutable broker lifecycle state, and at most one terminal transition. A +> command reissue replaces its still-pending command revision and executable +> legs according to the documented replacement rule. After terminal +> completion, the same command key may create a new revision with the same +> public ID. + +Close-only scheduling does not create a once-per-ID-per-bar guarantee. An old +resting order can fill on an intrabar trade; the confirmed-close calculation +can then issue a new command revision with the same ID, and +`process_orders_on_close` or `immediately = true` can fill a new executable leg +on that bar's observed closing event. Any operational duplicate guard for +webhook consumers must therefore be downstream idempotency keyed by `fill_id`, +not Pine broker semantics keyed by public ID and bar. + +One `strategy.exit()` command revision can generate TP, SL, and trailing legs +for multiple matching entry lots. Entry-relative prices can differ by lot, so +those child legs require independent activation, watermark, reservation, OCA, +and terminal state even when they share one exit ID. + +## PineForge current-state audit + +Audited branch base: `main` at `7a8fc3b`. + +| Requirement | Current evidence | Assessment | +|---|---|---| +| Same-instance historical warmup and handoff | `stream_begin()` calls `run()` once, then preserves the instance; `test_streaming` verifies position, pending order, equity/trade continuity | Implemented for the tested path | +| Ordered provider-neutral trade input | `TradeTick` validates price, quantity, nondecreasing time, and strictly increasing nonzero sequence | Implemented; zero sequence delegates tie ordering to the caller | +| Forming OHLCV without lookahead | Events update O/H/L/C/V before confirmation; exact-boundary ticks are assigned after the prior bar is finalized | Implemented for fixed input timeframes; broader cases need tests | +| Higher-timeframe partial aggregation survives handoff | Existing streaming unit test spans historical and realtime input bars inside one 5-minute candle | Implemented for the tested case | +| Default close-only script scheduling | Broker evaluates every trade; `on_bar()` runs only when a script bar confirms | Implemented | +| Same-ID pending replacement | `strategy_entry()` and `strategy_order()` erase pending objects with the same raw ID; exits replace by `(id, from_entry)`; all preserve the old `created_seq` | Partial: command, priority, and executable-leg identity are conflated, and raw-ID erasure crosses command categories without a targeted parity probe proving the intended collision rule | +| One terminal transition per executable leg | Filled pending objects are erased from `pending_orders_` | Implicit only; generated legs, terminal state, and reason are not retained or observable | +| Market and stop fills on observed trade events | Streaming tests cover next-event market fill and stop gap-through at the observed price/time | Implemented for tested cases | +| Limit fill in streaming | Uses the generic point-bar evaluator | Not directly tested | +| Stop-limit activation across events | `PendingOrder::stop_limit_activated` exists, but normal `process_pending_orders()` does not persist the activation returned by `resolve_entry_stop_limit_fill()`; persistence is currently gated to the historical COOF scheduler | Defect | +| Trailing stop across events | Tick points update one global `trail_best_price_` for the position | Architectural defect for concurrent trailing legs with different lots, creation times, activation levels, or offsets | +| OCA cancel/reduce on one event | Generic fill loop contains OCA behavior and excludes siblings by public ID | Historical tests exist; public-ID exclusion cannot distinguish same-ID executable siblings or revisions | +| `process_orders_on_close` | A close-price point pass prevents newly created orders from inspecting the elapsed wick | Partial: it reuses the bar-open timestamp and has no closing-event sequence, so fill provenance is wrong; it can also fill a no-trade synthetic bar | +| Quiet intervals | Clock advancement creates zero-volume carry-forward bars | Contract issue: both the pre-script broker pass and the post-script POC pass can create synthetic fills, excursions, trailing changes, or margin actions without an observed trade | +| Out-of-session intervals | Empty closed-session intervals are skipped; an actual input event remains authoritative | Partial; session-edge and first-event-after-reopen tests are missing | +| Event-level diagnostics | No command-revision, executable-leg, fill, or decision log in the public report | Missing | +| Replay/live equivalence | `push_ticks()` is a loop over `push_tick()` and a basic ordering test exists | Structural implementation is sound; result equivalence needs an explicit test | +| Handoff state equivalence | Tests inspect selected visible fields after warmup | Missing a canonical digest covering all broker, Pine, TA, security, aggregation, risk, session, and sequence state | +| Three full corpus handoffs ending 2025-05-01 | `run_stream_corpus_mmap.py` supports exactly three starts and one contiguous mmap tape | Runner exists, but exit status currently fails only on runtime errors, not parity/invariant failure; current build artifacts prove only short 10-minute sessions | +| Historical TradingView regression | Required `ctest` and `scripts/run_corpus.sh` gates exist | Must be rerun after implementation | +| Public docs/tutorial | Streaming lifecycle and Python tutorial exist | Lifecycle is documented; normative state machine and diagnostics are missing | + +The currently stored short-session reports cover 252 probes per run. They show +zero runtime errors and equal processed-bar counts, but they are ten-minute +experiments and therefore do not satisfy the required three whole-session +experiments through 2025-05-01. + +## Goal + +Deliver PineForge Realtime Execution Contract v1: + +> A deterministic, inspectable transition from confirmed historical OHLCV to +> an ordered executed-trade stream, preserving the complete strategy and +> broker state, confirming bars without lookahead, and expanding each command +> revision into explicit executable order legs with deterministic state +> transitions and fills. Identical configuration, warmup bars, normalized bar +> policy, market events, clock events, and session metadata must produce +> identical command, leg, fill, trade, equity, and report state. + +TradingView historical corpus parity remains a regression requirement. Exact +TradingView realtime feed or executor parity is not a goal. + +## Scope + +### In scope for v1 + +- One long-lived strategy instance across warmup and realtime. +- Fixed-duration input timeframes and existing script-timeframe aggregation. +- Executed-trade events with timestamp, source sequence, price, and quantity. +- Explicit bar-close/clock advancement plus a documented provider/session gap + policy for deciding whether a quiet interval produces a confirmed bar. +- Default close-only strategy scheduling. +- Command revisions that generate independently stateful market, limit, stop, + stop-limit, and trailing executable legs. +- `strategy.entry`, `strategy.order`, `strategy.exit`, `strategy.close`, + `strategy.close_all`, cancellation, pyramiding, reversal, OCA cancel/reduce, + and `process_orders_on_close` as already supported by the historical engine. +- `strategy.exit()` fan-out, entry-relative leg prices, default persistence, + and `from_entry` creation-time cutoff semantics. +- Gaps between observed prices, quiet in-session bars, closed-session gaps, and + first events after a session reopens. +- Stable lifecycle diagnostics for create, replace, activate, ratchet, fill, + cancel, reduce, reject, and expire decisions. +- C++ and C API documentation, Python mirrors, and a runnable tutorial. +- Targeted state-machine tests, deterministic replay tests, and three full + corpus handoffs. + +### Explicitly out of scope for v1 + +- Exchange queue position, liquidity, partial execution from tape quantity, + market impact, latency, spread, bid/ask, or order-book simulation. +- Network reconnect, persistence, broker routing, acknowledgements, and live + exchange order IDs. +- Exact TradingView realtime feed or private broker-emulator reproduction. +- `calc_on_every_tick` rollback and `varip` semantics. +- Realtime `calc_on_order_fills` re-execution. Historical COOF support remains + unchanged; a streaming start using this profile must fail clearly until a + dedicated realtime scheduler is implemented. +- Alert/webhook delivery. The lifecycle events added here are intended to be + the deterministic source for a later alert layer, including JSON payloads. + +## Contract and implementation method + +### 1. Freeze public terminology and keys + +Use these distinct identities: + +- `command_key`: stable public intent key: + - entry: `(entry, id)`; + - raw order: `(order, id)`; + - exit: `(exit, id, from_entry)`; + - close: its explicit close target and internal command identity. +- `command_revision_id`: fresh monotonic identity for every initial placement + or replacement, with immutable command arguments and creation snapshot. +- `order_leg_id`: fresh immutable identity for every executable market, limit, + stop, or trailing child generated by a command revision. Exit revisions can + own several entry-lot groups and several legs per group. +- `priority_sequence`: broker ordering identity, distinct from revision + identity. Focused parity probes must define when replacement preserves or + resets it. +- `event_sequence`: monotonic normalized market-event ordering key. +- `fill_id`: monotonic broker execution identity and downstream idempotency key. +- `entry_lot_id`: immutable identity for each filled entry trade. +- `bar_close_provenance`: the confirming boundary plus the timestamp and + sequence of the last observed executable trade inside the bar, if any. + +Whether the same public text ID is one global order key or belongs to separate +entry/raw/exit namespaces must be pinned by focused TradingView probes before +changing the current implementation. `strategy.cancel(id)` is documented to +cancel all pending orders with that ID, but that does not by itself prove the +replacement collision rule between different placement commands. The internal +revision model must represent the proven public rule without relying on a raw +string comparison accidentally. + +The command revision is immutable; the executable leg's lifecycle state is +mutable. Activation, trailing watermark, active stop, remaining quantity, and +OCA reductions live on the leg, not on the command revision and not globally on +the net position. + +### 2. Define the order state machine + +Executable-leg states: + +- `PENDING_MARKET` +- `PENDING_LIMIT` +- `PENDING_STOP` +- `PENDING_STOP_LIMIT` +- `ACTIVE_STOP_LIMIT` +- `PENDING_TRAIL_ACTIVATION` +- `ACTIVE_TRAIL` +- terminal `FILLED`, `CANCELLED`, `REJECTED`, `EXPIRED`, `REPLACED` + +Required transitions: + +- command creation -> one command revision -> one or more pending executable + legs; +- same-key reissue while pending -> old command revision and remaining legs + `REPLACED`, new command revision and legs with a fresh revision identity; +- exit fan-out -> independently priced/reserved legs for each eligible entry + lot or lot group, including distinct TP, SL, and trailing legs; +- stop-limit stop touch -> `ACTIVE_STOP_LIMIT`, durably retaining the limit; +- trail activation -> `ACTIVE_TRAIL`; favorable events ratchet its watermark; +- eligible price event -> exactly one `FILLED` transition; +- OCA fill -> sibling leg `CANCELLED` or quantity-reduced event, excluding only + the exact filled `order_leg_id` rather than every sibling with the same + public ID; +- risk/margin gate -> `REJECTED` with a reason code; +- explicit cancellation -> `CANCELLED`; +- stream end does not implicitly fill a partial bar. + +Every transition records the before/after state, command revision, executable +leg, public IDs, entry-lot scope, bar index, event timestamp/sequence, observed +price, evaluated trigger level, resulting quantity, and stable reason code. + +The contract must separately specify: + +- whether unchanged stop-limit parameters preserve activation across a + same-key reissue; +- whether unchanged trailing parameters preserve activation/watermark across a + reissue, and when changed parameters reset them; +- how `strategy.exit()` without `from_entry` persists for later entry lots; +- the creation-time cutoff for `strategy.exit(..., from_entry = X)`; +- exact simultaneous-event priority among carried market orders, price-based + entries, exits, trailing legs, OCA effects, reversals, risk gates, and margin + actions, including whether eligibility is recomputed after every fill. + +### 3. Separate scheduler, bar builder, and broker + +Treat these as independent normalized events: + +- `TRADE(timestamp, sequence, price, quantity)` supplies an executable price + and updates a forming bar. +- `BAR_CLOSE(boundary, policy)` confirms a bar selected by the configured + provider/session gap policy. It carries no new executable price by itself. + +For each trade event: + +1. Validate time, sequence, price, and quantity. +2. Confirm every bar boundary strictly before or at the event timestamp. +3. Run close-only strategy calculations for newly confirmed script bars. +4. Apply the event to the new/current input bar (`close` is this exact trade + price; high/low/volume update monotonically). +5. Evaluate eligible executable legs at that exact observed price. +6. Apply fills, OCA/risk effects, excursion state, margin effects, and + diagnostics using the published priority rules, recomputing eligibility + after each state-changing fill when the contract requires it. + +For a bar-close/clock event with no new trade: + +1. Use the explicit gap policy and session calendar to decide which elapsed + intervals become bars. Fixed-grid carry-forward bars are one supported + policy, not a universal inference from a trade tape. +2. Confirm those bars and run close-only strategy calculations. +3. Do not fill market, limit, stop, stop-limit, trailing, immediate, or POC + legs; do not ratchet trails, update price excursions, or run price-triggered + margin liquidation. A clock confirms time; it is not an executed trade. + +When a bar with at least one trade confirms, retain the bar's last trade +timestamp and sequence as its closing-event provenance. A POC/immediate leg +created by the confirmed-close calculation may evaluate only against that +observed close event and must report its actual event timestamp/sequence, not +the OHLCV bar-open timestamp. Previously evaluated carried legs are not walked +over the close a second time. When a confirmed bar has no observed trade, new +POC/immediate legs remain pending until a later eligible trade because no +executable closing event exists. + +For an event exactly on a boundary, the preceding bar confirms first and the +event belongs to the new bar. An order created by the preceding close can then +fill on that boundary event because it is the next observed price event. + +### 4. Make unsupported calculation profiles explicit + +`stream_begin()` must inspect effective strategy properties. V1 accepts only +the close-only profile. It rejects realtime `calc_on_every_tick` or +`calc_on_order_fills` with a stable diagnostic instead of executing a subtly +different model. Validation must occur before warmup mutates the handle, or the +entire begin operation must be transactional and restore a clean pre-call state +on rejection. Tests cover declaration defaults and runtime property overrides. + +### 5. Add diagnostics without coupling to transport + +Add a bounded or caller-drainable order-event stream to the engine report/API. +Diagnostics are simulator facts, not log strings. JSON serialization, +webhooks, retries, and delivery belong above this API. + +Batching and caller drain timing must not change simulator results or the +canonical lifecycle sequence. Maintain a rolling canonical event hash/count +independent of retention, and define deterministic capacity, overflow, and +drop counters. Equivalence tests compare the canonical hash and, when no +overflow occurs, the complete retained records. + +At minimum expose: + +- command revision, executable leg, priority, fill, and entry-lot identities; +- timestamp, source sequence, input/script bar index; +- command kind, leg kind, order state, transition, reason; +- `id`, `from_entry`, OCA name/type; +- side, requested/remaining/filled quantity; +- observed, stop, limit, trail activation, trail watermark, and fill prices; +- position size and equity immediately before and after the transition. + +### 6. Keep historical and realtime kernels consistent but not conflated + +Reuse risk, sizing, commission, trade allocation, and position mutation logic. +Do not send one-price realtime events through a full inferred-OHLC path helper +when that helper has path assumptions or state side effects. The event evaluator +should choose eligibility at one observed point; the historical evaluator can +continue resolving an inferred or magnified path into the same transition and +fill-application kernel. + +As part of this split, move stop-limit activation and every trailing activation, +watermark, and active-stop value onto the relevant executable leg. The current +global position trail watermark cannot represent concurrent trails created at +different times or for different entry lots. + +## Verification method + +### A. State-machine unit tests + +For both long and short directions where applicable: + +| Surface | Required event sequence | +|---|---| +| Market | created at confirmed close; fills once at next observed trade | +| Limit | no fill on wrong side; fills limit-or-better on first eligible observed trade; gap behavior pinned | +| Stop | no fill before trigger; gaps to first observed price through the stop | +| Stop-limit | limit seen before stop does nothing; stop activation persists across events/bars; later limit touch fills once | +| Trailing | inactive before activation; per-leg activation persists; watermark only ratchets favorably; two concurrent lots/offsets remain independent; reversal fills at the active level/gap rule | +| OCA cancel | deterministic winner cancels every sibling except the exact filled leg before another can fill on the same event, including same-public-ID siblings | +| OCA reduce | fill quantity reduces sibling remaining quantities exactly once | +| Pyramiding | multiple entry lots can share an entry ID; cap applies to open `strategy.entry` trades, not raw orders | +| Same-ID lifecycle | reissue before fill replaces with a fresh command revision while priority follows the pinned rule; reissue after fill creates new legs; no leg fills twice | +| Cross-command ID | focused entry/raw/exit/cancel probes pin whether identical public text collides or coexists | +| Exit fan-out | two same-ID entry lots at different prices receive independently priced relative TP/SL/trailing legs; no-`from_entry` persistence and `from_entry` cutoff are pinned | +| Process on close | only an observed closing event is visible to a newly created leg; event time/sequence are exact; an old same-ID resting leg and a new POC leg can both fill in one bar without elapsed-wick lookahead | +| Simultaneous eligibility | exact market/entry/exit/trail/OCA/reversal/risk/margin priority and eligibility recomputation are pinned | +| Gap | fills use the first observed post-gap event under the order-type rule | +| Quiet interval | provider policy decides bar existence; strategy may calculate, but market/limit/stop/trail/POC/margin paths emit no synthetic price transition | +| Session boundary | closed intervals are skipped; first valid reopen event belongs to the correct new bar | + +### B. Invariant and metamorphic tests + +- Replaying the same events one by one and through one contiguous + `strategy_stream_push_ticks()` call must produce the same canonical lifecycle + hash/count, trades, equity, and report metrics. Retained diagnostics are + byte-equivalent when neither run overflows its documented capacity. +- Repeating an already accepted nonzero sequence, reordering events, or moving + time backwards must fail before state mutation. +- Appending future events cannot change any state snapshot taken before those + events. +- Every command revision and executable leg has exactly one creation; every + executable leg has at most one terminal transition. +- Filled quantity conservation holds across position, entry lots, OCA + reductions, and closed-trade allocation. +- Confirmed bars reconstructed from the raw tape equal the trade-derived 1m + OHLCV source for open/high/low/close/volume and timestamps, subject only to + documented source-cleaning differences. +- A canonical handoff-state digest must match between a normal OHLCV run stopped + at T and the historical portion of `stream_begin()` stopped at T. The digest + covers entry lots, command/leg state, cash/equity, risk counters, session + state, Pine series/variables, TA state, security evaluators, timeframe + aggregators, and deterministic sequence counters. + +### C. Historical regression gates + +Run the repository-mandated gates after every behavior change: + +```bash +cmake -B build -S . \ + -DCMAKE_BUILD_TYPE=Release \ + -DPINEFORGE_BUILD_TESTS=ON \ + -DPINEFORGE_BUILD_CORPUS_STRATEGIES=ON +cmake --build build -j4 +ctest --test-dir build --output-on-failure +./scripts/run_corpus.sh +``` + +Any historical corpus drift is a blocker unless the change fixes a documented +historical defect and the reference is deliberately updated. + +### D. Three whole-session corpus experiments + +Use all compiled corpus probes and three fixed, minute-aligned, non-round +handoffs, each ending exclusively at `2025-05-01T00:00:00Z`. Suggested starts, +all within the available local raw-trade interval and the requested range: + +- `2025-02-07T03:17:00Z` +- `2025-03-11T14:23:00Z` +- `2025-03-29T22:41:00Z` + +For each probe and start: + +1. Slice warmup OHLCV strictly before the handoff from the canonical source. +2. Slice one contiguous raw-trade tape from handoff through the common end. +3. Run a bar-only baseline from the same OHLCV origin through the common end. +4. Run warmup plus the one contiguous event tape on the same strategy instance. +5. Produce per-probe and aggregate scores. + +Before implementation, run and freeze the same three-session experiment on the +current merged streaming implementation. That baseline does not define correct +order semantics, but it prevents an arbitrary post-change acceptance threshold +and exposes performance/parity regressions introduced by the refactor. + +Required report scores: + +- runtime success count; +- input/script confirmed-bar count equality; +- trade-count equality; +- ordered structural match percentage using direction, entry/exit minute, bar + indices, and quantity; +- entry/exit price absolute and basis-point p50/p90/p99 deltas; +- net-profit absolute and relative delta; +- first divergence with order-event diagnostic context; +- deterministic rerun hash for every session. + +Slight price/P&L differences between point-event execution and bar-path exports +are reportable, not automatically defects. State discontinuity, lookahead, +nondeterminism, impossible duplicate leg fills, or unexplained bar-count +differences are hard failures. + +Acceptance thresholds are frozen before the post-change runs: + +- hard `756/756` runtime success and equal input/script confirmed-bar counts; +- exact deterministic rerun hashes and zero lifecycle invariant violations; +- exact handoff-state digest equality at every start; +- exact trade-tape-to-trade-derived-OHLCV reconciliation under the versioned + normalization rules; +- no historical corpus regression; +- per-session aggregate ordered structural match no worse than the frozen + pre-change baseline; any individual-probe regression requires an identified + intended semantic correction and published first-divergence trace; +- price and P&L distributions are reported and reviewed rather than hidden by + the structural threshold. + +The existing `scripts/run_stream_corpus_mmap.py` is the correct transport shape: +one mmap-backed contiguous tape per session and no artificial engine chunks. +It needs scoring expansion, enforced invariant/threshold exit status, and a +completed run with the fixed dates above. The report records normalization +version, source hashes, timezone/session metadata, interval inclusivity, tape +hash, OHLCV hash, and tape/OHLCV reconciliation results. + +### E. Documentation and tutorial verification + +- Publish the normative contract and state transition table in cdocs. +- Document every C ABI field and its ownership/lifetime. +- Keep Python `ctypes` layouts synchronized with the C structs. +- Extend the tutorial to print a small order lifecycle trace, including a + persistent stop-limit activation and a quiet-bar confirmation. +- Run the tutorial under CTest and sanitizer configurations. + +## Independent review disposition + +Three independent review passes challenged the initial plan from +trading-system architecture, PineForge implementation, and concise red-team +perspectives. Their high-severity corrections are incorporated above: + +- replace the two-level revision/fill model with command revision -> executable + legs -> fills; +- remove every claimed once-per-public-ID-per-bar guarantee; +- separate immutable creation snapshots from mutable leg lifecycle state; +- make trailing and OCA state leg-specific; +- distinguish `TRADE` from `BAR_CLOSE`, including the no-trade POC case; +- preserve actual closing-event timestamp/sequence for POC fills; +- validate unsupported profiles before warmup mutation; +- add handoff-state digests, explicit priority rules, deterministic diagnostic + overflow semantics, and enforced corpus acceptance thresholds. + +## Completion gate + +The feature is complete only when all of the following are simultaneously +true: + +- the normative contract is public and matches the implementation; +- every in-scope order surface has event-level tests; +- command revisions, executable legs, fills, and their lifecycle are observable + and deterministic; +- unsupported calculation profiles fail explicitly; +- unit, integration, sanitizer, ABI, and full historical corpus gates pass; +- all probes complete all three whole-session experiments through 2025-05-01; +- aggregate scores and every accepted divergence are published; +- the tutorial runs from a clean build. From a4709158870fec9cd2bf9b92c3f4c1018647a8f7 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sat, 11 Jul 2026 05:14:01 +0800 Subject: [PATCH 2/5] Incorporate fourth multi-perspective review pass into the realtime contract plan Corrections and additions from a seven-perspective audit (TradingView docs, PineTS/PyneCore at pinned revisions, FIX/Nautilus/LEAN comparison, two codebase verifications at 98ad849, red-team, practitioner evidence; serious findings adversarially re-verified): - fix the stored-artifact claim (a whole-session 2025-05-01 run existed) and the dead PyneCore live-mode citation; re-base the audit on 98ad849 (#89) - add wait-for-entry exit deferral, slippage, trailing immediate-activation, calc_on_every_history_tick, and command-revision states incl. REJECTED_AT_PLACEMENT to the contract - record the exit REMARKS fill-first rule as working-order bookkeeping (proven by the pinned three-way bracket export); name the two deliberate TV divergences (fixed-grid quiet bars, OCA one-winner pending P8) - add the section 7 TradingView probe plan (P1-P9) and completion-gate item - pin bar_magnifier gating and digest scoping, gap-policy enumeration, quiet-bar equity marking, canonical event ordering, hash serialization, clock-event validation, silent-drop diagnostics, position_episode_id, OCA membership observability, fill-cardinality wording, restart caveat, baseline re-basing escape hatch with pinned engine/corpus/scorer hashes, and section D data/resource prerequisites Co-Authored-By: Claude Fable 5 --- docs/design/realtime-execution-audit.md | 485 +++++++++++++++++++++--- 1 file changed, 425 insertions(+), 60 deletions(-) diff --git a/docs/design/realtime-execution-audit.md b/docs/design/realtime-execution-audit.md index f2d4361..c8fa286 100644 --- a/docs/design/realtime-execution-audit.md +++ b/docs/design/realtime-execution-audit.md @@ -2,7 +2,8 @@ Status: design audit; implementation is not approved by this document alone -Date: 2026-07-11 +Date: 2026-07-11 (revised same day after a fourth, multi-perspective review +pass; see the review disposition section) Worktree branch: `codex/realtime-execution-contract` @@ -70,6 +71,37 @@ Relevant documented behavior: leg remains active until it fills, is replaced, or is cancelled. - `process_orders_on_close` adds a closing-tick processing opportunity. It is not a claim that a real venue will fill an order after the session closes. +- If a `strategy.exit()` call occurs before its referenced entry order + executes, the strategy waits and creates the exit orders only after the + entry order fills; entry-relative exit prices depend on that actual fill. +- The `strategy.exit()` reference states that a call with both stop-loss and + trailing arguments "places only the order that is supposed to fill first, + because both orders are of the 'stop' type". Pinned corpus exports prove + this is working-order bookkeeping, not mutual exclusion: probe + `bracket-exit-three-way-set-once-entry-01` (one set-once exit with stop, + limit, and `trail_points`) shows 18 fixed-stop fills and 774 trail fills + from the same call, with exact engine parity. Both protections stay + behaviorally in force; at any moment one effective stop-type working order + exists and the first-triggered level wins. +- Trailing legs are not always created inactive: a `trail_price` less + favorable than the market at creation, or a negative `trail_points`, creates + the trailing stop immediately; when both `trail_price` and `trail_points` + are supplied, the level expected to activate first is used. +- `slippage` is documented to shift market/stop fill prices unfavorably, and + `backtest_fill_limits_assumption` gates limit fills by a tick margin. The + engine already implements `slippage_`; both must have a defined realtime + rule. +- The strategy declaration also has `calc_on_every_history_tick`, which + changes historical (warmup) execution cadence — a distinct calculation + profile dimension. +- OCA cancellation is not documented as instantaneous: "if order prices are + the same or they are close, more than 1 order of the same group may be + filled". An OCA group's identity is `(oca_name, oca_type)` — same name with + different types is two distinct groups. +- Broker state is exempt from Pine's realtime rollback: "The data from + strategy orders placed or filled on the ticks within a bar is not subject + to rollback" (execution model). Any future `calc_on_every_tick` profile + must roll back Pine variables but never command/leg/fill state. ### PineTS @@ -89,6 +121,9 @@ Relevant design observations: - Its current fill engine evaluates pending orders from bar OHLC. The live stream tests cover indicator/runtime streaming, but the audited strategy tests do not establish a tick-by-tick live broker contract. +- `entry()` performs no same-ID pending replacement — it appends + unconditionally — so PineTS contributes no evidence for the command + replacement rule; only PyneCore and TradingView probes do. - Therefore PineTS is useful evidence for separating order, trade, and position identity, but not a realtime fill oracle for PineForge. @@ -101,16 +136,36 @@ Relevant design observations: - PyneCore's simulator keeps one pending entry/normal order per order ID and replaces the still-unfilled object on reissue. Exit intents are keyed by - `(exit_id, from_entry)`, allowing one exit ID to fan out across entry scopes. + `(exit_id, from_entry)`, gaining a third `book_seq` component for same-bar + stacked partial closes, allowing one exit ID to fan out across entry scopes. - Its open-trade list is distinct from the pending-order maps and net position. -- Its [live-mode contract](https://pynecore.org/docs/advanced/live-mode/) - deliberately suppresses strategy commands during historical warmup and - starts paper trading in the live phase. That is not PineForge's required - model because PineForge must preserve warmup positions, equity, and pending - orders. -- PyneCore separates its simulated `SimPosition` path from exchange-backed - broker plugins. That separation supports keeping PineForge's deterministic +- Its live-mode design suppresses strategy commands during historical warmup: + `lib/__init__.py:127-128` defines a `_strategy_suppressed` flag ("prevents + strategy order placement during historical phase in live mode") gating + every strategy command, though nothing in the OSS repository at the pinned + revision sets it, and PyneCore's docs describe live market support as + planned. (An earlier revision of this document cited + `pynecore.org/docs/advanced/live-mode/`, which no longer resolves.) That + enter-live-flat model is not PineForge's required model because PineForge + must preserve warmup positions, equity, and pending orders. +- PyneCore's abstract `PositionBase` seam is designed for an out-of-repo + `BrokerPosition`; only the simulated `SimPosition` ships in the audited + revision. That separation still supports keeping PineForge's deterministic simulator distinct from future real order routing. +- PyneCore has already probed one rule this plan defers: a reissued trailing + leg keeps its activated watermark only when the trailing parameters compare + equal; a reissue with changed parameters is a cancel+replace that re-arms + from the reissue bar's close (`strategy/__init__.py` ~4226-4262, annotated + "Verified against a TV reference"). Use this as the prior for the plan's + own trailing-reissue probe. Its OCA cancellation also excludes by exact + order object identity, not public ID — direct precedent for the + `order_leg_id` exclusion rule below. +- Neither counterpart implements durable stop-limit activation: PineTS treats + stops and limits independently, and PyneCore turns an order with both stop + and limit into two OCA legs — which conflicts with TradingView's documented + stop-limit. The stop-limit rules in this contract therefore rest on + TradingView documentation plus new focused probes alone; those probes + cannot be deprioritized. ## The named-order hypothesis @@ -126,7 +181,7 @@ An ID does not name a position in Pine: | Open trade / entry lot | immutable `entry_lot_id`, plus public entry ID | Many per net position; several can share a public ID | | Command revision | immutable `command_revision_id` for one placement/reissue | One creation snapshot; may generate several executable legs | | Executable order leg | immutable `order_leg_id`, parent command revision, entry-lot scope, and leg kind | Market, limit, stop, or trail state machine; one command can own many | -| Fill event | immutable `fill_id`, order leg, and market-event provenance | One terminal fill per executable leg | +| Fill event | immutable `fill_id`, order leg, and market-event provenance | In the v1 close-only profile every fill is a full terminal execution, so one fill per leg; the schema reserves several `fill_id`s per leg so a later partial-execution profile is additive, not breaking | | Closed-trade row | FIFO/ANY allocation of a fill to entry lots | One fill can create several rows | A global `(ID, bar) -> at most one fill` rule would conflict with documented @@ -155,11 +210,26 @@ not Pine broker semantics keyed by public ID and bar. One `strategy.exit()` command revision can generate TP, SL, and trailing legs for multiple matching entry lots. Entry-relative prices can differ by lot, so those child legs require independent activation, watermark, reservation, OCA, -and terminal state even when they share one exit ID. +and terminal state even when they share one exit ID. Per lot, the SL and +trailing legs collapse to one effective stop-type working order at any moment +(first-triggered level wins, per the documented fill-first bookkeeping and the +pinned `bracket-exit-three-way-set-once-entry-01` export); the internal legs +model both levels so the working order can be re-selected as the trail +ratchets. An exit revision whose referenced entry order has not yet filled is +deferred: its legs are created only by the entry fill event and are priced off +the actual fill. ## PineForge current-state audit -Audited branch base: `main` at `7a8fc3b`. +Audited branch base: `main` at `7a8fc3b`; re-verified against `98ad849` +(PR #89, "Keep within-cap same-direction entries co-queued with a deferred +close"). #89 does not alter any row below, but it is directly relevant to the +contract: it pins TradingView's pyramiding gate at order PLACEMENT (the engine +previously gated at fill) via a new immutable +`PendingOrder::over_pyramiding_cap_at_placement` creation snapshot — exactly +the creation-snapshot/lifecycle-state split this plan mandates — and it adds +another silent-cancel discriminator to the deferred-close wipe, reinforcing +the diagnostics gap. | Requirement | Current evidence | Assessment | |---|---|---| @@ -168,12 +238,12 @@ Audited branch base: `main` at `7a8fc3b`. | Forming OHLCV without lookahead | Events update O/H/L/C/V before confirmation; exact-boundary ticks are assigned after the prior bar is finalized | Implemented for fixed input timeframes; broader cases need tests | | Higher-timeframe partial aggregation survives handoff | Existing streaming unit test spans historical and realtime input bars inside one 5-minute candle | Implemented for the tested case | | Default close-only script scheduling | Broker evaluates every trade; `on_bar()` runs only when a script bar confirms | Implemented | -| Same-ID pending replacement | `strategy_entry()` and `strategy_order()` erase pending objects with the same raw ID; exits replace by `(id, from_entry)`; all preserve the old `created_seq` | Partial: command, priority, and executable-leg identity are conflated, and raw-ID erasure crosses command categories without a targeted parity probe proving the intended collision rule | +| Same-ID pending replacement | `strategy_entry()` and `strategy_order()` erase pending objects with the same raw ID (bare `o.id == id`, no `OrderType` filter); exits replace by `(id, from_entry)`; all preserve the old `created_seq` | Partial: command, priority, and executable-leg identity are conflated. The namespace rule is already half-pinned in-repo: `clear_existing_exit_order`'s regression comment records that a bare-ID predicate once deleted a still-pending entry (zero trades) and that entry and exit IDs are independent namespaces — yet the entry/order-side erase still treats the namespace as global, so it is a likely live bug, not merely an unproven rule. The remaining probe targets entry-vs-raw-order collision | | One terminal transition per executable leg | Filled pending objects are erased from `pending_orders_` | Implicit only; generated legs, terminal state, and reason are not retained or observable | | Market and stop fills on observed trade events | Streaming tests cover next-event market fill and stop gap-through at the observed price/time | Implemented for tested cases | | Limit fill in streaming | Uses the generic point-bar evaluator | Not directly tested | -| Stop-limit activation across events | `PendingOrder::stop_limit_activated` exists, but normal `process_pending_orders()` does not persist the activation returned by `resolve_entry_stop_limit_fill()`; persistence is currently gated to the historical COOF scheduler | Defect | -| Trailing stop across events | Tick points update one global `trail_best_price_` for the position | Architectural defect for concurrent trailing legs with different lots, creation times, activation levels, or offsets | +| Stop-limit activation across events | `PendingOrder::stop_limit_activated` exists, but normal `process_pending_orders()` does not persist the activation returned by `resolve_entry_stop_limit_fill()` (the out-value is a discarded local); persistence is currently gated to the historical COOF scheduler | Defect. Strictly worse in streaming: events are point bars (O=H=L=C), so a stop-limit entry can only ever fill when one single tick satisfies both the stop and the limit — cross-event activation is impossible today | +| Trailing stop across events | Tick points update one global `trail_best_price_` for the position; `clear_existing_exit_order` resets that global watermark to the current close whenever a fresh trail request with no matching prior `(id, from_entry)` order arrives in position, and entry-relative activation uses the aggregate position entry price, not the per-lot entry | Architectural defect, and not hypothetical: arming a second trailing exit destroys the first trail's already-ratcheted watermark today | | OCA cancel/reduce on one event | Generic fill loop contains OCA behavior and excludes siblings by public ID | Historical tests exist; public-ID exclusion cannot distinguish same-ID executable siblings or revisions | | `process_orders_on_close` | A close-price point pass prevents newly created orders from inspecting the elapsed wick | Partial: it reuses the bar-open timestamp and has no closing-event sequence, so fill provenance is wrong; it can also fill a no-trade synthetic bar | | Quiet intervals | Clock advancement creates zero-volume carry-forward bars | Contract issue: both the pre-script broker pass and the post-script POC pass can create synthetic fills, excursions, trailing changes, or margin actions without an observed trade | @@ -181,14 +251,25 @@ Audited branch base: `main` at `7a8fc3b`. | Event-level diagnostics | No command-revision, executable-leg, fill, or decision log in the public report | Missing | | Replay/live equivalence | `push_ticks()` is a loop over `push_tick()` and a basic ordering test exists | Structural implementation is sound; result equivalence needs an explicit test | | Handoff state equivalence | Tests inspect selected visible fields after warmup | Missing a canonical digest covering all broker, Pine, TA, security, aggregation, risk, session, and sequence state | -| Three full corpus handoffs ending 2025-05-01 | `run_stream_corpus_mmap.py` supports exactly three starts and one contiguous mmap tape | Runner exists, but exit status currently fails only on runtime errors, not parity/invariant failure; current build artifacts prove only short 10-minute sessions | +| Three full corpus handoffs ending 2025-05-01 | `run_stream_corpus_mmap.py` supports exactly three starts and one contiguous mmap tape | Runner exists, but exit status currently fails only on runtime errors, not parity/invariant failure. Whole-session feasibility is already proven: a stored three-session run through 2025-05-01 exists (see below), though with clustered starts and pre-#86..#89 binaries | +| Calculation-profile validation at `stream_begin()` | None exists: realtime dispatch calls `on_bar()` directly and never the COOF scheduler (`coof_scheduler_active_` is set only inside `run()`), so a COOF strategy warms up with COOF semantics then silently degrades to close-only after handoff. `calc_on_every_tick` does not exist as an engine property at all (codegen drops the declaration; both corpus probes declaring it compile to close-only artifacts) | Defect: exactly the silent-approximation hazard section 4 forbids, live today; plus a codegen prerequisite before the gate can inspect every-tick profiles | +| `bar_magnifier` across handoff | `stream_begin()` silently forces warmup to run with magnifier off, then enables `bar_magnifier_enabled_` for the realtime phase; four corpus probes configure `"bar_magnifier": true` | Unaddressed contract question: for those probes a magnifier-on normal run cannot digest-equal the magnifier-off warmup, so the handoff-digest invariant must be scoped and section 4 must pin accept/reject/override | | Historical TradingView regression | Required `ctest` and `scripts/run_corpus.sh` gates exist | Must be rerun after implementation | | Public docs/tutorial | Streaming lifecycle and Python tutorial exist | Lifecycle is documented; normative state machine and diagnostics are missing | -The currently stored short-session reports cover 252 probes per run. They show -zero runtime errors and equal processed-bar counts, but they are ten-minute -experiments and therefore do not satisfy the required three whole-session -experiments through 2025-05-01. +Stored artifacts (correcting an earlier revision of this document, which +wrongly claimed only ten-minute sessions existed): besides the ten-minute +smoke reports, the build directory holds three 60-minute sweeps and one full +three-session run through `2025-05-01T00:00:00Z` +(`stream_corpus_to_20250501.json`): handoffs 2025-03-29/03-30/03-31T00:00Z, +252 probes per session over 154.9M-161.8M ticks each, 216.7 seconds total +elapsed, and per-session summaries errors=0, input/script bars equal 252/252, +trade-count equal 250/252, ordered structural match 243/252. That run proves +whole-session feasibility and gives a real parity reference level, but it +does not satisfy section D as specified: its starts are clustered midnights +rather than the spread non-round starts, it predates the behavior-changing +merges #86-#89, and the runner still lacks the expanded scoring and enforced +exit status. The section D baseline must be re-run on current `main`. ## Goal @@ -203,7 +284,15 @@ Deliver PineForge Realtime Execution Contract v1: > identical command, leg, fill, trade, equity, and report state. TradingView historical corpus parity remains a regression requirement. Exact -TradingView realtime feed or executor parity is not a goal. +TradingView realtime feed or executor parity is not a goal — and is not even +well-defined: TradingView documents that reloading a chart erases and +re-simulates the Strategy Tester from OHLC-only history ("not a bug"), and +realtime tick streams are plan-tier dependent and conflated, so no two +TradingView sessions see the same tape. The published contract should state +that the v1 profile corresponds to TradingView's REALTIME fill model +(tick-driven resting-order evaluation with close-only script calculation), +not its historical bar-path model, so users are not surprised when streamed +results differ from their Strategy Tester backtest. ## Scope @@ -221,7 +310,17 @@ TradingView realtime feed or executor parity is not a goal. `strategy.close_all`, cancellation, pyramiding, reversal, OCA cancel/reduce, and `process_orders_on_close` as already supported by the historical engine. - `strategy.exit()` fan-out, entry-relative leg prices, default persistence, - and `from_entry` creation-time cutoff semantics. + `from_entry` creation-time cutoff semantics, and wait-for-entry deferral + (exit issued before its entry fills expands only on the entry fill). +- `slippage` semantics for event fills (the engine already implements + historical slippage; the realtime rule is part of this contract). + `backtest_fill_limits_assumption` is explicitly unsupported and assumed 0, + so the limit-or-better rule is unambiguous. +- A `request.security()` realtime decision: either pin what a close-only + realtime calculation sees for a higher-timeframe series whose parent bar is + still forming (per `barmerge.lookahead` mode) with unit tests, or reject + strategies using `request.security()` at `stream_begin()` for v1. Silent + approximation is not an option; this is a known-sharp area. - Gaps between observed prices, quiet in-session bars, closed-session gaps, and first events after a session reopens. - Stable lifecycle diagnostics for create, replace, activate, ratchet, fill, @@ -237,12 +336,26 @@ TradingView realtime feed or executor parity is not a goal. - Network reconnect, persistence, broker routing, acknowledgements, and live exchange order IDs. - Exact TradingView realtime feed or private broker-emulator reproduction. -- `calc_on_every_tick` rollback and `varip` semantics. +- `calc_on_every_tick` rollback and `varip` semantics. The future profile + must honor the documented asymmetry: Pine variables roll back per tick, but + broker order/fill state never does. The command/leg/fill separation defined + here (immutable creation snapshots, mutable leg state outside Pine's + rollback domain) is the intended substrate for that profile. +- `calc_on_every_history_tick` warmup cadence (see section 4's gate). - Realtime `calc_on_order_fills` re-execution. Historical COOF support remains unchanged; a streaming start using this profile must fail clearly until a dedicated realtime scheduler is implemented. +- Session restart continuity. A restarted session is NOT replay-equivalent to + the continuous stream: re-warming from derived OHLCV re-simulates through + the historical bar-path kernel intervals previously executed tick-by-tick, + so fills can move or disappear (the exact analog of TradingView's + documented reload divergence). Determinism holds per tape; operators + needing continuity must persist and replay the raw trade tape. - Alert/webhook delivery. The lifecycle events added here are intended to be the deterministic source for a later alert layer, including JSON payloads. + Lifecycle events are simulator facts, not account truth; delivery will be + at-least-once in any transport, so consumers must dedupe on `fill_id`, and + broker adapters own account reconciliation. ## Contract and implementation method @@ -269,13 +382,20 @@ Use these distinct identities: - `bar_close_provenance`: the confirming boundary plus the timestamp and sequence of the last observed executable trade inside the bar, if any. -Whether the same public text ID is one global order key or belongs to separate -entry/raw/exit namespaces must be pinned by focused TradingView probes before -changing the current implementation. `strategy.cancel(id)` is documented to -cancel all pending orders with that ID, but that does not by itself prove the -replacement collision rule between different placement commands. The internal -revision model must represent the proven public rule without relying on a raw -string comparison accidentally. +The `command_key` schema above is provisional until probe P1 (section 7) +concludes. Whether the same public text ID is one global order key or belongs +to separate entry/raw/exit namespaces must be pinned by focused TradingView +probes before changing the current implementation; kind-tagged keys can still +express a global-collision answer via a cross-kind replacement rule, but the +schema must not be treated as frozen. Half the rule is already pinned +in-repo: `clear_existing_exit_order`'s regression comment records, from a +real zero-trades bug, that entry-order IDs and exit-order IDs are independent +namespaces — which makes the current entry/order-side bare-ID erase a likely +live bug. `strategy.cancel(id)` is documented to cancel all pending orders +with that ID, but that does not by itself prove the replacement collision +rule between different placement commands. The internal revision model must +represent the proven public rule without relying on a raw string comparison +accidentally. The command revision is immutable; the executable leg's lifecycle state is mutable. Activation, trailing watermark, active stop, remaining quantity, and @@ -284,6 +404,18 @@ the net position. ### 2. Define the order state machine +Command-revision states (the revision itself has a small lifecycle distinct +from its legs): + +- `WAITING_FOR_ENTRY`: an exit revision whose referenced entry order has not + filled; it owns no legs yet and expands deterministically on the entry fill + event, pricing entry-relative legs off the actual fill; +- `EXPANDED`: legs created; +- terminal `REJECTED_AT_PLACEMENT`: the command is refused before any leg + exists (for example TradingView's placement-time pyramiding gate, pinned by + PR #89), with a reason code; +- terminal `REPLACED`: superseded by a same-key reissue. + Executable-leg states: - `PENDING_MARKET` @@ -292,25 +424,36 @@ Executable-leg states: - `PENDING_STOP_LIMIT` - `ACTIVE_STOP_LIMIT` - `PENDING_TRAIL_ACTIVATION` -- `ACTIVE_TRAIL` +- `ACTIVE_TRAIL` (a leg can be born in this state: `trail_price` already + reached or less favorable than market at creation, or negative + `trail_points`) - terminal `FILLED`, `CANCELLED`, `REJECTED`, `EXPIRED`, `REPLACED` Required transitions: - command creation -> one command revision -> one or more pending executable - legs; + legs (or `WAITING_FOR_ENTRY` deferral for early exits); - same-key reissue while pending -> old command revision and remaining legs `REPLACED`, new command revision and legs with a fresh revision identity; - exit fan-out -> independently priced/reserved legs for each eligible entry - lot or lot group, including distinct TP, SL, and trailing legs; + lot or lot group, including distinct TP, SL, and trailing legs; per lot the + stop-type legs collapse to one effective working order (first-triggered + level wins), matching the documented fill-first bookkeeping and the pinned + three-way bracket export; - stop-limit stop touch -> `ACTIVE_STOP_LIMIT`, durably retaining the limit; - trail activation -> `ACTIVE_TRAIL`; favorable events ratchet its watermark; -- eligible price event -> exactly one `FILLED` transition; +- eligible price event -> exactly one `FILLED` transition (v1 fills are full + terminal executions; the schema reserves several fills per leg for a later + partial-execution profile); - OCA fill -> sibling leg `CANCELLED` or quantity-reduced event, excluding only the exact filled `order_leg_id` rather than every sibling with the same public ID; - risk/margin gate -> `REJECTED` with a reason code; - explicit cancellation -> `CANCELLED`; +- `EXPIRED` is produced only by the enumerated expiry rules: a POC/immediate + market leg whose only eligible closing event has passed (the existing + POC/COOF rule). Expiry is order management, not a fill, so it may be + clock-driven; no other expiry producer exists in v1; - stream end does not implicitly fill a partial bar. Every transition records the before/after state, command revision, executable @@ -322,9 +465,24 @@ The contract must separately specify: - whether unchanged stop-limit parameters preserve activation across a same-key reissue; - whether unchanged trailing parameters preserve activation/watermark across a - reissue, and when changed parameters reset them; + reissue, and when changed parameters reset them (PyneCore's TV-probed + parameter-equality rule is the prior); +- trailing creation-time activation: the immediate-activation + parameterizations (unfavorable `trail_price`, negative `trail_points`) and + the fill-first selection when both `trail_price` and `trail_points` are + given; - how `strategy.exit()` without `from_entry` persists for later entry lots; - the creation-time cutoff for `strategy.exit(..., from_entry = X)`; +- the realtime `slippage` rule (expected: applied on top of the observed + trade price exactly as the historical kernel applies it to market/stop + fills); +- the OCA cancellation rule for several same-group legs eligible on one + event. TradingView documents that same/close-priced same-group orders "may + be filled" together, and the historical kernel encodes a probed TV quirk + (group cancel only after a full fill). A strict + one-winner-per-event rule is a deliberate, named divergence unless a + focused probe pins TV's realtime behavior; either way the historical + kernel's corpus-parity OCA behavior must be preserved verbatim; - exact simultaneous-event priority among carried market orders, price-based entries, exits, trailing legs, OCA effects, reversals, risk gates, and margin actions, including whether eligibility is recomputed after every fill. @@ -338,6 +496,15 @@ Treat these as independent normalized events: - `BAR_CLOSE(boundary, policy)` confirms a bar selected by the configured provider/session gap policy. It carries no new executable price by itself. +Clock events are validated like trade events: boundaries must be strictly +increasing, a boundary earlier than the last accepted trade timestamp is +rejected (or resolved by an explicitly documented tie rule), and a duplicate +boundary is rejected or idempotent — never double-confirming. An invalid +clock event fails before state mutation, exactly like an invalid trade, +because clock events trigger strategy calculation. Hard-failing repeated +trade sequences is deliberate for the deterministic core: real feeds are +at-least-once, and dedup belongs to the caller/adapter layer above this API. + For each trade event: 1. Validate time, sequence, price, and quantity. @@ -353,12 +520,32 @@ For each trade event: For a bar-close/clock event with no new trade: 1. Use the explicit gap policy and session calendar to decide which elapsed - intervals become bars. Fixed-grid carry-forward bars are one supported - policy, not a universal inference from a trade tape. -2. Confirm those bars and run close-only strategy calculations. + intervals become bars. V1 supports exactly two policies: + - `fixed-grid` (default, and the one used by the section D corpus + experiments): every elapsed in-session interval becomes a zero-volume + carry-forward bar on the input-timeframe grid, keeping `bar_index` + aligned with the 1m OHLCV grid; + - `data-driven`: a bar confirms only on the first trade after its + boundary — TradingView's actual behavior, where a tickless bar never + confirms and the script never runs. + Fixed-grid is a published, named divergence from TradingView: clock- + confirmed quiet bars RUN strategy logic (bars-since exits, time/session + flattening, na-volume-sensitive conditions) at moments a purely + data-driven TradingView chart would be silent. The session calendar + source is the engine's `syminfo` session/timezone metadata. Note the + 24/7 perp corpus cannot exercise closed-session rows; those exist as + synthetic unit tests only. +2. Confirm those bars and run close-only strategy calculations. Open-position + equity and `strategy.openprofit` on a quiet confirmed bar are marked at + the last observed trade price (the carried close); the mark price never + moves without an observed trade. 3. Do not fill market, limit, stop, stop-limit, trailing, immediate, or POC legs; do not ratchet trails, update price excursions, or run price-triggered margin liquidation. A clock confirms time; it is not an executed trade. + Order management is not restricted: the clock-confirmed calculation may + create, cancel, and replace command revisions and legs (transitions carry + clock provenance); only price-contingent transitions — fill, trigger, + ratchet, excursion, margin — require an observed trade. When a bar with at least one trade confirms, retain the bar's last trade timestamp and sequence as its closing-event provenance. A POC/immediate leg @@ -369,6 +556,13 @@ over the close a second time. When a confirmed bar has no observed trade, new POC/immediate legs remain pending until a later eligible trade because no executable closing event exists. +When the confirming trigger is a later event (the next bar's first trade or a +clock event), the retroactive POC evaluation against the retained closing +event completes before any leg is evaluated at the new boundary event, within +one dispatch. Diagnostics order by processing order; a fill's provenance +timestamp/sequence is carried as data and may be older than events already +processed (see section 5's canonical-ordering rule). + For an event exactly on a boundary, the preceding bar confirms first and the event belongs to the new bar. An order created by the preceding close can then fill on that boundary event because it is the next observed price event. @@ -376,11 +570,37 @@ fill on that boundary event because it is the next observed price event. ### 4. Make unsupported calculation profiles explicit `stream_begin()` must inspect effective strategy properties. V1 accepts only -the close-only profile. It rejects realtime `calc_on_every_tick` or -`calc_on_order_fills` with a stable diagnostic instead of executing a subtly -different model. Validation must occur before warmup mutates the handle, or the -entire begin operation must be transactional and restore a clean pre-call state -on rejection. Tests cover declaration defaults and runtime property overrides. +the close-only profile. It rejects realtime `calc_on_every_tick`, +`calc_on_every_history_tick`, or `calc_on_order_fills` with a stable +diagnostic instead of executing a subtly different model. Validation must +occur before warmup mutates the handle, or the entire begin operation must be +transactional and restore a clean pre-call state on rejection. Tests cover +declaration defaults and runtime property overrides. + +`bar_magnifier` belongs in the same gate: `stream_begin()` currently forces +warmup magnifier off and enables it for the realtime phase, silently. The +contract must pin whether magnifier configurations are rejected, honored +during warmup, or documented as forced-off — and the handoff-digest invariant +in section B is scoped by that decision. The `request.security()` decision +from the scope section (pin realtime semantics or reject) is also enforced +here. + +Current-state notes that shape this work: + +- The silent-approximation hazard is live today: realtime dispatch calls + `on_bar()` directly and never the COOF scheduler, so a COOF strategy warms + up with COOF semantics and then silently degrades to close-only after + handoff. `calc_on_order_fills_` exists and can be validated immediately. +- `calc_on_every_tick` does not exist as an engine property: codegen drops + the declaration entirely (no member, no override field, no emission). The + two corpus probes declaring `calc_on_every_tick = true` therefore compile + to effectively close-only artifacts and stream and score normally in the + section D gate today. Plumbing the property from codegen is a prerequisite + for enforcing this half of the gate; when that lands and corpus artifacts + are regenerated, the section D acceptance rule must first define + expected-rejection scoring (expected-reject counts as pass, or the probes + are excluded from the denominator) so the gate and this section can + coexist. ### 5. Add diagnostics without coupling to transport @@ -394,19 +614,47 @@ independent of retention, and define deterministic capacity, overflow, and drop counters. Equivalence tests compare the canonical hash and, when no overflow occurs, the complete retained records. +The diagnostic stream's processing order (equivalently a global transition +sequence, of which `fill_id` order is a subsequence) is the sole canonical +total order. Event timestamp/sequence on a transition is provenance only — +non-unique (a POC fill and a resting fill can share one event) and +non-monotonic across transitions (a retroactive POC fill carries provenance +older than events already processed). Consumers must never order by +provenance. + +The canonical hash input is the raw IEEE 754 bit patterns of the canonical +fields in a fixed byte order — never formatted decimals. The equality claim +is scoped: same binary plus same input gives the same hash; cross-platform +equality is best-effort unless floating-point arithmetic is pinned. + At minimum expose: - command revision, executable leg, priority, fill, and entry-lot identities; - timestamp, source sequence, input/script bar index; - command kind, leg kind, order state, transition, reason; -- `id`, `from_entry`, OCA name/type; +- `id`, `from_entry`, OCA name/type, and on every OCA cancel/reduce the + resolved member `order_leg_id` set (or an `oca_group_instance_id`), so + membership at the moment of the effect is observable without re-deriving + name resolution across revisions; +- a monotonic `position_episode_id` that increments each time the net + position leaves flat, so the later alert layer never reconstructs episode + boundaries from fill deltas; - side, requested/remaining/filled quantity; - observed, stop, limit, trail activation, trail watermark, and fill prices; - position size and equity immediately before and after the transition. +Today's fully invisible transitions must become first-class events with +reason codes: risk-rejected entries are currently dropped silently inside the +fill loop, and OCA cancels, the post-full-close same-direction wipe, and the +flat-position purge all erase in place with no record. The streaming +fast path that skips the broker pass when no orders are pending and the +position is flat must either remain provably event-free or be removed, so +canonical event hashes are replay-invariant. + ### 6. Keep historical and realtime kernels consistent but not conflated -Reuse risk, sizing, commission, trade allocation, and position mutation logic. +Reuse risk, sizing, commission, slippage, trade allocation, and position +mutation logic. Do not send one-price realtime events through a full inferred-OHLC path helper when that helper has path assumptions or state side effects. The event evaluator should choose eligibility at one observed point; the historical evaluator can @@ -416,7 +664,45 @@ fill-application kernel. As part of this split, move stop-limit activation and every trailing activation, watermark, and active-stop value onto the relevant executable leg. The current global position trail watermark cannot represent concurrent trails created at -different times or for different entry lots. +different times or for different entry lots — and it already fails today: +arming a second trailing exit resets the global watermark to the current +close, destroying the first trail's ratchet. + +The historical kernel's corpus-parity behaviors are load-bearing regression +anchors and must be preserved verbatim under this split — in particular its +probed OCA behavior (multi-fill at coincident path points, group cancel after +a full fill). The realtime leg-ID exclusion rule must not leak into the +historical path, or the historical corpus gate will drift. + +### 7. TradingView probe plan + +The state machine's replacement rule, exit fan-out, and fill loop cannot be +finalized without answers to the semantics this document defers to "focused +TradingView probes". Those probes are scheduled work with a defined method, +not a hand-wave; the completion gate requires every one resolved and +archived. + +Method: every probe below concerns deterministic broker-emulator behavior and +is observable in historical Strategy Tester exports — the corpus method this +repository already uses — so no live-session observation is required for v1. +Each probe is one minimal Pine script plus its exported trade list, archived +as a corpus probe under `corpus/validation/` with the pinned export, and its +conclusion recorded in this document before the dependent implementation +lands. + +| # | Open semantic | Prior / partial evidence | +|---|---|---| +| P1 | Public-ID collision across command kinds on placement (entry vs raw order vs exit sharing a text ID) | Exit-vs-entry independence already pinned in-repo by the `clear_existing_exit_order` regression; remaining: entry-vs-raw-order and placement-collision direction | +| P2 | Whether replacement preserves or resets broker priority (`priority_sequence`) | None | +| P3 | Stop-limit activation persistence across a same-key reissue (unchanged vs changed parameters) | No counterpart precedent exists; TradingView docs pin durability without reissue only | +| P4 | Trailing activation/watermark persistence across a reissue | PyneCore's TV-probed parameter-equality rule is the prior | +| P5 | `strategy.exit()` without `from_entry`: persistence for later entry lots | Docs give the creation-time cutoff for `from_entry`; the no-`from_entry` persistence needs an export | +| P6 | `strategy.exit(..., from_entry = X)` creation-time cutoff edge cases (same-bar entry, reissue) | Docs pin the base rule | +| P7 | Simultaneous-event priority and per-fill eligibility recomputation (carried market vs price-based entry vs exit vs trail vs OCA vs reversal vs risk/margin) | Partial coverage by existing corpus probes; needs targeted coincident-level probes | +| P8 | OCA same-event multi-trigger: does TV cancel before the second same-tick fill? | TV docs say close-priced same-group orders "may be filled"; historical kernel encodes multi-fill; decide divergence or parity | +| P9 | Per-lot stop-type working-order selection when entry-relative levels differ by lot | Three-way bracket probe pins the single-lot case | + +Section 1's `command_key` schema is provisional until P1 concludes. ## Verification method @@ -426,17 +712,18 @@ For both long and short directions where applicable: | Surface | Required event sequence | |---|---| -| Market | created at confirmed close; fills once at next observed trade | -| Limit | no fill on wrong side; fills limit-or-better on first eligible observed trade; gap behavior pinned | -| Stop | no fill before trigger; gaps to first observed price through the stop | +| Market | created at confirmed close; fills once at next observed trade, with the configured `slippage` applied by the documented realtime rule | +| Limit | no fill on wrong side; fills limit-or-better on first eligible observed trade (`backtest_fill_limits_assumption` fixed at 0); gap behavior pinned | +| Stop | no fill before trigger; gaps to first observed price through the stop; `slippage` applied by the documented realtime rule | | Stop-limit | limit seen before stop does nothing; stop activation persists across events/bars; later limit touch fills once | -| Trailing | inactive before activation; per-leg activation persists; watermark only ratchets favorably; two concurrent lots/offsets remain independent; reversal fills at the active level/gap rule | -| OCA cancel | deterministic winner cancels every sibling except the exact filled leg before another can fill on the same event, including same-public-ID siblings | +| Trailing | inactive before activation, and a leg born `ACTIVE_TRAIL` for the immediate-activation parameterizations; per-leg activation persists; watermark only ratchets favorably; two concurrent lots/offsets remain independent; reversal fills at the active level/gap rule | +| Deferred exit | exit issued in the same calculation as its entry produces no legs until the entry fills; legs are then priced off the actual fill; interaction with same-event eligibility recomputation pinned | +| OCA cancel | the rule pinned by probe P8 (strict one-winner exclusion of every sibling except the exact filled leg, including same-public-ID siblings — or TV's documented same-event multi-fill), and the historical kernel's behavior stays unchanged | | OCA reduce | fill quantity reduces sibling remaining quantities exactly once | -| Pyramiding | multiple entry lots can share an entry ID; cap applies to open `strategy.entry` trades, not raw orders | -| Same-ID lifecycle | reissue before fill replaces with a fresh command revision while priority follows the pinned rule; reissue after fill creates new legs; no leg fills twice | -| Cross-command ID | focused entry/raw/exit/cancel probes pin whether identical public text collides or coexists | -| Exit fan-out | two same-ID entry lots at different prices receive independently priced relative TP/SL/trailing legs; no-`from_entry` persistence and `from_entry` cutoff are pinned | +| Pyramiding | multiple entry lots can share an entry ID; cap applies to open `strategy.entry` trades, not raw orders; the placement-time gate pinned by #89 rejects at the command revision (`REJECTED_AT_PLACEMENT`), not at fill | +| Same-ID lifecycle | reissue before fill replaces with a fresh command revision while priority follows the pinned rule; reissue after fill creates new legs; no leg fills twice; #89's within-cap co-queued entry + deferred close case is covered | +| Cross-command ID | focused entry/raw/exit/cancel probes (P1) pin whether identical public text collides or coexists | +| Exit fan-out | two same-ID entry lots at different prices receive independently priced relative TP/SL/trailing legs; per lot exactly one effective stop-type working order exists at any moment (first-triggered wins, re-selected as the trail ratchets); no-`from_entry` persistence and `from_entry` cutoff are pinned | | Process on close | only an observed closing event is visible to a newly created leg; event time/sequence are exact; an old same-ID resting leg and a new POC leg can both fill in one bar without elapsed-wick lookahead | | Simultaneous eligibility | exact market/entry/exit/trail/OCA/reversal/risk/margin priority and eligibility recomputation are pinned | | Gap | fills use the first observed post-gap event under the order-type rule | @@ -451,6 +738,9 @@ For both long and short directions where applicable: byte-equivalent when neither run overflows its documented capacity. - Repeating an already accepted nonzero sequence, reordering events, or moving time backwards must fail before state mutation. +- Clock events obey the section 3 validation rules: regressed or duplicate + boundaries, and boundaries earlier than the last accepted trade timestamp, + fail before state mutation. - Appending future events cannot change any state snapshot taken before those events. - Every command revision and executable leg has exactly one creation; every @@ -464,7 +754,10 @@ For both long and short directions where applicable: at T and the historical portion of `stream_begin()` stopped at T. The digest covers entry lots, command/leg state, cash/equity, risk counters, session state, Pine series/variables, TA state, security evaluators, timeframe - aggregators, and deterministic sequence counters. + aggregators, and deterministic sequence counters. The comparison run uses + the same effective `bar_magnifier` setting that `stream_begin()` applies + during warmup, per the section 4 decision — otherwise the invariant is + unsatisfiable for magnifier-configured probes. ### C. Historical regression gates @@ -493,6 +786,23 @@ all within the available local raw-trade interval and the requested range: - `2025-03-11T14:23:00Z` - `2025-03-29T22:41:00Z` +Data and resource prerequisites (explicit, so the gate is schedulable): + +- Raw daily ETHUSDT-perp trade archives live on the external mount + `/Volumes/PineforgeData/binance_ethusdtp_1y/raw/trades` (2025-01-01 through + 2026-07-08), not in the repository or LFS; the experiments depend on that + mount being present. +- The existing mmap tape starts 2025-03-29; the 2025-02-07 start requires + rebuilding one contiguous tape roughly 2.5x larger (~83 days). +- Expected volume: session tick counts approximately 424.8M / 230.6M / + 158.2M (813.6M per probe across the three sessions), about 205 billion + tick evaluations per full experiment, doubled by the pre-change baseline. + The stored three-session run (33-day sessions) completed in 216.7 seconds + total, so the full experiment is an hours-scale job, not days. +- The runner warms all 756 engine instances up-front before the worker pool, + so peak resident memory scales with 756 warmed states; verify headroom or + add a per-session sequencing option. + For each probe and start: 1. Slice warmup OHLCV strictly before the handoff from the canonical source. @@ -504,7 +814,12 @@ For each probe and start: Before implementation, run and freeze the same three-session experiment on the current merged streaming implementation. That baseline does not define correct order semantics, but it prevents an arbitrary post-change acceptance threshold -and exposes performance/parity regressions introduced by the refactor. +and exposes performance/parity regressions introduced by the refactor. The +frozen baseline artifact must record the engine commit hash, corpus revision, +and the scoring-harness commit hash; "no worse than baseline" is meaningful +only against the same scorer version. The stored 2025-05-01 run is the +feasibility and format precedent, but it predates #86-#89 and cannot be +adopted as the frozen baseline. Required report scores: @@ -512,7 +827,10 @@ Required report scores: - input/script confirmed-bar count equality; - trade-count equality; - ordered structural match percentage using direction, entry/exit minute, bar - indices, and quantity; + indices, and quantity — reported both with and without trailing-exit + trades, because tick-path trailing vs the baseline's bar-path trailing is + the expected dominant divergence class and must not mask regressions in + other order surfaces; - entry/exit price absolute and basis-point p50/p90/p99 deltas; - net-profit absolute and relative delta; - first divergence with order-event diagnostic context; @@ -533,7 +851,15 @@ Acceptance thresholds are frozen before the post-change runs: - no historical corpus regression; - per-session aggregate ordered structural match no worse than the frozen pre-change baseline; any individual-probe regression requires an identified - intended semantic correction and published first-divergence trace; + intended semantic correction and published first-divergence trace. Because + the baseline embeds behaviors this document classifies as defects (quiet-bar + synthetic fills, POC provenance, the global trail watermark), an intended + systemic correction may lower the aggregate: a documented aggregate + re-baseline is permitted when the divergence class is traced to an + enumerated intended correction, published with its trace — the bound has an + escape hatch for corrections, never for unexplained drift; +- if profile-rejection ever applies to compiled probes (see section 4), the + expected-rejection scoring rule is defined before the run; - price and P&L distributions are reported and reviewed rather than hidden by the structural threshold. @@ -570,17 +896,56 @@ perspectives. Their high-severity corrections are incorporated above: - add handoff-state digests, explicit priority rules, deterministic diagnostic overflow semantics, and enforced corpus acceptance thresholds. +Those three passes are summarized here without archived artifacts; future +review rounds must link their artifacts and record rejected findings, not +only accepted ones. + +A fourth, seven-perspective review pass (2026-07-11: official TradingView +docs verification, PineTS/PyneCore re-verification at the pinned revisions, +industry architecture comparison against FIX/NautilusTrader/LEAN, two +independent codebase verifications at `98ad849`, an internal red-team, and +TradingView practitioner/community evidence — every serious finding +adversarially re-verified by a second agent) produced this revision. Its +material corrections: + +- corrected the stored-artifact claim (a whole-session 2025-05-01 run + already existed) and the dead PyneCore live-mode citation; +- re-based the current-state audit on `98ad849` and folded #89's + placement-time pyramiding snapshot into the command-revision model + (`REJECTED_AT_PLACEMENT`); +- added the documented wait-for-entry exit deferral, slippage, + trailing immediate-activation, and `calc_on_every_history_tick` to the + contract; recorded the exit REMARKS fill-first sentence as working-order + bookkeeping proven by the pinned three-way bracket export (two proposed + "corrections" here were themselves refuted by corpus evidence and did NOT + change the leg model — the adversarial verify pass earned its cost); +- named the two deliberate TradingView divergences (fixed-grid quiet-bar + calculation; OCA one-winner rule pending P8) instead of leaving them + implicit; +- replaced the open-ended probe deferrals with the section 7 probe plan and + a completion-gate item; +- scoped the handoff digest for `bar_magnifier`, enumerated the gap + policies, pinned quiet-bar equity marking, canonical event ordering, hash + serialization, clock-event validation, diagnostics for today's silent + drops, `position_episode_id`, OCA membership observability, fill-cardinality + wording (partial fills stay additive), the restart-divergence caveat, and + the re-baselining escape hatch with pinned baseline/scorer hashes. + ## Completion gate The feature is complete only when all of the following are simultaneously true: - the normative contract is public and matches the implementation; +- every section 7 probe (P1-P9) is resolved, its export archived as a corpus + probe, and its conclusion recorded in this document; - every in-scope order surface has event-level tests; - command revisions, executable legs, fills, and their lifecycle are observable and deterministic; - unsupported calculation profiles fail explicitly; - unit, integration, sanitizer, ABI, and full historical corpus gates pass; -- all probes complete all three whole-session experiments through 2025-05-01; +- all probes complete all three whole-session experiments through 2025-05-01, + with the frozen baseline's engine/corpus/scorer hashes recorded in the + published report; - aggregate scores and every accepted divergence are published; - the tutorial runs from a clean build. From 106a4c3a17f5f4293fbe40106a942171f8aae0b2 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sat, 11 Jul 2026 06:42:32 +0800 Subject: [PATCH 3/5] Implement deterministic realtime execution contract --- CMakeLists.txt | 1 + README.md | 5 +- docker/run_json.py | 68 +++++- docs/design/realtime-execution-audit.md | 23 +- docs/pages/ffi-python.md | 32 ++- docs/pages/index.md | 6 +- docs/pages/report-schema.md | 26 ++- docs/pages/streaming.md | 45 +++- docs/realtime-contract-3-session-report.md | 68 ++++++ include/pineforge/engine.hpp | 96 +++++++- include/pineforge/pineforge.h | 65 +++++- scripts/check_c_abi_runtime.py | 1 + scripts/run_strategy.py | 69 +++++- scripts/run_stream_corpus.py | 50 ++++ src/c_abi.cpp | 10 + src/engine_fills.cpp | 120 +++++++++- src/engine_order_events.cpp | 164 +++++++++++++ src/engine_orders.cpp | 60 ++++- src/engine_report.cpp | 42 ++++ src/engine_run.cpp | 19 ++ src/engine_strategy_commands.cpp | 149 +++++++++++- src/engine_stream.cpp | 134 +++++++++-- tests/test_streaming.cpp | 255 +++++++++++++++++++++ tutorial/README.md | 4 + tutorial/run.py | 39 +++- tutorial/run_stream.py | 12 + 26 files changed, 1495 insertions(+), 68 deletions(-) create mode 100644 docs/realtime-contract-3-session-report.md create mode 100644 src/engine_order_events.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 57209dc..b8077f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,6 +80,7 @@ add_library(pineforge STATIC src/engine_lower_tf.cpp src/engine_metrics.cpp src/engine_orders.cpp + src/engine_order_events.cpp src/engine_path_resolve.cpp src/engine_report.cpp src/engine_risk.cpp diff --git a/README.md b/README.md index 2cfb740..bd6e17f 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ for the full tool catalog, request schemas, and env vars (`PINEFORGE_ALLOW_ANYWH - 🎯 **TradingView-exact.** 251 of 252 reference strategies match TV trade-for-trade. The lone outlier is a stress probe at the 1× margin boundary where TV's broker emulator is non-deterministic — engine is correct. **100 of 100** PineForge excellent vs PyneCore + PineTS on the public three-way benchmark (~167,000 TV trades; PyneCore: 85 of 100; PineTS indicator-only). - ⚡ **Microsecond-class.** Median **162× faster than PyneCore** across 99 commonly-timed strategies (full 41,307-bar OHLCV, magnifier-on hot loop; see [benchmarks/results/speed.md](benchmarks/results/speed.md)). Parameter sweeps load one `.so` and re-run with new inputs — no recompile, no fork, no IPC. -- 🔒 **Stable C ABI.** 26 functions, one header (``). Append-only across minor versions, `static_assert`-pinned struct layouts, hidden-visibility hygiene. Drop a strategy `.so` in any harness; it just runs. +- 🔒 **Stable C ABI.** 27 functions, one header (``). Append-only across minor versions, `static_assert`-pinned struct layouts, hidden-visibility hygiene. Drop a strategy `.so` in any harness; it just runs. - 🧪 **Reproducible to the bit.** Deterministic float ordering, deterministic bar magnifier, no internal RNG seeded from time. Two runs with the same inputs produce bit-identical trade lists. - 🧰 **FFI-friendly.** Call from Python (`ctypes`), Rust (`libloading`), Go (`cgo`), Node, Julia. Worked examples for [pure C](https://cdocs.pineforge.dev/examples_c.html), [Python sweep](https://cdocs.pineforge.dev/examples_python_sweep.html), [Rust](https://cdocs.pineforge.dev/examples_rust.html), [multi-strategy harness](https://cdocs.pineforge.dev/examples_multi.html), and [magnifier A/B](https://cdocs.pineforge.dev/examples_magnifier.html) ship in the docs. - 🌍 **Cross-platform CI.** Linux + macOS × Release + Debug. Universal mac binary. Static library, no runtime DSO surprises at deploy time. @@ -324,6 +324,7 @@ int main(void) { | `strategy_set_trace_enabled` | Toggle per-bar trace recording | | `strategy_set_trade_start_time` | Suppress historical order placement | | `strategy_stream_begin` | Warm on OHLCV, then enter realtime mode | +| `strategy_stream_set_gap_policy` | Select fixed-grid or data-driven quiet bars | | `strategy_stream_push_tick` | Push one normalized ordered trade | | `strategy_stream_push_ticks` | Push a raw-trade batch | | `strategy_stream_advance_time` | Confirm bars / materialize quiet intervals | @@ -341,7 +342,7 @@ int main(void) { | `pf_version_string` | Full runtime version string | -POD types (`pf_bar_t`, `pf_trade_tick_t`, `pf_trade_t`, `pf_report_t`, `pf_security_diag_t`, `pf_trace_entry_t`, `pf_version_t`, and — since ABI v2 — `pf_trade_stats_t`, `pf_equity_stats_t`, `pf_metrics_t`, `pf_equity_point_t`) and the `pf_magnifier_distribution_t` enum complete the surface. ABI v2 (`PF_ABI_VERSION == 2`, exported as `pf_abi_version()`) appends computed trading metrics and a per-script-bar equity curve to `pf_report_t`; callers must verify the version before running since the report struct is caller-allocated. +POD types (`pf_bar_t`, `pf_trade_tick_t`, `pf_trade_t`, `pf_report_t`, `pf_order_event_t`, diagnostics, metrics, and version records) plus magnifier and stream-gap enums complete the surface. ABI v3 (`PF_ABI_VERSION == 3`, exported as `pf_abi_version()`) appends deterministic order lifecycle events and their rolling count/hash; callers must verify the version before running since the report struct is caller-allocated. **Stability guarantee:** within the same `PINEFORGE_VERSION_MAJOR`, struct layouts and `extern "C"` signatures are append-only. New fields may be appended; existing fields are never reordered, removed, or retyped. New functions may be added; existing functions are never removed or signature-changed. Compile-time `static_assert`s in `src/c_abi.cpp` pin the layouts against drift. diff --git a/docker/run_json.py b/docker/run_json.py index 5f7d8eb..7b23fc6 100755 --- a/docker/run_json.py +++ b/docker/run_json.py @@ -401,6 +401,38 @@ class TraceEntryC(ctypes.Structure): ] +class OrderEventC(ctypes.Structure): + _fields_ = [ + ("transition_sequence", ctypes.c_uint64), + ("command_revision_id", ctypes.c_uint64), + ("order_leg_id", ctypes.c_uint64), + ("priority_sequence", ctypes.c_uint64), + ("fill_id", ctypes.c_uint64), + ("entry_lot_id", ctypes.c_uint64), + ("position_episode_id", ctypes.c_uint64), + ("event_timestamp", ctypes.c_int64), + ("event_sequence", ctypes.c_uint64), + ("input_bar_index", ctypes.c_int64), + ("script_bar_index", ctypes.c_int32), + ("command_kind", ctypes.c_int32), ("leg_kind", ctypes.c_int32), + ("state_before", ctypes.c_int32), ("state_after", ctypes.c_int32), + ("transition", ctypes.c_int32), ("reason", ctypes.c_int32), + ("side", ctypes.c_int32), ("oca_type", ctypes.c_int32), + ("requested_quantity", ctypes.c_double), + ("remaining_quantity", ctypes.c_double), + ("filled_quantity", ctypes.c_double), + ("observed_price", ctypes.c_double), ("stop_price", ctypes.c_double), + ("limit_price", ctypes.c_double), + ("trail_activation_price", ctypes.c_double), + ("trail_watermark", ctypes.c_double), ("fill_price", ctypes.c_double), + ("position_size_before", ctypes.c_double), + ("position_size_after", ctypes.c_double), + ("equity_before", ctypes.c_double), ("equity_after", ctypes.c_double), + ("id", ctypes.c_char_p), ("from_entry", ctypes.c_char_p), + ("oca_name", ctypes.c_char_p), + ] + + class ReportC(ctypes.Structure): _fields_ = [ ("total_trades", ctypes.c_int), @@ -428,6 +460,11 @@ class ReportC(ctypes.Structure): ("metrics", MetricsC), ("equity_curve", ctypes.POINTER(EquityPointC)), ("equity_curve_len", ctypes.c_int64), # int64, NOT c_int + ("order_events", ctypes.POINTER(OrderEventC)), + ("order_events_len", ctypes.c_int64), + ("order_event_count", ctypes.c_uint64), + ("order_event_hash", ctypes.c_uint64), + ("order_event_dropped", ctypes.c_uint64), ] @@ -456,7 +493,7 @@ def engine_version(lib: ctypes.CDLL) -> dict: # pf_report_t is CALLER-allocated: a .so built against a different ABI # writes past (or short of) our ReportC buffer. Assert version up front. -EXPECTED_PF_ABI = 2 +EXPECTED_PF_ABI = 3 def check_abi(lib: ctypes.CDLL) -> None: @@ -650,6 +687,31 @@ def build_report_dict(report: ReportC, ohlcv_path: Path, "open_profit": _num(p.open_profit), }) + order_events = [] + for i in range(int(report.order_events_len)): + e = report.order_events[i] + text = lambda p: p.decode("utf-8", "replace") if p else "" + order_events.append({ + "transition_sequence": int(e.transition_sequence), + "command_revision_id": int(e.command_revision_id), + "order_leg_id": int(e.order_leg_id), + "priority_sequence": int(e.priority_sequence), + "fill_id": int(e.fill_id), + "entry_lot_id": int(e.entry_lot_id), + "position_episode_id": int(e.position_episode_id), + "event_timestamp": int(e.event_timestamp), + "event_sequence": int(e.event_sequence), + "script_bar_index": int(e.script_bar_index), + "command_kind": int(e.command_kind), "leg_kind": int(e.leg_kind), + "state_before": int(e.state_before), "state_after": int(e.state_after), + "transition": int(e.transition), "reason": int(e.reason), + "id": text(e.id), "from_entry": text(e.from_entry), + "oca_name": text(e.oca_name), + "filled_quantity": _num(e.filled_quantity), + "observed_price": _num(e.observed_price), + "fill_price": _num(e.fill_price), + }) + return { "engine": "pineforge", "input": { @@ -682,10 +744,14 @@ def build_report_dict(report: ReportC, ohlcv_path: Path, "magnifier_sub_bars_total": int(report.magnifier_sub_bars_total), "magnifier_sample_ticks_total": int(report.magnifier_sample_ticks_total), "bar_magnifier_enabled": bool(report.bar_magnifier_enabled), + "order_event_count": int(report.order_event_count), + "order_event_hash": f"{int(report.order_event_hash):016x}", + "order_event_dropped": int(report.order_event_dropped), }, "trades": trades, "metrics": metrics, "equity_curve": equity_curve, + "order_events": order_events, } diff --git a/docs/design/realtime-execution-audit.md b/docs/design/realtime-execution-audit.md index c8fa286..f892210 100644 --- a/docs/design/realtime-execution-audit.md +++ b/docs/design/realtime-execution-audit.md @@ -1,12 +1,33 @@ # Realtime execution contract: audit and development plan -Status: design audit; implementation is not approved by this document alone +Status: implementation in progress on the draft PR; the completion gate below +remains authoritative Date: 2026-07-11 (revised same day after a fourth, multi-perspective review pass; see the review disposition section) Worktree branch: `codex/realtime-execution-contract` +Implemented in the first contract slice: + +- close-only profile rejection before warmup for every-tick, + every-history-tick, and order-fill recalculation modes; +- fixed-grid and data-driven gap policies, with clock events forbidden from + producing price-contingent broker transitions; +- exact last-trade process-on-close provenance; +- durable cross-event realtime stop-limit activation and per-order realtime + trailing activation/watermarks; +- immutable command-revision, order-leg, fill, and entry-lot identities; +- bounded structured lifecycle diagnostics with an overflow-independent + canonical count/hash, ABI v3, Python mirrors, docs, and tutorial output; +- realtime OCA exclusion by exact order-leg identity while retaining the + historical public-ID behavior. + +This slice does not close the completion gate: P1-P9 exports, full per-lot exit +fan-out, a canonical whole-engine handoff digest, and the frozen three-session +pre/post experiment remain explicit follow-up work and must not be represented +as complete by the draft PR. + ## Decision summary PineForge should implement a deterministic historical-to-live simulator with a diff --git a/docs/pages/ffi-python.md b/docs/pages/ffi-python.md index bca0275..73faeee 100644 --- a/docs/pages/ffi-python.md +++ b/docs/pages/ffi-python.md @@ -112,6 +112,28 @@ class pf_trace_entry_t(ctypes.Structure): ("value", ctypes.c_double), ] +class pf_order_event_t(ctypes.Structure): + _fields_ = [ + ("transition_sequence", ctypes.c_uint64), + ("command_revision_id", ctypes.c_uint64), ("order_leg_id", ctypes.c_uint64), + ("priority_sequence", ctypes.c_uint64), ("fill_id", ctypes.c_uint64), + ("entry_lot_id", ctypes.c_uint64), ("position_episode_id", ctypes.c_uint64), + ("event_timestamp", ctypes.c_int64), ("event_sequence", ctypes.c_uint64), + ("input_bar_index", ctypes.c_int64), ("script_bar_index", ctypes.c_int32), + ("command_kind", ctypes.c_int32), ("leg_kind", ctypes.c_int32), + ("state_before", ctypes.c_int32), ("state_after", ctypes.c_int32), + ("transition", ctypes.c_int32), ("reason", ctypes.c_int32), + ("side", ctypes.c_int32), ("oca_type", ctypes.c_int32), + ("requested_quantity", ctypes.c_double), ("remaining_quantity", ctypes.c_double), + ("filled_quantity", ctypes.c_double), ("observed_price", ctypes.c_double), + ("stop_price", ctypes.c_double), ("limit_price", ctypes.c_double), + ("trail_activation_price", ctypes.c_double), ("trail_watermark", ctypes.c_double), + ("fill_price", ctypes.c_double), ("position_size_before", ctypes.c_double), + ("position_size_after", ctypes.c_double), ("equity_before", ctypes.c_double), + ("equity_after", ctypes.c_double), ("id", ctypes.c_char_p), + ("from_entry", ctypes.c_char_p), ("oca_name", ctypes.c_char_p), + ] + class pf_report_t(ctypes.Structure): _fields_ = [ ("total_trades", ctypes.c_int), @@ -147,6 +169,12 @@ class pf_report_t(ctypes.Structure): ("metrics", pf_metrics_t), ("equity_curve", ctypes.POINTER(pf_equity_point_t)), ("equity_curve_len", ctypes.c_int64), # int64 in the C header, NOT c_int + # ABI v3: deterministic order lifecycle transitions + ("order_events", ctypes.POINTER(pf_order_event_t)), + ("order_events_len", ctypes.c_int64), + ("order_event_count", ctypes.c_uint64), + ("order_event_hash", ctypes.c_uint64), + ("order_event_dropped", ctypes.c_uint64), ] class pf_version_t(ctypes.Structure): @@ -175,9 +203,9 @@ itself. Open it with `ctypes.CDLL`: lib = ctypes.CDLL("./my_strategy.so") # ABI guard — pf_report_t is CALLER-allocated, so running an old .so -# against the v2 mirror above (or vice versa) silently corrupts memory. +# against the v3 mirror above (or vice versa) silently corrupts memory. # Verify the .so's layout version before any run: -EXPECTED_PF_ABI = 2 # PF_ABI_VERSION in +EXPECTED_PF_ABI = 3 # PF_ABI_VERSION in try: lib.pf_abi_version.restype = ctypes.c_int abi = lib.pf_abi_version() diff --git a/docs/pages/index.md b/docs/pages/index.md index f205504..cf9ac84 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -80,18 +80,18 @@ End-to-end, runnable examples that go beyond the MACD tutorial: ## API at a glance -The entire public surface fits in **one header** and **26 functions**: +The entire public surface fits in **one header** and **27 functions**: | Group | Symbols | Reference | | --- | --- | --- | | Lifecycle | `strategy_create`, `strategy_free`, `run_backtest`, `run_backtest_full`, `report_free` | @ref pf_lifecycle | -| Streaming | `strategy_stream_begin`, `strategy_stream_push_tick`, `strategy_stream_push_ticks`, `strategy_stream_advance_time`, `strategy_stream_end`, `strategy_stream_fill_report` | @ref pf_streaming | +| Streaming | `strategy_stream_set_gap_policy`, `strategy_stream_begin`, `strategy_stream_push_tick`, `strategy_stream_push_ticks`, `strategy_stream_advance_time`, `strategy_stream_end`, `strategy_stream_fill_report` | @ref pf_streaming | | Configuration | Inputs, strategy overrides, tracing, trade start, chart / symbol timezone, session, tick size, point value, and numeric metadata | @ref pf_config | | Diagnostics | `strategy_get_last_error` | #strategy_get_last_error | | Version | `pf_version_get`, `pf_abi_version`, `pf_version_string` | @ref pf_version | | Types | `pf_bar_t`, `pf_trade_tick_t`, `pf_trade_t`, `pf_report_t`, metrics, diagnostics, trace, equity, version, and `pf_magnifier_distribution_t` | @ref pf_types | -Every PineForge-generated strategy `.so` exports exactly these 26 symbols +Every PineForge-generated strategy `.so` exports exactly these 27 symbols and zero internal C++ symbols — see **[ABI stability](@ref abi_stability)** for the full guarantee. diff --git a/docs/pages/report-schema.md b/docs/pages/report-schema.md index c370d04..94557d1 100644 --- a/docs/pages/report-schema.md +++ b/docs/pages/report-schema.md @@ -53,12 +53,19 @@ typedef struct pf_report_s { /* Per-script-bar equity curve (ABI v2) */ pf_equity_point_t* equity_curve; int64_t equity_curve_len; /* NOTE: int64, not int */ + + /* ABI v3: deterministic broker lifecycle diagnostics */ + pf_order_event_t* order_events; + int64_t order_events_len; + uint64_t order_event_count; + uint64_t order_event_hash; + uint64_t order_event_dropped; } pf_report_t; ``` -@note The `metrics` / `equity_curve` fields were appended in **ABI -version 2** (`PF_ABI_VERSION`). `pf_report_t` is caller-allocated, so -consumers must check `pf_abi_version() == 2` before running — a `.so` +@note Metrics/equity fields were appended in ABI v2 and lifecycle fields in +**ABI version 3** (`PF_ABI_VERSION`). `pf_report_t` is caller-allocated, so +consumers must check `pf_abi_version() == 3` before running — a `.so` with no `pf_abi_version` symbol is ABI v1 and predates these fields. ## Trade fields @@ -227,6 +234,19 @@ exception mid-run can truncate the curve — check `strategy_get_last_error`). The array is heap-allocated and freed by #report_free. Note the length field is `int64_t`, not `int`. +## Order lifecycle (ABI v3) + +`order_events` retains structured simulator transitions with command revision, +order leg, priority, fill, entry-lot and position-episode identities; market +provenance; state/transition/reason codes; quantities/prices; position/equity +before and after; and deep-copied Pine/OCA strings. #report_free releases the +array and embedded strings. + +`transition_sequence` is the canonical order. Provenance timestamps may be +non-monotonic for retained process-on-close events. The rolling count/hash +include transitions beyond retained capacity; check `order_event_dropped` +before expecting the retained arrays to be complete. + ## Lifetime and ownership Every heap pointer in `pf_report_t` is freed by a single call to diff --git a/docs/pages/streaming.md b/docs/pages/streaming.md index a679d74..59d0c8a 100644 --- a/docs/pages/streaming.md +++ b/docs/pages/streaming.md @@ -10,7 +10,8 @@ partially formed higher-timeframe candle. This is the runtime model used by a continuously running strategy: -1. Call #strategy_stream_begin with every confirmed historical input bar. +1. Optionally select a gap policy with #strategy_stream_set_gap_policy, then + call #strategy_stream_begin with every confirmed historical input bar. 2. Feed normalized trades with #strategy_stream_push_tick, or pass one contiguous replay tape to #strategy_stream_push_ticks. 3. Call #strategy_stream_advance_time at wall-clock boundaries, including @@ -51,6 +52,9 @@ remain outside the runtime. ```c pf_strategy_t strategy = strategy_create(NULL); +if (strategy_stream_set_gap_policy(strategy, PF_STREAM_GAP_FIXED_GRID) != 0) + fail(strategy_get_last_error(strategy)); + if (strategy_stream_begin(strategy, history, history_n, "1", "1") != 0) fail(strategy_get_last_error(strategy)); @@ -97,9 +101,34 @@ Every stream function returns `0` on success and `-1` on failure. Read - A market order created at the preceding close fills at the first subsequent trade. Stops and limits see the observed trade path, not an inferred OHLC traversal. -- #strategy_stream_advance_time materializes elapsed quiet in-session intervals - as zero-volume carry-forward bars. Intervals outside the configured syminfo - session are skipped, so closed markets do not acquire synthetic bars. +- Fixed-grid policy materializes elapsed quiet in-session intervals as + zero-volume carry-forward bars. Strategy code runs, but the clock supplies + no executable price: it cannot fill orders, ratchet trails, update + excursions, or trigger price-based margin actions. +- Data-driven policy skips tickless intervals. Both policies skip configured + closed-session intervals. +- Process-on-close can use only the retained last executed trade of a non-empty + bar. Its fill timestamp/sequence are that trade's provenance. +- Stop-limit activation survives across trade events. Realtime trailing + activation and watermarks are stored per executable order leg. + +## Calculation profile gate + +Realtime v1 accepts only Pine's default close-only profile. `stream_begin` +rejects `calc_on_every_tick`, `calc_on_every_history_tick`, and +`calc_on_order_fills` before warmup mutates the handle. Exact trade events are +not reported as historical bar-magnifier execution. + +## Order lifecycle diagnostics + +Command revisions and executable legs receive immutable numeric identities; +public Pine IDs can repeat and broker priority is separate. The report's +`order_events` records structured lifecycle transitions. Processing order +(`transition_sequence`) is canonical; timestamp/sequence are provenance only. + +The rolling `order_event_count` and `order_event_hash` cover every transition +even if retained capacity overflows (`order_event_dropped`). Webhook transports +belong above this API and should deduplicate fill delivery by `fill_id`. ## One tick versus a contiguous replay @@ -123,9 +152,11 @@ warmup plus realtime result. The report follows the normal ownership rules in ## Current calculation scope The first streaming release implements TradingView's default close-only -strategy cadence and raw-trade broker fills. `calc_on_every_tick`, -`calc_on_order_fills`, realtime rollback/`varip`, and alert delivery are -separate surfaces and are not implied by using this lifecycle. +strategy cadence and raw-trade broker fills. It is a deterministic PineForge +contract, not a claim to reproduce TradingView's private feed or realtime +executor. Exchange queues/liquidity, partial tape-quantity fills, latency, +reconnect/persistence, broker routing, rollback/`varip`, and alert delivery are +separate surfaces. See [Lifecycle](@ref lifecycle) for handle ownership and [FFI from Python](@ref ffi_python) for the complete POD mirrors. diff --git a/docs/realtime-contract-3-session-report.md b/docs/realtime-contract-3-session-report.md new file mode 100644 index 0000000..96541da --- /dev/null +++ b/docs/realtime-contract-3-session-report.md @@ -0,0 +1,68 @@ +# Realtime contract three-session corpus report + +Run date: 2026-07-11 + +This report compares a bar-only run with same-instance OHLCV warmup plus one +contiguous normalized ETHUSDT-perpetual trade tape. Every session ends +exclusively at `2025-05-01T00:00:00Z`; no artificial chunks or short sessions +were used. + +## Inputs + +- Corpus: 252 compiled validation probes, three starts (756 runs). +- Starts: `2025-02-07T03:17Z`, `2025-03-11T14:23Z`, + `2025-03-29T22:41Z`. +- Tape: 424,801,937 records, 13,593,661,984 bytes, 32-byte + `pf_trade_tick_t` records. +- Runtime: 293.59 seconds including warmup, all stream runs, bar baselines, + scoring, and report generation (tape construction excluded). +- Branch base at experiment time: `6d6e369`; the final implementation commit is + recorded by draft PR #90. + +## Scores + +| Handoff | Runtime | Input bars | Script bars | Trade count exact | Fully structural | Weighted ordered match | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 2025-02-07 03:17 | 252/252 | 252/252 | 252/252 | 241/252 | 228/252 | 99.0363% | +| 2025-03-11 14:23 | 252/252 | 252/252 | 252/252 | 242/252 | 234/252 | 99.3784% | +| 2025-03-29 22:41 | 252/252 | 252/252 | 252/252 | 243/252 | 235/252 | 99.5667% | + +`Fully structural` requires exact trade count and exact ordered structural +signature. Weighted ordered match uses direction, entry/exit minute, entry/exit +bar index, and quantity, with the denominator equal to the larger trade list. + +Across probes, the median p50/p90/p99 entry and exit price delta is zero for +all sessions. At the 90th percentile probe, the per-probe p99 deltas were: + +| Handoff | Entry p99 (bps) | Exit p99 (bps) | +| --- | ---: | ---: | +| 2025-02-07 03:17 | 0.0935 | 0.1595 | +| 2025-03-11 14:23 | 0.0612 | 0.0633 | +| 2025-03-29 22:41 | 0.0559 | 0.0601 | + +Absolute summed P&L differences across all 252 probes were $2,339.34, +$1,500.51, and $1,051.71 respectively. These are reporting aggregates, not a +portfolio P&L, because every probe has its own independent account. + +## Determinism and diagnostics + +All 252 runs in every session emitted a canonical lifecycle hash. Total +lifecycle transitions were 4,984,478 / 3,112,033 / 1,966,731. Retained event +capacity overflowed for active probes (1,042,018 / 428,753 / 94,124 dropped +records), but the rolling count/hash includes dropped records by contract. + +The machine-readable report is generated at +`build/realtime_contract_3_sessions.json`. It includes per-probe price/P&L +distributions, first divergence, and retained lifecycle context. + +## Interpretation and remaining gate items + +The hard runtime and bar-continuity gates pass. Tick execution is expected to +differ slightly from inferred bar paths; the high ordered-match percentages +show that divergence is concentrated. This run is not a claim of exact +TradingView realtime parity. + +The revised design's full completion gate remains open: a frozen pre-change +run with the same scorer, TradingView probes P1-P9, per-lot exit fan-out, the +whole-engine handoff digest, and a deterministic second execution of all 756 +sessions are not supplied by this post-change report. diff --git a/include/pineforge/engine.hpp b/include/pineforge/engine.hpp index 72afec9..b2b7c85 100644 --- a/include/pineforge/engine.hpp +++ b/include/pineforge/engine.hpp @@ -53,6 +53,7 @@ struct PyramidEntry { // the full bar). bool skip_entry_bar_high = false; bool skip_entry_bar_low = false; + uint64_t entry_lot_id = 0; }; struct Trade { @@ -116,6 +117,30 @@ struct TraceEntryC { double value; }; +enum class OrderLifecycleState : int32_t { + NONE = 0, PENDING_MARKET = 1, PENDING_LIMIT = 2, PENDING_STOP = 3, + PENDING_STOP_LIMIT = 4, ACTIVE_STOP_LIMIT = 5, + PENDING_TRAIL_ACTIVATION = 6, ACTIVE_TRAIL = 7, + FILLED = 8, CANCELLED = 9, REPLACED = 10, REJECTED = 11, EXPIRED = 12 +}; +enum class OrderTransition : int32_t { + CREATED = 1, REPLACED = 2, ACTIVATED = 3, RATCHETED = 4, + FILLED = 5, CANCELLED = 6, REDUCED = 7, REJECTED = 8, EXPIRED = 9 +}; +enum class OrderTransitionReason : int32_t { + NONE = 0, COMMAND_REISSUE = 1, USER_CANCEL = 2, + STOP_LIMIT_TRIGGER = 3, TRAIL_TRIGGER = 4, TRAIL_FAVORABLE_PRICE = 5, + PRICE_FILL = 6, RISK_REJECT = 7, OCA_EFFECT = 8, FLAT_PURGE = 9, + STREAM_CLOSE_EVENT = 10 +}; + +struct OrderLifecycleEvent { + pf_order_event_t value{}; + std::string id; + std::string from_entry; + std::string oca_name; +}; + struct ReportC { int total_trades; TradeC* trades; @@ -149,6 +174,11 @@ struct ReportC { pf_metrics_t metrics; pf_equity_point_t* equity_curve; int64_t equity_curve_len; + pf_order_event_t* order_events; + int64_t order_events_len; + uint64_t order_event_count; + uint64_t order_event_hash; + uint64_t order_event_dropped; }; enum class OrderType { MARKET, ENTRY, EXIT, RAW_ORDER }; @@ -176,6 +206,19 @@ struct PendingOrder { int oca_type; // 0=none, 1=cancel, 2=reduce int created_bar; // bar_index when order was created int64_t created_seq = 0; + // Stable broker identities are distinct from the public Pine ID and from + // priority. Reissuing the same command key creates a fresh revision and + // leg while preserving or resetting priority according to created_seq. + uint64_t command_revision_id = 0; + uint64_t order_leg_id = 0; + uint64_t priority_sequence = 0; + uint64_t terminal_fill_id = 0; + // Realtime trailing state is executable-leg state, never position-global. + // The historical OHLC kernel continues to use trail_best_price_ until its + // path semantics can be refactored without corpus drift. + double realtime_trail_best_price = + std::numeric_limits::quiet_NaN(); + bool realtime_trail_activated = false; // Entry stop-limit activation is durable broker state. Once the stop leg // fires, later bars—and later COOF scheduler segments on the same bar— // evaluate only the live limit leg until the order fills or is replaced. @@ -380,6 +423,8 @@ struct StrategyOverrides { int default_qty_type = -1; int process_orders_on_close = -1; int calc_on_order_fills = -1; + int calc_on_every_tick = -1; + int calc_on_every_history_tick = -1; int close_entries_rule = -1; }; @@ -469,6 +514,11 @@ class BacktestEngine { // Historical fill-triggered recalculation is strictly opt-in. The false // branch in dispatch_bar remains the legacy control path. bool calc_on_order_fills_ = false; + // Realtime execution profiles beyond close-only are deliberately explicit. + // Codegen may set these directly from strategy() declarations; stream_begin + // rejects them before warmup until their rollback schedulers exist. + bool calc_on_every_tick_ = false; + bool calc_on_every_history_tick_ = false; QtyType default_qty_type_ = QtyType::FIXED; double default_qty_value_ = 1.0; int pyramiding_ = 1; // max additional entries in same direction @@ -1355,6 +1405,17 @@ class BacktestEngine { // finite historical/magnifier event budget as every other broker fill. uint64_t coof_direct_fill_events_remaining_ = 0; uint64_t broker_fill_event_seq_ = 0; + uint64_t next_command_revision_id_ = 1; + uint64_t next_order_leg_id_ = 1; + uint64_t next_entry_lot_id_ = 1; + uint64_t next_transition_sequence_ = 1; + uint64_t position_episode_id_ = 0; + std::vector order_events_; + uint64_t order_event_count_ = 0; + uint64_t order_event_hash_ = 1469598103934665603ULL; + uint64_t order_event_dropped_ = 0; + bool order_event_recording_enabled_ = false; + static constexpr std::size_t kOrderEventRetentionCapacity = 65536; // input.source histories are base-owned script state and must roll back // with generated state between historical fill recalculations. @@ -1394,7 +1455,9 @@ class BacktestEngine { // path exactly once, then these fields carry the SAME broker, Pine series, // TA and timeframe-aggregator state forward while normalized trades arrive. enum class StreamPhase { IDLE, REALTIME, ENDED }; + enum class StreamGapPolicy { FIXED_GRID = 0, DATA_DRIVEN = 1 }; StreamPhase stream_phase_ = StreamPhase::IDLE; + StreamGapPolicy stream_gap_policy_ = StreamGapPolicy::FIXED_GRID; bool stream_warmup_mode_ = false; int64_t stream_input_tf_ms_ = 0; int64_t stream_next_input_open_ms_ = 0; @@ -1404,10 +1467,14 @@ class BacktestEngine { bool stream_seen_sequence_ = false; bool stream_has_input_bar_ = false; Bar stream_input_bar_{}; + int64_t stream_input_last_trade_ms_ = 0; + uint64_t stream_input_last_trade_sequence_ = 0; double stream_last_price_ = 0.0; bool stream_has_last_price_ = false; int stream_next_script_bar_index_ = 0; bool stream_script_bar_had_tick_ = false; + int64_t stream_script_last_trade_ms_ = 0; + uint64_t stream_script_last_trade_sequence_ = 0; bool stream_script_tick_seen_ = false; // --- request.security state --- @@ -1771,11 +1838,14 @@ class BacktestEngine { void execute_partial_exit(double fill_price, double qty_percent); void execute_partial_exit_by_entry(double fill_price, const std::string& from_entry); void execute_partial_exit_by_entry_percent(double fill_price, const std::string& from_entry, double qty_percent); - void cancel_oca_group(const std::string& oca_name, const std::string& exclude_id); + void cancel_oca_group(const std::string& oca_name, int oca_type, + const std::string& exclude_id, + uint64_t exclude_order_leg_id); // Pine v6 oca.reduce: when one sibling fills qty Q, reduce remaining // siblings' qty by Q. Siblings whose qty becomes <= 0 are cancelled. - void reduce_oca_group(const std::string& oca_name, const std::string& exclude_id, - double filled_qty); + void reduce_oca_group(const std::string& oca_name, int oca_type, + const std::string& exclude_id, + uint64_t exclude_order_leg_id, double filled_qty); void purge_exit_orders(bool retain_for_pending_entries = false); // process_pending_orders helpers (defined in engine_fills.cpp). @@ -1834,6 +1904,16 @@ class BacktestEngine { int& exit_closed_from_bar, bool& exit_closed_was_long); void materialize_relative_exit_prices_for_live_position(); + void assign_pending_order_identity(PendingOrder& order); + void assign_entry_lot_identity(PyramidEntry& entry); + OrderLifecycleState lifecycle_state(const PendingOrder& order) const; + void record_order_transition( + const PendingOrder& order, OrderLifecycleState before, + OrderLifecycleState after, OrderTransition transition, + OrderTransitionReason reason, double filled_quantity = 0.0, + double fill_price = std::numeric_limits::quiet_NaN(), + double position_before = std::numeric_limits::quiet_NaN(), + double equity_before = std::numeric_limits::quiet_NaN()); // Inner-loop phase split for process_pending_orders. // The inner loop iterates `pending_orders_` and processes each via @@ -1987,14 +2067,19 @@ class BacktestEngine { void run_aggregation_bar_loop(const Bar* input_bars, int n_input, bool bar_magnifier, int expected_script_bars); bool stream_finalize_until(int64_t timestamp_ms); - void stream_feed_input_bar(const Bar& bar, bool had_tick); - void stream_dispatch_script_bar(const Bar& bar, bool had_tick); + void stream_feed_input_bar(const Bar& bar, bool had_tick, + int64_t last_trade_ms, + uint64_t last_trade_sequence); + void stream_dispatch_script_bar(const Bar& bar, bool had_tick, + int64_t last_trade_ms, + uint64_t last_trade_sequence); // fill_report helpers (defined in engine_report.cpp). void fill_trades_section(ReportC* out) const; void fill_metrics_section(ReportC* out) const; void fill_security_diag_section(ReportC* out) const; void fill_trace_section(ReportC* out) const; + void fill_order_events_section(ReportC* out) const; public: virtual ~BacktestEngine() = default; @@ -2018,6 +2103,7 @@ class BacktestEngine { bool stream_begin(const Bar* warmup_bars, int n_warmup, const std::string& input_tf, const std::string& script_tf = ""); + bool stream_set_gap_policy(int policy); bool stream_push_tick(const TradeTick& tick); bool stream_push_ticks(const TradeTick* ticks, int n); bool stream_advance_time(int64_t timestamp_ms); diff --git a/include/pineforge/pineforge.h b/include/pineforge/pineforge.h index a7c4ed8..63244c5 100644 --- a/include/pineforge/pineforge.h +++ b/include/pineforge/pineforge.h @@ -75,7 +75,7 @@ * versioned layout (metrics + equity curve); .so files predating this * macro have no pf_abi_version symbol — treat dlsym failure as * version 1. */ -#define PF_ABI_VERSION 2 +#define PF_ABI_VERSION 3 #ifdef __cplusplus extern "C" { @@ -127,6 +127,11 @@ typedef struct pf_trade_tick_s { double quantity; /**< Traded quantity in symbol volume units (>= 0). */ } pf_trade_tick_t; +typedef enum pf_stream_gap_policy_e { + PF_STREAM_GAP_FIXED_GRID = 0, + PF_STREAM_GAP_DATA_DRIVEN = 1 +} pf_stream_gap_policy_t; + /** Closed-trade record returned in pf_report_t::trades. * * Layout-compatible with internal `pineforge::TradeC`. */ @@ -296,6 +301,48 @@ typedef struct pf_trace_entry_s { double value; /**< Traced expression value on this bar. */ } pf_trace_entry_t; +/** Deterministic broker lifecycle transition. String fields are deep-copied + * into the report snapshot and released by #report_free. Processing order is + * authoritative; event timestamp/sequence are provenance and may be older + * than a preceding transition for a retained process-on-close event. */ +typedef struct pf_order_event_s { + uint64_t transition_sequence; + uint64_t command_revision_id; + uint64_t order_leg_id; + uint64_t priority_sequence; + uint64_t fill_id; + uint64_t entry_lot_id; + uint64_t position_episode_id; + int64_t event_timestamp; + uint64_t event_sequence; + int64_t input_bar_index; + int32_t script_bar_index; + int32_t command_kind; + int32_t leg_kind; + int32_t state_before; + int32_t state_after; + int32_t transition; + int32_t reason; + int32_t side; + int32_t oca_type; + double requested_quantity; + double remaining_quantity; + double filled_quantity; + double observed_price; + double stop_price; + double limit_price; + double trail_activation_price; + double trail_watermark; + double fill_price; + double position_size_before; + double position_size_after; + double equity_before; + double equity_after; + char* id; + char* from_entry; + char* oca_name; +} pf_order_event_t; + /** Backtest report filled by #run_backtest / #run_backtest_full. * * Layout-compatible with internal `pineforge::ReportC`. @@ -303,7 +350,7 @@ typedef struct pf_trace_entry_s { * ### Ownership and lifetime * The struct itself is caller-owned (typically stack). The embedded * arrays (`trades`, `security_diag`, `trace`, `trace_names`, - * `equity_curve`) are heap-allocated by the runtime; the caller must + * `equity_curve`, `order_events`) are heap-allocated by the runtime; the caller must * invoke #report_free exactly once on each filled report. * `trace_names` string pointers remain owned by the strategy handle * until #strategy_free. */ @@ -359,6 +406,14 @@ typedef struct pf_report_s { * (ctypes: c_int64). */ pf_equity_point_t* equity_curve; int64_t equity_curve_len; + + /* Canonical order lifecycle diagnostics. The rolling count/hash covers + * every transition even when retained capacity overflows. */ + pf_order_event_t* order_events; + int64_t order_events_len; + uint64_t order_event_count; + uint64_t order_event_hash; + uint64_t order_event_dropped; } pf_report_t; /** @} */ /* end of pf_types */ @@ -528,6 +583,12 @@ PF_API int strategy_stream_begin(pf_strategy_t s, const char* input_tf, const char* script_tf); +/** Select how quiet in-session intervals are normalized. Must be called before + * strategy_stream_begin. Fixed-grid (default) emits zero-volume carry bars; + * data-driven confirms only intervals that observed at least one trade. */ +PF_API int strategy_stream_set_gap_policy(pf_strategy_t s, + pf_stream_gap_policy_t policy); + /** Push one normalized realtime trade. Returns 0 on success, -1 on failure. */ PF_API int strategy_stream_push_tick(pf_strategy_t s, const pf_trade_tick_t* tick); diff --git a/scripts/check_c_abi_runtime.py b/scripts/check_c_abi_runtime.py index 6631588..95cabd4 100644 --- a/scripts/check_c_abi_runtime.py +++ b/scripts/check_c_abi_runtime.py @@ -26,6 +26,7 @@ "strategy_set_syminfo_metadata", "strategy_get_last_error", "strategy_stream_begin", + "strategy_stream_set_gap_policy", "strategy_stream_push_tick", "strategy_stream_push_ticks", "strategy_stream_advance_time", diff --git a/scripts/run_strategy.py b/scripts/run_strategy.py index 255a25d..23657d6 100644 --- a/scripts/run_strategy.py +++ b/scripts/run_strategy.py @@ -406,6 +406,38 @@ class TraceEntryC(ctypes.Structure): ] +class OrderEventC(ctypes.Structure): + _fields_ = [ + ("transition_sequence", ctypes.c_uint64), + ("command_revision_id", ctypes.c_uint64), + ("order_leg_id", ctypes.c_uint64), + ("priority_sequence", ctypes.c_uint64), + ("fill_id", ctypes.c_uint64), + ("entry_lot_id", ctypes.c_uint64), + ("position_episode_id", ctypes.c_uint64), + ("event_timestamp", ctypes.c_int64), + ("event_sequence", ctypes.c_uint64), + ("input_bar_index", ctypes.c_int64), + ("script_bar_index", ctypes.c_int32), + ("command_kind", ctypes.c_int32), ("leg_kind", ctypes.c_int32), + ("state_before", ctypes.c_int32), ("state_after", ctypes.c_int32), + ("transition", ctypes.c_int32), ("reason", ctypes.c_int32), + ("side", ctypes.c_int32), ("oca_type", ctypes.c_int32), + ("requested_quantity", ctypes.c_double), + ("remaining_quantity", ctypes.c_double), + ("filled_quantity", ctypes.c_double), + ("observed_price", ctypes.c_double), ("stop_price", ctypes.c_double), + ("limit_price", ctypes.c_double), + ("trail_activation_price", ctypes.c_double), + ("trail_watermark", ctypes.c_double), ("fill_price", ctypes.c_double), + ("position_size_before", ctypes.c_double), + ("position_size_after", ctypes.c_double), + ("equity_before", ctypes.c_double), ("equity_after", ctypes.c_double), + ("id", ctypes.c_char_p), ("from_entry", ctypes.c_char_p), + ("oca_name", ctypes.c_char_p), + ] + + class ReportC(ctypes.Structure): _fields_ = [ ("total_trades", ctypes.c_int), @@ -433,6 +465,11 @@ class ReportC(ctypes.Structure): ("metrics", MetricsC), ("equity_curve", ctypes.POINTER(EquityPointC)), ("equity_curve_len", ctypes.c_int64), # int64 in the C header, NOT c_int + ("order_events", ctypes.POINTER(OrderEventC)), + ("order_events_len", ctypes.c_int64), + ("order_event_count", ctypes.c_uint64), + ("order_event_hash", ctypes.c_uint64), + ("order_event_dropped", ctypes.c_uint64), ] @@ -440,7 +477,7 @@ class ReportC(ctypes.Structure): # pf_report_t is CALLER-allocated: running an old .so against the v2 # ReportC mirror (or vice versa) silently corrupts memory, so the .so's # pf_abi_version() export is asserted before any run. -EXPECTED_PF_ABI = 2 +EXPECTED_PF_ABI = 3 def _check_abi(lib: ctypes.CDLL) -> None: @@ -617,6 +654,8 @@ def _setup_signatures(self) -> None: L.strategy_get_last_error.argtypes = [ctypes.c_void_p] L.strategy_get_last_error.restype = ctypes.c_char_p if hasattr(L, "strategy_stream_begin"): + L.strategy_stream_set_gap_policy.argtypes = [ctypes.c_void_p, ctypes.c_int] + L.strategy_stream_set_gap_policy.restype = ctypes.c_int L.strategy_stream_begin.argtypes = [ ctypes.c_void_p, ctypes.POINTER(BarC), ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p] @@ -943,6 +982,30 @@ def _report_to_dict(r: ReportC) -> dict: "name": name, "value": float(e.value), }) + order_events = [] + for i in range(r.order_events_len): + e = r.order_events[i] + text = lambda p: p.decode("utf-8", "replace") if p else "" + order_events.append({ + "transition_sequence": int(e.transition_sequence), + "command_revision_id": int(e.command_revision_id), + "order_leg_id": int(e.order_leg_id), + "priority_sequence": int(e.priority_sequence), + "fill_id": int(e.fill_id), + "entry_lot_id": int(e.entry_lot_id), + "position_episode_id": int(e.position_episode_id), + "event_timestamp": int(e.event_timestamp), + "event_sequence": int(e.event_sequence), + "script_bar_index": int(e.script_bar_index), + "command_kind": int(e.command_kind), "leg_kind": int(e.leg_kind), + "state_before": int(e.state_before), "state_after": int(e.state_after), + "transition": int(e.transition), "reason": int(e.reason), + "id": text(e.id), "from_entry": text(e.from_entry), + "oca_name": text(e.oca_name), + "filled_quantity": float(e.filled_quantity), + "observed_price": float(e.observed_price), + "fill_price": float(e.fill_price), + }) return { "total_trades": int(r.total_trades), "net_profit": float(r.net_profit), @@ -954,6 +1017,10 @@ def _report_to_dict(r: ReportC) -> dict: "trades": trades, "trace": trace, "trace_names": trace_names, + "order_events": order_events, + "order_event_count": int(r.order_event_count), + "order_event_hash": f"{int(r.order_event_hash):016x}", + "order_event_dropped": int(r.order_event_dropped), } diff --git a/scripts/run_stream_corpus.py b/scripts/run_stream_corpus.py index 9ca487f..828b994 100644 --- a/scripts/run_stream_corpus.py +++ b/scripts/run_stream_corpus.py @@ -271,6 +271,41 @@ def signature(trade: dict) -> tuple: autojunk=False, ) ordered_match = sum(block.size for block in matcher.get_matching_blocks()) + def percentile(values: list[float], q: float) -> float | None: + if not values: + return None + ordered = sorted(values) + pos = (len(ordered) - 1) * q + lo = int(pos) + hi = min(lo + 1, len(ordered) - 1) + return ordered[lo] + (ordered[hi] - ordered[lo]) * (pos - lo) + + entry_abs = [abs(a["entry_price"] - b["entry_price"]) + for a, b in zip(left, right)] + exit_abs = [abs(a["exit_price"] - b["exit_price"]) + for a, b in zip(left, right)] + entry_bps = [delta / max(abs(a["entry_price"]), 1e-12) * 10_000.0 + for delta, a in zip(entry_abs, left)] + exit_bps = [delta / max(abs(a["exit_price"]), 1e-12) * 10_000.0 + for delta, a in zip(exit_abs, left)] + first_divergence = None + for index, (a, b) in enumerate(zip(left, right)): + if signature(a) != signature(b): + cutoff = min(a["entry_time"], b["entry_time"]) + context = [event for event in stream.get("order_events", []) + if event["event_timestamp"] <= cutoff][-10:] + first_divergence = { + "index": index, "batch": a, "stream": b, + "order_event_context": context, + } + break + if first_divergence is None and len(left) != len(right): + first_divergence = { + "index": min(len(left), len(right)), + "batch": left[min(len(left), len(right))] if len(left) > len(right) else None, + "stream": right[min(len(left), len(right))] if len(right) > len(left) else None, + "order_event_context": stream.get("order_events", [])[-10:], + } batch_profit = float(batch["net_profit"]) stream_profit = float(stream["net_profit"]) return { @@ -287,6 +322,21 @@ def signature(trade: dict) -> tuple: "batch_net_profit": batch_profit, "stream_net_profit": stream_profit, "net_profit_delta": stream_profit - batch_profit, + "net_profit_relative_delta": ( + None if abs(batch_profit) <= 1e-12 + else (stream_profit - batch_profit) / abs(batch_profit)), + "entry_price_abs_delta": {key: percentile(entry_abs, q) for key, q in + (("p50", .50), ("p90", .90), ("p99", .99))}, + "exit_price_abs_delta": {key: percentile(exit_abs, q) for key, q in + (("p50", .50), ("p90", .90), ("p99", .99))}, + "entry_price_bps_delta": {key: percentile(entry_bps, q) for key, q in + (("p50", .50), ("p90", .90), ("p99", .99))}, + "exit_price_bps_delta": {key: percentile(exit_bps, q) for key, q in + (("p50", .50), ("p90", .90), ("p99", .99))}, + "first_divergence": first_divergence, + "lifecycle_event_count": stream.get("order_event_count", 0), + "lifecycle_event_hash": stream.get("order_event_hash", ""), + "lifecycle_event_dropped": stream.get("order_event_dropped", 0), "input_bars_equal": batch["input_bars_processed"] == stream["input_bars_processed"], "script_bars_equal": batch["script_bars_processed"] diff --git a/src/c_abi.cpp b/src/c_abi.cpp index 317f156..c19dc16 100644 --- a/src/c_abi.cpp +++ b/src/c_abi.cpp @@ -119,6 +119,9 @@ static_assert(offsetof(pf_report_t, equity_curve) == offsetof(pineforge::ReportC "pf_report_t::equity_curve offset mismatch"); static_assert(offsetof(pf_report_t, equity_curve_len) == offsetof(pineforge::ReportC, equity_curve_len), "pf_report_t::equity_curve_len offset mismatch"); +static_assert(offsetof(pf_report_t, order_event_dropped) + == offsetof(pineforge::ReportC, order_event_dropped), + "pf_report_t::order_event_dropped tail offset mismatch"); /* ── Magnifier distribution enum parity ─────────────────────────── */ @@ -176,6 +179,13 @@ PF_API void strategy_set_trade_start_time(pf_strategy_t s, int64_t timestamp_ms) static_cast(s)->set_trade_start_time(timestamp_ms); } +PF_API int strategy_stream_set_gap_policy( + pf_strategy_t s, pf_stream_gap_policy_t policy) { + if (!s) return -1; + return static_cast(s)->stream_set_gap_policy( + static_cast(policy)) ? 0 : -1; +} + PF_API int strategy_stream_begin(pf_strategy_t s, const pf_bar_t* warmup_bars, int n_warmup, diff --git a/src/engine_fills.cpp b/src/engine_fills.cpp index 8eac397..3f1f6c1 100644 --- a/src/engine_fills.cpp +++ b/src/engine_fills.cpp @@ -1133,6 +1133,8 @@ void BacktestEngine::apply_filled_order_to_state( const double position_qty_before_fill = position_qty_; const size_t pyramid_lots_before_fill = pyramid_entries_.size(); double signed_pos_before = signed_pos(); + const double equity_before_fill = + current_equity() + open_profit(bar.close); // Priced (stop/limit) fills happen mid-bar: any trade they close must // fold the pre-fill portion of the bar's path into its excursion @@ -1214,6 +1216,7 @@ void BacktestEngine::apply_filled_order_to_state( || trades_.size() != trades_before; if (primary_fill_applied) { ++broker_fill_event_seq_; + order.terminal_fill_id = broker_fill_event_seq_; } // Queue the one-shot 1x-long post-fill affordability event at the single @@ -1286,6 +1289,12 @@ void BacktestEngine::apply_filled_order_to_state( double signed_pos_after = signed_pos(); double filled_qty = std::abs(signed_pos_after - signed_pos_before); + if (primary_fill_applied) { + record_order_transition( + order, lifecycle_state(order), OrderLifecycleState::FILLED, + OrderTransition::FILLED, OrderTransitionReason::PRICE_FILL, + filled_qty, fill_price, signed_pos_before, equity_before_fill); + } // This fill just opened a position from FLAT via an entry order. Freeze // any LAYERED strategy.exit legs bound to that entry that were armed while @@ -1330,9 +1339,11 @@ void BacktestEngine::apply_filled_order_to_state( bool fully_filled = std::isnan(order.qty) || filled_qty + kOcaQtyEpsilon >= order.qty; if (order.oca_type == 1 && fully_filled) { - cancel_oca_group(order.oca_name, order.id); + cancel_oca_group(order.oca_name, order.oca_type, order.id, + order.order_leg_id); } else if (order.oca_type == 2) { - reduce_oca_group(order.oca_name, order.id, filled_qty); + reduce_oca_group(order.oca_name, order.oca_type, order.id, + order.order_leg_id, filled_qty); } } // When an exit fill causes position to go flat, subsequent EXIT @@ -1678,7 +1689,10 @@ void BacktestEngine::apply_raw_order_fill(PendingOrder& order, double fill_price trail_best_price_ = fill_price; pyramid_entries_.clear(); id_unclosed_qty_.clear(); - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, qty, order.id, bar_index_}); + pyramid_entries_.push_back({fill_price, current_bar_.timestamp, qty, + order.id, bar_index_, "", 0.0, 0.0, + false, false, 0}); + assign_entry_lot_identity(pyramid_entries_.back()); id_unclosed_qty_[order.id] += qty; if (!std::isnan(order.stop_price) || !std::isnan(order.limit_price)) { set_entry_fill_excursion_masks(pyramid_entries_.back(), current_bar_, fill_price); @@ -1729,7 +1743,10 @@ void BacktestEngine::apply_raw_order_fill(PendingOrder& order, double fill_price position_qty_ = total_qty; position_entry_count_++; trail_best_price_ = fill_price; - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, new_qty, order.id, bar_index_}); + pyramid_entries_.push_back({fill_price, current_bar_.timestamp, + new_qty, order.id, bar_index_, "", + 0.0, 0.0, false, false, 0}); + assign_entry_lot_identity(pyramid_entries_.back()); id_unclosed_qty_[order.id] += new_qty; if (is_priced_entry) { set_entry_fill_excursion_masks(pyramid_entries_.back(), current_bar_, fill_price); @@ -2180,6 +2197,83 @@ BacktestEngine::FillEvaluation BacktestEngine::evaluate_fill_price( return {FillEvaluation::Kind::NoFill, 0.0}; } + // Realtime uses one exact observed price per dispatch. Keep mutable trail + // activation/watermark on this executable leg rather than feeding point + // bars through the historical position-global OHLC trail path. + if (stream_phase_ == StreamPhase::REALTIME && exit_style && has_trail) { + const bool closing_long = position_side_ == PositionSide::LONG; + const double observed = bar.close; + const double activation = !std::isnan(order.trail_points) + ? position_entry_price_ + + (closing_long ? 1.0 : -1.0) + * std::ceil(order.trail_points) * syminfo_mintick_ + : order.trail_price; + if (std::isnan(order.realtime_trail_best_price)) { + order.realtime_trail_best_price = observed; + } + + const double offset = std::isnan(order.trail_offset) + ? std::numeric_limits::quiet_NaN() + : std::ceil(order.trail_offset) * syminfo_mintick_; + bool stop_hit = has_stop + && (closing_long ? observed <= stop_price : observed >= stop_price); + bool limit_hit = has_limit + && (closing_long ? observed >= limit_price : observed <= limit_price); + bool trail_hit = false; + if (order.realtime_trail_activated) { + const double trail_level = std::isnan(offset) + ? activation + : (closing_long + ? order.realtime_trail_best_price - offset + : order.realtime_trail_best_price + offset); + trail_hit = std::isfinite(trail_level) + && (closing_long ? observed <= trail_level + : observed >= trail_level); + } + + if (stop_hit || trail_hit) { + last_exit_fill_was_trail_ = trail_hit && !stop_hit; + return {FillEvaluation::Kind::Fill, observed, false}; + } + if (limit_hit) { + return {FillEvaluation::Kind::Fill, observed, true}; + } + + if (!order.realtime_trail_activated && std::isfinite(activation)) { + order.realtime_trail_activated = closing_long + ? observed >= activation : observed <= activation; + if (order.realtime_trail_activated) { + record_order_transition( + order, OrderLifecycleState::PENDING_TRAIL_ACTIVATION, + OrderLifecycleState::ACTIVE_TRAIL, + OrderTransition::ACTIVATED, + OrderTransitionReason::TRAIL_TRIGGER); + } + // A trail without an offset exits at its activation event. + if (order.realtime_trail_activated && std::isnan(offset)) { + last_exit_fill_was_trail_ = true; + return {FillEvaluation::Kind::Fill, observed, false}; + } + } + const double best_before = order.realtime_trail_best_price; + if (closing_long) { + order.realtime_trail_best_price = std::max( + order.realtime_trail_best_price, observed); + } else { + order.realtime_trail_best_price = std::min( + order.realtime_trail_best_price, observed); + } + if (order.realtime_trail_activated + && order.realtime_trail_best_price != best_before) { + record_order_transition( + order, OrderLifecycleState::ACTIVE_TRAIL, + OrderLifecycleState::ACTIVE_TRAIL, + OrderTransition::RATCHETED, + OrderTransitionReason::TRAIL_FAVORABLE_PRICE); + } + return {FillEvaluation::Kind::NoFill, 0.0}; + } + bool exit_same_bar_reissue = exit_style && !has_trail && process_orders_on_close_ && order.created_bar == bar_index_ && !order.created_during_coof_recalc; @@ -2240,8 +2334,11 @@ BacktestEngine::FillEvaluation BacktestEngine::evaluate_fill_price( // and the limit can only fill after activation along the OHLC path. // The actual fill is the LIMIT leg (at the limit price or better), // so it takes the unslipped limit-or-better price path. - bool activated = calc_on_order_fills_ && coof_scheduler_active_ - ? order.stop_limit_activated : false; + const bool coof_owns_activation = + calc_on_order_fills_ && coof_scheduler_active_; + const bool was_activated = order.stop_limit_activated; + bool activated = coof_owns_activation + ? order.stop_limit_activated : was_activated; should_fill = resolve_entry_stop_limit_fill( bar, order.is_long, @@ -2249,6 +2346,17 @@ BacktestEngine::FillEvaluation BacktestEngine::evaluate_fill_price( limit_price, &fill_price, &activated); + if (!coof_owns_activation) { + order.stop_limit_activated = activated; + } + if (!coof_owns_activation && stream_phase_ == StreamPhase::REALTIME + && !was_activated && activated) { + record_order_transition( + order, OrderLifecycleState::PENDING_STOP_LIMIT, + OrderLifecycleState::ACTIVE_STOP_LIMIT, + OrderTransition::ACTIVATED, + OrderTransitionReason::STOP_LIMIT_TRIGGER); + } is_limit_fill = should_fill; } else if (!should_fill && has_stop) { // Entry stop order diff --git a/src/engine_order_events.cpp b/src/engine_order_events.cpp new file mode 100644 index 0000000..3a5d2e8 --- /dev/null +++ b/src/engine_order_events.cpp @@ -0,0 +1,164 @@ +/* Deterministic command/order lifecycle diagnostics. */ + +#include "engine_internal.hpp" + +#include +#include +#include +#include + +namespace pineforge { +namespace { + +template +void hash_unsigned(uint64_t& hash, UInt value) { + static_assert(std::is_unsigned::value, "unsigned hash input"); + for (std::size_t i = 0; i < sizeof(UInt); ++i) { + hash ^= static_cast((value >> (i * 8)) & 0xffU); + hash *= 1099511628211ULL; + } +} + +void hash_double(uint64_t& hash, double value) { + uint64_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + hash_unsigned(hash, bits); +} + +void hash_string(uint64_t& hash, const std::string& value) { + hash_unsigned(hash, static_cast(value.size())); + for (unsigned char byte : value) { + hash ^= byte; + hash *= 1099511628211ULL; + } +} + +double trail_activation_price(const PendingOrder& order, + PositionSide position_side, + double entry_price, double mintick) { + if (!std::isnan(order.trail_points)) { + const double direction = position_side == PositionSide::SHORT ? -1.0 : 1.0; + return entry_price + direction * std::ceil(order.trail_points) * mintick; + } + return order.trail_price; +} + +} // namespace + +OrderLifecycleState BacktestEngine::lifecycle_state( + const PendingOrder& order) const { + const bool has_trail = !std::isnan(order.trail_points) + || !std::isnan(order.trail_price); + if (has_trail) { + return order.realtime_trail_activated + ? OrderLifecycleState::ACTIVE_TRAIL + : OrderLifecycleState::PENDING_TRAIL_ACTIVATION; + } + const bool has_stop = !std::isnan(order.stop_price); + const bool has_limit = !std::isnan(order.limit_price); + if (has_stop && has_limit) { + return order.stop_limit_activated + ? OrderLifecycleState::ACTIVE_STOP_LIMIT + : OrderLifecycleState::PENDING_STOP_LIMIT; + } + if (has_stop) return OrderLifecycleState::PENDING_STOP; + if (has_limit) return OrderLifecycleState::PENDING_LIMIT; + return OrderLifecycleState::PENDING_MARKET; +} + +void BacktestEngine::record_order_transition( + const PendingOrder& order, OrderLifecycleState before, + OrderLifecycleState after, OrderTransition transition, + OrderTransitionReason reason, double filled_quantity, + double fill_price, double position_before, double equity_before) { + if (!order_event_recording_enabled_) return; + OrderLifecycleEvent event; + pf_order_event_t& v = event.value; + v.transition_sequence = next_transition_sequence_++; + v.command_revision_id = order.command_revision_id; + v.order_leg_id = order.order_leg_id; + v.priority_sequence = order.priority_sequence; + v.fill_id = order.terminal_fill_id; + v.entry_lot_id = 0; + if (!pyramid_entries_.empty() + && (order.type == OrderType::MARKET + || order.type == OrderType::ENTRY + || order.type == OrderType::RAW_ORDER)) { + v.entry_lot_id = pyramid_entries_.back().entry_lot_id; + } + const double position_after = signed_position_size(); + if (std::isfinite(position_before) + && std::abs(position_before) <= internal::kQtyEpsilon + && std::abs(position_after) > internal::kQtyEpsilon) { + ++position_episode_id_; + } + v.position_episode_id = position_episode_id_; + v.event_timestamp = current_bar_.timestamp; + v.event_sequence = stream_phase_ == StreamPhase::REALTIME + ? stream_last_sequence_ : 0; + v.input_bar_index = std::max( + 0, static_cast(diag_input_bars_processed_) - 1); + v.script_bar_index = bar_index_; + v.command_kind = static_cast(order.type) + 1; + const bool has_trail = !std::isnan(order.trail_points) + || !std::isnan(order.trail_price); + const bool has_stop = !std::isnan(order.stop_price); + const bool has_limit = !std::isnan(order.limit_price); + v.leg_kind = has_trail ? 5 : (has_stop && has_limit ? 4 + : (has_stop ? 3 : (has_limit ? 2 : 1))); + v.state_before = static_cast(before); + v.state_after = static_cast(after); + v.transition = static_cast(transition); + v.reason = static_cast(reason); + v.side = order.is_long ? 1 : -1; + v.oca_type = order.oca_type; + v.requested_quantity = order.qty; + v.remaining_quantity = transition == OrderTransition::FILLED ? 0.0 : order.qty; + v.filled_quantity = filled_quantity; + v.observed_price = current_bar_.close; + v.stop_price = order.stop_price; + v.limit_price = order.limit_price; + v.trail_activation_price = trail_activation_price( + order, position_side_, position_entry_price_, syminfo_mintick_); + v.trail_watermark = order.realtime_trail_best_price; + v.fill_price = fill_price; + v.position_size_before = position_before; + v.position_size_after = position_after; + v.equity_before = equity_before; + v.equity_after = current_equity() + open_profit(current_bar_.close); + event.id = order.id; + event.from_entry = order.from_entry; + event.oca_name = order.oca_name; + + uint64_t& hash = order_event_hash_; +#define PF_HASH_U(field) hash_unsigned(hash, static_cast(v.field)) +#define PF_HASH_D(field) hash_double(hash, v.field) + PF_HASH_U(transition_sequence); PF_HASH_U(command_revision_id); + PF_HASH_U(order_leg_id); PF_HASH_U(priority_sequence); PF_HASH_U(fill_id); + PF_HASH_U(entry_lot_id); PF_HASH_U(position_episode_id); + PF_HASH_U(event_timestamp); PF_HASH_U(event_sequence); + PF_HASH_U(input_bar_index); PF_HASH_U(script_bar_index); + PF_HASH_U(command_kind); PF_HASH_U(leg_kind); PF_HASH_U(state_before); + PF_HASH_U(state_after); PF_HASH_U(transition); PF_HASH_U(reason); + PF_HASH_U(side); PF_HASH_U(oca_type); + PF_HASH_D(requested_quantity); PF_HASH_D(remaining_quantity); + PF_HASH_D(filled_quantity); PF_HASH_D(observed_price); PF_HASH_D(stop_price); + PF_HASH_D(limit_price); PF_HASH_D(trail_activation_price); + PF_HASH_D(trail_watermark); PF_HASH_D(fill_price); + PF_HASH_D(position_size_before); PF_HASH_D(position_size_after); + PF_HASH_D(equity_before); PF_HASH_D(equity_after); +#undef PF_HASH_D +#undef PF_HASH_U + hash_string(hash, event.id); + hash_string(hash, event.from_entry); + hash_string(hash, event.oca_name); + + ++order_event_count_; + if (order_events_.size() < kOrderEventRetentionCapacity) { + order_events_.push_back(std::move(event)); + } else { + ++order_event_dropped_; + } +} + +} // namespace pineforge diff --git a/src/engine_orders.cpp b/src/engine_orders.cpp index 0f32008..bc883bc 100644 --- a/src/engine_orders.cpp +++ b/src/engine_orders.cpp @@ -179,7 +179,8 @@ double BacktestEngine::fifo_drain(const std::string* from_entry, double qty_limi pe.max_runup * keep_scale, pe.max_drawdown * keep_scale, pe.skip_entry_bar_high, - pe.skip_entry_bar_low}); + pe.skip_entry_bar_low, + pe.entry_lot_id}); } } @@ -273,12 +274,30 @@ void BacktestEngine::execute_partial_exit_by_entry_percent(double fill_price, // Internal helper: cancel OCA group members (except the one that just filled) -void BacktestEngine::cancel_oca_group(const std::string& oca_name, const std::string& exclude_id) { +void BacktestEngine::cancel_oca_group(const std::string& oca_name, int oca_type, + const std::string& exclude_id, + uint64_t exclude_order_leg_id) { if (oca_name.empty()) return; + const bool leg_specific = stream_phase_ == StreamPhase::REALTIME; + for (const PendingOrder& order : pending_orders_) { + const bool excluded = leg_specific + ? order.order_leg_id == exclude_order_leg_id + : order.id == exclude_id; + if (order.oca_name == oca_name && order.oca_type == oca_type + && !excluded) { + record_order_transition( + order, lifecycle_state(order), OrderLifecycleState::CANCELLED, + OrderTransition::CANCELLED, OrderTransitionReason::OCA_EFFECT); + } + } pending_orders_.erase( std::remove_if(pending_orders_.begin(), pending_orders_.end(), [&](const PendingOrder& o) { - return o.oca_name == oca_name && o.id != exclude_id; + const bool excluded = leg_specific + ? o.order_leg_id == exclude_order_leg_id + : o.id == exclude_id; + return o.oca_name == oca_name && o.oca_type == oca_type + && !excluded; }), pending_orders_.end()); } @@ -291,17 +310,38 @@ void BacktestEngine::cancel_oca_group(const std::string& oca_name, const std::st // per-order qty applied at place time, so we conservatively cancel them // (this matches the prior, blanket-cancel behaviour for that subset). void BacktestEngine::reduce_oca_group(const std::string& oca_name, + int oca_type, const std::string& exclude_id, + uint64_t exclude_order_leg_id, double filled_qty) { if (oca_name.empty()) return; if (!(filled_qty > 0.0)) return; // nothing to subtract + const bool leg_specific = stream_phase_ == StreamPhase::REALTIME; pending_orders_.erase( std::remove_if(pending_orders_.begin(), pending_orders_.end(), [&](PendingOrder& o) { - if (o.oca_name != oca_name || o.id == exclude_id) return false; - if (std::isnan(o.qty)) return true; // default-sized: cancel + const bool excluded = leg_specific + ? o.order_leg_id == exclude_order_leg_id + : o.id == exclude_id; + if (o.oca_name != oca_name || o.oca_type != oca_type || excluded) + return false; + if (std::isnan(o.qty)) { + record_order_transition( + o, lifecycle_state(o), OrderLifecycleState::CANCELLED, + OrderTransition::CANCELLED, + OrderTransitionReason::OCA_EFFECT); + return true; + } o.qty -= filled_qty; - return o.qty <= kOcaQtyEpsilon; + const bool cancelled = o.qty <= kOcaQtyEpsilon; + record_order_transition( + o, lifecycle_state(o), + cancelled ? OrderLifecycleState::CANCELLED + : lifecycle_state(o), + cancelled ? OrderTransition::CANCELLED + : OrderTransition::REDUCED, + OrderTransitionReason::OCA_EFFECT); + return cancelled; }), pending_orders_.end()); } @@ -545,7 +585,9 @@ void BacktestEngine::open_fresh_position(PositionSide requested, double fill_pri id_unclosed_qty_.clear(); close_reserved_qty_.clear(); consumed_partial_exit_ids_.clear(); - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, qty, id, bar_index_}); + pyramid_entries_.push_back({fill_price, current_bar_.timestamp, qty, id, + bar_index_, "", 0.0, 0.0, false, false, 0}); + assign_entry_lot_identity(pyramid_entries_.back()); id_unclosed_qty_[id] += qty; } @@ -704,7 +746,9 @@ void BacktestEngine::add_to_pyramid_market(const std::string& id, bool is_long, position_qty_ = total_qty; position_entry_count_++; trail_best_price_ = fill_price; - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, new_qty, id, bar_index_}); + pyramid_entries_.push_back({fill_price, current_bar_.timestamp, new_qty, id, + bar_index_, "", 0.0, 0.0, false, false, 0}); + assign_entry_lot_identity(pyramid_entries_.back()); id_unclosed_qty_[id] += new_qty; } diff --git a/src/engine_report.cpp b/src/engine_report.cpp index 69f24f0..15771a1 100644 --- a/src/engine_report.cpp +++ b/src/engine_report.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -54,6 +55,34 @@ void BacktestEngine::fill_report(ReportC* out) const { fill_security_diag_section(out); fill_trace_section(out); + fill_order_events_section(out); +} + +void BacktestEngine::fill_order_events_section(ReportC* out) const { + const int64_t n = static_cast(order_events_.size()); + out->order_events_len = n; + out->order_event_count = order_event_count_; + out->order_event_hash = order_event_hash_; + out->order_event_dropped = order_event_dropped_; + if (n == 0) { + out->order_events = nullptr; + return; + } + out->order_events = new pf_order_event_t[n]{}; + auto copy_string = [](const std::string& source) { + char* result = new char[source.size() + 1]; + std::memcpy(result, source.c_str(), source.size() + 1); + return result; + }; + for (int64_t i = 0; i < n; ++i) { + out->order_events[i] = order_events_[static_cast(i)].value; + out->order_events[i].id = copy_string( + order_events_[static_cast(i)].id); + out->order_events[i].from_entry = copy_string( + order_events_[static_cast(i)].from_entry); + out->order_events[i].oca_name = copy_string( + order_events_[static_cast(i)].oca_name); + } } @@ -214,6 +243,19 @@ void BacktestEngine::free_report(ReportC* report) { report->equity_curve = nullptr; report->equity_curve_len = 0; } + if (report && report->order_events) { + for (int64_t i = 0; i < report->order_events_len; ++i) { + delete[] report->order_events[i].id; + delete[] report->order_events[i].from_entry; + delete[] report->order_events[i].oca_name; + } + delete[] report->order_events; + report->order_events = nullptr; + report->order_events_len = 0; + report->order_event_count = 0; + report->order_event_hash = 0; + report->order_event_dropped = 0; + } } } // namespace pineforge diff --git a/src/engine_run.cpp b/src/engine_run.cpp index 61aada5..1f45617 100644 --- a/src/engine_run.cpp +++ b/src/engine_run.cpp @@ -399,6 +399,16 @@ void BacktestEngine::reset_run_state() { intraday_cap_hit_ = false; intraday_fill_count_ = 0; broker_fill_event_seq_ = 0; + next_command_revision_id_ = 1; + next_order_leg_id_ = 1; + next_entry_lot_id_ = 1; + next_transition_sequence_ = 1; + position_episode_id_ = 0; + order_events_.clear(); + order_event_count_ = 0; + order_event_hash_ = 1469598103934665603ULL; + order_event_dropped_ = 0; + order_event_recording_enabled_ = false; coof_scheduler_active_ = false; coof_fill_recalc_active_ = false; coof_cursor_is_bar_close_ = false; @@ -428,10 +438,14 @@ void BacktestEngine::reset_run_state() { stream_seen_sequence_ = false; stream_has_input_bar_ = false; stream_input_bar_ = Bar{}; + stream_input_last_trade_ms_ = 0; + stream_input_last_trade_sequence_ = 0; stream_last_price_ = 0.0; stream_has_last_price_ = false; stream_next_script_bar_index_ = 0; stream_script_bar_had_tick_ = false; + stream_script_last_trade_ms_ = 0; + stream_script_last_trade_sequence_ = 0; stream_script_tick_seen_ = false; // Native source-series history (input.source(...) ring buffers). Must list @@ -1270,6 +1284,11 @@ void BacktestEngine::run(const Bar* input_bars, int n_input, process_orders_on_close_ = (overrides->process_orders_on_close != 0); if (overrides->calc_on_order_fills >= 0) calc_on_order_fills_ = (overrides->calc_on_order_fills != 0); + if (overrides->calc_on_every_tick >= 0) + calc_on_every_tick_ = (overrides->calc_on_every_tick != 0); + if (overrides->calc_on_every_history_tick >= 0) + calc_on_every_history_tick_ = + (overrides->calc_on_every_history_tick != 0); if (overrides->close_entries_rule >= 0) close_entries_rule_any_ = (overrides->close_entries_rule != 0); } diff --git a/src/engine_strategy_commands.cpp b/src/engine_strategy_commands.cpp index 23ec70a..83a5bd9 100644 --- a/src/engine_strategy_commands.cpp +++ b/src/engine_strategy_commands.cpp @@ -76,6 +76,27 @@ inline bool trading_is_active(int64_t current_ms, int64_t start_ms, : 0; return current_ms >= start_ms - buffer_ms; } + +} + +void BacktestEngine::assign_pending_order_identity(PendingOrder& order) { + order.command_revision_id = next_command_revision_id_++; + order.order_leg_id = next_order_leg_id_++; + order.priority_sequence = static_cast( + std::max(0, order.created_seq)); + order.terminal_fill_id = 0; + const double pos = signed_position_size(); + record_order_transition( + order, OrderLifecycleState::NONE, lifecycle_state(order), + OrderTransition::CREATED, OrderTransitionReason::NONE, + 0.0, std::numeric_limits::quiet_NaN(), pos, + current_equity() + open_profit(current_bar_.close)); +} + +void BacktestEngine::assign_entry_lot_identity(PyramidEntry& entry) { + if (entry.entry_lot_id == 0) { + entry.entry_lot_id = next_entry_lot_id_++; + } } void BacktestEngine::strategy_entry(const std::string& id, bool is_long, @@ -164,7 +185,15 @@ void BacktestEngine::strategy_entry(const std::string& id, bool is_long, } } - // Remove existing pending order with same id + // Remove existing pending order with same id. + for (const auto& pending : pending_orders_) { + if (pending.id == id) { + record_order_transition( + pending, lifecycle_state(pending), OrderLifecycleState::REPLACED, + OrderTransition::REPLACED, + OrderTransitionReason::COMMAND_REISSUE); + } + } pending_orders_.erase( std::remove_if(pending_orders_.begin(), pending_orders_.end(), [&](const PendingOrder& o) { return o.id == id; }), @@ -298,6 +327,7 @@ void BacktestEngine::strategy_entry(const std::string& id, bool is_long, order.stop_price = stop_price; } + assign_pending_order_identity(order); pending_orders_.push_back(std::move(order)); } @@ -529,9 +559,34 @@ void BacktestEngine::flush_same_bar_close() { size_t trades_before = trades_.size(); PositionSide side_before = position_side_; double qty_before = position_qty_; + const double signed_before = signed_position_size(); + const double equity_before = + current_equity() + open_profit(current_bar_.close); const double broker_price = coof_scheduler_active_ && std::isfinite(coof_cursor_price_) ? coof_cursor_price_ : current_bar_.close; + PendingOrder diagnostic_order; + diagnostic_order.id = "__close__" + id; + diagnostic_order.from_entry = close_entries_rule_any_ ? id : ""; + diagnostic_order.type = OrderType::EXIT; + diagnostic_order.is_long = position_side_ == PositionSide::SHORT; + diagnostic_order.limit_price = std::numeric_limits::quiet_NaN(); + diagnostic_order.stop_price = std::numeric_limits::quiet_NaN(); + diagnostic_order.trail_points = std::numeric_limits::quiet_NaN(); + diagnostic_order.trail_price = std::numeric_limits::quiet_NaN(); + diagnostic_order.trail_offset = std::numeric_limits::quiet_NaN(); + diagnostic_order.qty = target; + diagnostic_order.qty_type = -1; + diagnostic_order.qty_percent = qty_before > eps + ? target / qty_before * 100.0 : 100.0; + diagnostic_order.oca_type = 0; + diagnostic_order.created_bar = bar_index_; + diagnostic_order.created_position_side = position_side_; + diagnostic_order.comment = comment; + if (order_event_recording_enabled_) { + diagnostic_order.created_seq = next_order_seq_; + assign_pending_order_identity(diagnostic_order); + } if (closes_full_position) { const bool closed_long = (position_side_ == PositionSide::LONG); // Exit-order cancel/purge already ran at CALL time in @@ -557,6 +612,15 @@ void BacktestEngine::flush_same_bar_close() { || std::abs(position_qty_ - qty_before) > eps || trades_.size() != trades_before) { ++broker_fill_event_seq_; + if (order_event_recording_enabled_) { + diagnostic_order.terminal_fill_id = broker_fill_event_seq_; + record_order_transition( + diagnostic_order, OrderLifecycleState::PENDING_MARKET, + OrderLifecycleState::FILLED, OrderTransition::FILLED, + OrderTransitionReason::STREAM_CLOSE_EVENT, + std::abs(signed_position_size() - signed_before), broker_price, + signed_before, equity_before); + } if (coof_scheduler_active_ && coof_direct_fill_events_remaining_ > 0) { --coof_direct_fill_events_remaining_; } @@ -775,10 +839,34 @@ void BacktestEngine::strategy_exit(const std::string& id, const std::string& fro order.comment = comment; order.created_while_in_position = !effectively_flat; + if (has_trail_request && !effectively_flat + && std::isfinite(current_bar_.close)) { + order.realtime_trail_best_price = current_bar_.close; + const double activation = !std::isnan(trail_points) + ? position_entry_price_ + + (position_side_ == PositionSide::LONG ? 1.0 : -1.0) + * std::ceil(trail_points) * syminfo_mintick_ + : trail_price; + if (std::isfinite(activation)) { + order.realtime_trail_activated = + position_side_ == PositionSide::LONG + ? current_bar_.close >= activation + : current_bar_.close <= activation; + } + } + + assign_pending_order_identity(order); pending_orders_.push_back(std::move(order)); } void BacktestEngine::strategy_cancel(const std::string& id) { + for (const auto& pending : pending_orders_) { + if (pending.id == id) { + record_order_transition( + pending, lifecycle_state(pending), OrderLifecycleState::CANCELLED, + OrderTransition::CANCELLED, OrderTransitionReason::USER_CANCEL); + } + } pending_orders_.erase( std::remove_if(pending_orders_.begin(), pending_orders_.end(), [&](const PendingOrder& o) { return o.id == id; }), @@ -786,6 +874,11 @@ void BacktestEngine::strategy_cancel(const std::string& id) { } void BacktestEngine::strategy_cancel_all() { + for (const auto& pending : pending_orders_) { + record_order_transition( + pending, lifecycle_state(pending), OrderLifecycleState::CANCELLED, + OrderTransition::CANCELLED, OrderTransitionReason::USER_CANCEL); + } pending_orders_.clear(); } @@ -801,6 +894,14 @@ void BacktestEngine::strategy_order(const std::string& id, bool is_long, double } } + for (const auto& pending : pending_orders_) { + if (pending.id == id) { + record_order_transition( + pending, lifecycle_state(pending), OrderLifecycleState::REPLACED, + OrderTransition::REPLACED, + OrderTransitionReason::COMMAND_REISSUE); + } + } // Remove existing pending order with same id pending_orders_.erase( std::remove_if(pending_orders_.begin(), pending_orders_.end(), @@ -872,6 +973,7 @@ void BacktestEngine::strategy_order(const std::string& id, bool is_long, double order.stop_price = stop_price; } + assign_pending_order_identity(order); pending_orders_.push_back(std::move(order)); } @@ -1033,9 +1135,34 @@ void BacktestEngine::execute_immediate_close(const std::string& id, size_t trades_before = trades_.size(); PositionSide side_before = position_side_; double qty_before = position_qty_; + const double signed_before = signed_position_size(); + const double equity_before = + current_equity() + open_profit(current_bar_.close); const double broker_price = coof_scheduler_active_ && std::isfinite(coof_cursor_price_) ? coof_cursor_price_ : current_bar_.close; + PendingOrder diagnostic_order; + if (order_event_recording_enabled_) { + diagnostic_order.id = "__close__" + id; + diagnostic_order.from_entry = close_entries_rule_any_ ? id : ""; + diagnostic_order.type = OrderType::EXIT; + diagnostic_order.is_long = position_side_ == PositionSide::SHORT; + diagnostic_order.limit_price = std::numeric_limits::quiet_NaN(); + diagnostic_order.stop_price = std::numeric_limits::quiet_NaN(); + diagnostic_order.trail_points = std::numeric_limits::quiet_NaN(); + diagnostic_order.trail_price = std::numeric_limits::quiet_NaN(); + diagnostic_order.trail_offset = std::numeric_limits::quiet_NaN(); + diagnostic_order.qty = qty_to_close; + diagnostic_order.qty_type = -1; + diagnostic_order.qty_percent = matching_qty > eps + ? qty_to_close / matching_qty * 100.0 : 100.0; + diagnostic_order.oca_type = 0; + diagnostic_order.created_bar = bar_index_; + diagnostic_order.created_seq = next_order_seq_; + diagnostic_order.created_position_side = position_side_; + diagnostic_order.comment = comment; + assign_pending_order_identity(diagnostic_order); + } if (closes_full_position) { const bool closed_long = (position_side_ == PositionSide::LONG); execute_market_exit(broker_price); @@ -1063,6 +1190,15 @@ void BacktestEngine::execute_immediate_close(const std::string& id, || std::abs(position_qty_ - qty_before) > eps || trades_.size() != trades_before) { ++broker_fill_event_seq_; + if (order_event_recording_enabled_) { + diagnostic_order.terminal_fill_id = broker_fill_event_seq_; + record_order_transition( + diagnostic_order, OrderLifecycleState::PENDING_MARKET, + OrderLifecycleState::FILLED, OrderTransition::FILLED, + OrderTransitionReason::STREAM_CLOSE_EVENT, + std::abs(signed_position_size() - signed_before), broker_price, + signed_before, equity_before); + } if (coof_scheduler_active_ && coof_direct_fill_events_remaining_ > 0) { --coof_direct_fill_events_remaining_; } @@ -1118,6 +1254,7 @@ void BacktestEngine::queue_deferred_close_order(const std::string& id, // debited (ANY rule / explicit qty / close_all). order.suppressed_close_consumed_ledger_qty = consumed_ledger_qty; + assign_pending_order_identity(order); pending_orders_.push_back(std::move(order)); } @@ -1145,6 +1282,16 @@ void BacktestEngine::clear_existing_exit_order(const std::string& id, } } + for (const auto& pending : pending_orders_) { + if (pending.type == OrderType::EXIT && pending.id == id + && pending.from_entry == from_entry) { + record_order_transition( + pending, lifecycle_state(pending), OrderLifecycleState::REPLACED, + OrderTransition::REPLACED, + OrderTransitionReason::COMMAND_REISSUE); + } + } + if (has_trail_request && !had_existing_order && position_side_ != PositionSide::FLAT) { trail_best_price_ = current_bar_.close; } diff --git a/src/engine_stream.cpp b/src/engine_stream.cpp index 5e9bc09..a96b96e 100644 --- a/src/engine_stream.cpp +++ b/src/engine_stream.cpp @@ -20,6 +20,24 @@ Bar price_point(double price, double volume, int64_t timestamp) { } // namespace +bool BacktestEngine::stream_set_gap_policy(int policy) { + last_error_.clear(); + if (stream_phase_ == StreamPhase::REALTIME) { + last_error_ = "stream gap policy cannot change after stream_begin"; + return false; + } + if (policy == static_cast(StreamGapPolicy::FIXED_GRID)) { + stream_gap_policy_ = StreamGapPolicy::FIXED_GRID; + return true; + } + if (policy == static_cast(StreamGapPolicy::DATA_DRIVEN)) { + stream_gap_policy_ = StreamGapPolicy::DATA_DRIVEN; + return true; + } + last_error_ = "unknown stream gap policy"; + return false; +} + bool BacktestEngine::stream_begin(const Bar* warmup_bars, int n_warmup, const std::string& input_tf, const std::string& script_tf) { @@ -28,6 +46,24 @@ bool BacktestEngine::stream_begin(const Bar* warmup_bars, int n_warmup, if (stream_phase_ == StreamPhase::REALTIME) { throw std::runtime_error("stream is already realtime"); } + // Validate the calculation profile before run() resets or otherwise + // mutates the strategy instance. Realtime v1 is close-only: silently + // warming with one scheduler and continuing with another would make + // the handoff state impossible to reason about or reproduce. + if (calc_on_every_tick_) { + throw std::runtime_error( + "stream profile unsupported: calc_on_every_tick requires " + "realtime rollback semantics"); + } + if (calc_on_every_history_tick_) { + throw std::runtime_error( + "stream profile unsupported: calc_on_every_history_tick"); + } + if (calc_on_order_fills_) { + throw std::runtime_error( + "stream profile unsupported: calc_on_order_fills requires " + "a realtime fill-recalculation scheduler"); + } if (warmup_bars == nullptr || n_warmup <= 0) { throw std::runtime_error( "stream warmup requires at least one confirmed OHLCV bar"); @@ -73,19 +109,46 @@ bool BacktestEngine::stream_begin(const Bar* warmup_bars, int n_warmup, stream_seen_sequence_ = false; stream_has_input_bar_ = false; stream_input_bar_ = Bar{}; + stream_input_last_trade_ms_ = 0; + stream_input_last_trade_sequence_ = 0; stream_last_price_ = warmup_bars[n_warmup - 1].close; stream_has_last_price_ = true; stream_next_script_bar_index_ = static_cast(diag_script_bars_processed_); stream_script_bar_had_tick_ = false; + stream_script_last_trade_ms_ = 0; + stream_script_last_trade_sequence_ = 0; stream_script_tick_seen_ = false; stream_phase_ = StreamPhase::REALTIME; + // Lifecycle diagnostics describe the realtime contract. Do not retain + // a potentially year-long warmup command log; instead start a fresh + // canonical sequence and snapshot every broker leg carried across the + // handoff with its already-assigned immutable identity. + order_events_.clear(); + order_event_count_ = 0; + order_event_hash_ = 1469598103934665603ULL; + order_event_dropped_ = 0; + next_transition_sequence_ = 1; + position_episode_id_ = position_side_ == PositionSide::FLAT ? 0 : 1; + order_event_recording_enabled_ = true; + for (const PendingOrder& pending : pending_orders_) { + record_order_transition( + pending, OrderLifecycleState::NONE, lifecycle_state(pending), + OrderTransition::CREATED, OrderTransitionReason::NONE, + 0.0, std::numeric_limits::quiet_NaN(), + signed_position_size(), + current_equity() + open_profit(current_bar_.close)); + } + // Exact normalized trades now drive the broker instead of inferred // OHLC paths. Strategy code remains close-only unless codegen opts in // to calc_on_every_tick; resting orders are nevertheless fillable on // each normalized trade, as on TradingView's realtime broker emulator. - bar_magnifier_enabled_ = true; + // Exact trade events are their own execution model, not the historical + // bar magnifier. Reporting the magnifier as enabled here conflates two + // distinct schedulers and makes handoff configuration diagnostics lie. + bar_magnifier_enabled_ = false; bar_index_ = stream_next_script_bar_index_; last_bar_index_ = bar_index_; last_bar_time_ = stream_next_input_open_ms_; @@ -142,6 +205,8 @@ bool BacktestEngine::stream_push_tick(const TradeTick& tick) { stream_last_price_ = tick.price; stream_has_last_price_ = true; stream_last_tick_ms_ = tick.timestamp; + stream_input_last_trade_ms_ = tick.timestamp; + stream_input_last_trade_sequence_ = tick.sequence; stream_clock_ms_ = tick.timestamp; if (tick.sequence != 0) { stream_last_sequence_ = tick.sequence; @@ -228,7 +293,10 @@ bool BacktestEngine::stream_end(bool finalize_partial_input_bar) { throw std::runtime_error("stream_end requires a realtime stream"); } if (finalize_partial_input_bar && stream_has_input_bar_) { - stream_feed_input_bar(stream_input_bar_, /*had_tick=*/true); + stream_feed_input_bar( + stream_input_bar_, /*had_tick=*/true, + stream_input_last_trade_ms_, + stream_input_last_trade_sequence_); stream_has_input_bar_ = false; stream_next_input_open_ms_ += stream_input_tf_ms_; } @@ -259,6 +327,11 @@ bool BacktestEngine::stream_finalize_until(int64_t timestamp_ms) { stream_next_input_open_ms_ += stream_input_tf_ms_; continue; } + if (!had_tick && stream_gap_policy_ == StreamGapPolicy::DATA_DRIVEN) { + stream_input_bar_ = Bar{}; + stream_next_input_open_ms_ += stream_input_tf_ms_; + continue; + } Bar completed; if (had_tick) { @@ -272,15 +345,22 @@ bool BacktestEngine::stream_finalize_until(int64_t timestamp_ms) { stream_last_price_, 0.0, stream_next_input_open_ms_); } - stream_feed_input_bar(completed, had_tick); + stream_feed_input_bar( + completed, had_tick, + had_tick ? stream_input_last_trade_ms_ : 0, + had_tick ? stream_input_last_trade_sequence_ : 0); stream_has_input_bar_ = false; stream_input_bar_ = Bar{}; + stream_input_last_trade_ms_ = 0; + stream_input_last_trade_sequence_ = 0; stream_next_input_open_ms_ += stream_input_tf_ms_; } return true; } -void BacktestEngine::stream_feed_input_bar(const Bar& bar, bool had_tick) { +void BacktestEngine::stream_feed_input_bar(const Bar& bar, bool had_tick, + int64_t last_trade_ms, + uint64_t last_trade_sequence) { ++diag_input_bars_processed_; last_bar_time_ = bar.timestamp; @@ -289,7 +369,8 @@ void BacktestEngine::stream_feed_input_bar(const Bar& bar, bool had_tick) { } if (!diag_needs_aggregation_) { - stream_dispatch_script_bar(bar, had_tick); + stream_dispatch_script_bar( + bar, had_tick, last_trade_ms, last_trade_sequence); return; } @@ -300,18 +381,35 @@ void BacktestEngine::stream_feed_input_bar(const Bar& bar, bool had_tick) { // The current input bar opened the next bucket; the aggregator emitted // the preceding partial bucket before retaining this bar as its new // current state. - stream_dispatch_script_bar(ab.bar, stream_script_bar_had_tick_); + stream_dispatch_script_bar( + ab.bar, stream_script_bar_had_tick_, + stream_script_last_trade_ms_, + stream_script_last_trade_sequence_); stream_script_bar_had_tick_ = had_tick; + stream_script_last_trade_ms_ = had_tick ? last_trade_ms : 0; + stream_script_last_trade_sequence_ = + had_tick ? last_trade_sequence : 0; } else { stream_script_bar_had_tick_ = stream_script_bar_had_tick_ || had_tick; + if (had_tick) { + stream_script_last_trade_ms_ = last_trade_ms; + stream_script_last_trade_sequence_ = last_trade_sequence; + } if (ab.is_complete) { - stream_dispatch_script_bar(ab.bar, stream_script_bar_had_tick_); + stream_dispatch_script_bar( + ab.bar, stream_script_bar_had_tick_, + stream_script_last_trade_ms_, + stream_script_last_trade_sequence_); stream_script_bar_had_tick_ = false; + stream_script_last_trade_ms_ = 0; + stream_script_last_trade_sequence_ = 0; } } } -void BacktestEngine::stream_dispatch_script_bar(const Bar& bar, bool had_tick) { +void BacktestEngine::stream_dispatch_script_bar( + const Bar& bar, bool had_tick, int64_t last_trade_ms, + uint64_t last_trade_sequence) { const int this_bar_index = stream_next_script_bar_index_++; bar_index_ = this_bar_index; last_bar_index_ = this_bar_index; @@ -322,20 +420,6 @@ void BacktestEngine::stream_dispatch_script_bar(const Bar& bar, bool had_tick) { ++diag_script_bars_processed_; pending_close_qty_in_bar_ = 0.0; - // A synthesized zero-volume interval has no raw broker pass. Give resting - // market orders one carried-price point at its open so time advancement is - // deterministic even through quiet in-session intervals. - if (!had_tick) { - current_bar_ = price_point(bar.open, 0.0, bar.timestamp); - process_pending_orders(current_bar_); - update_per_trade_extremes(); - const std::size_t trades_before_mc = trades_.size(); - process_margin_call(current_bar_); - if (trades_.size() != trades_before_mc) { - refresh_frozen_default_sizing_after_margin_call(); - } - } - current_bar_ = bar; session_ismarket_ = pine_session_ismarket( syminfo_.session, syminfo_.timezone, current_bar_.timestamp); @@ -351,18 +435,20 @@ void BacktestEngine::stream_dispatch_script_bar(const Bar& bar, bool had_tick) { _push_source_series(); on_bar(current_bar_); - if (process_orders_on_close_) { + if (process_orders_on_close_ && had_tick) { flush_same_bar_close(); // New close-time orders only get the closing price point. Re-walking // the full OHLC range would let a just-created order see prices that // occurred before it existed. const Bar completed_bar = current_bar_; current_bar_ = price_point( - completed_bar.close, 0.0, completed_bar.timestamp); + completed_bar.close, 0.0, last_trade_ms); process_pending_orders(current_bar_); current_bar_ = completed_bar; } + (void)last_trade_sequence; // retained for lifecycle provenance reporting + finalize_bar(); prev_in_session_ = session_ismarket_; update_equity_extremes(); diff --git a/tests/test_streaming.cpp b/tests/test_streaming.cpp index 2c0ffc7..f067969 100644 --- a/tests/test_streaming.cpp +++ b/tests/test_streaming.cpp @@ -72,6 +72,89 @@ class CaptureStrategy final : public BacktestEngine { } }; +class StopLimitStrategy final : public BacktestEngine { +public: + void on_bar(const Bar&) override { + if (bar_index_ == 0) { + strategy_entry("SL", true, 103.0, 105.0); + } + } + + double position_size() const { return signed_position_size(); } + double entry_price() const { return position_entry_price_; } + bool activated() const { + return pending_orders_.size() == 1 + && pending_orders_.front().stop_limit_activated; + } +}; + +class QuietPendingMarketStrategy final : public BacktestEngine { +public: + void on_bar(const Bar&) override { + if (bar_index_ == 0) strategy_entry("quiet", true); + } + + double position_size() const { return signed_position_size(); } +}; + +class ProcessOnCloseProvenanceStrategy final : public BacktestEngine { +public: + ProcessOnCloseProvenanceStrategy() { process_orders_on_close_ = true; } + + void on_bar(const Bar&) override { + if (bar_index_ == 1) strategy_entry("poc", true); + } + + double position_size() const { return signed_position_size(); } + int64_t entry_time() const { return position_entry_time_; } +}; + +class UnsupportedProfileStrategy final : public BacktestEngine { +public: + enum class Profile { EveryTick, EveryHistoryTick, OrderFills }; + + explicit UnsupportedProfileStrategy(Profile profile) { + calc_on_every_tick_ = profile == Profile::EveryTick; + calc_on_every_history_tick_ = profile == Profile::EveryHistoryTick; + calc_on_order_fills_ = profile == Profile::OrderFills; + } + + void on_bar(const Bar&) override { ++calls; } + int calls = 0; +}; + +class ReplacementIdentityStrategy final : public BacktestEngine { +public: + void on_bar(const Bar&) override { + if (bar_index_ <= 1) { + strategy_entry("same", true, na(), 200.0); + CHECK(pending_orders_.size() == 1); + revisions.push_back(pending_orders_.front().command_revision_id); + legs.push_back(pending_orders_.front().order_leg_id); + priorities.push_back(pending_orders_.front().priority_sequence); + } + } + + std::vector revisions; + std::vector legs; + std::vector priorities; +}; + +class IndependentTrailStrategy final : public BacktestEngine { +public: + void on_bar(const Bar&) override { + if (bar_index_ == 0) strategy_entry("L", true); + if (bar_index_ == 1) { + strategy_exit("tight", "L", na(), na(), + 100.0, 50.0, na(), na(), "", 0.5); + strategy_exit("wide", "L", na(), na(), + 100.0, 100.0, na(), na(), "", 0.5); + } + } + + double position_size() const { return signed_position_size(); } +}; + void test_position_pending_order_and_equity_continue() { ContinuityStrategy strategy; const Bar warmup[] = { @@ -192,6 +275,170 @@ void test_clock_skips_out_of_session_intervals() { CHECK(strategy.bars.size() == 1); } +void test_data_driven_gap_policy_skips_tickless_intervals() { + CaptureStrategy strategy; + const Bar warmup[] = {flat_bar(42.0, 0, 3.0)}; + CHECK(strategy.stream_set_gap_policy(1)); + CHECK(strategy.stream_begin(warmup, 1, "1", "1")); + CHECK(strategy.stream_advance_time(240'000)); + CHECK(strategy.bars.size() == 1); + CHECK(!strategy.stream_set_gap_policy(0)); + + CHECK(strategy.stream_push_tick(tick(240'100, 1, 43.0))); + CHECK(strategy.stream_advance_time(300'000)); + CHECK(strategy.bars.size() == 2); + CHECK(strategy.bars.back().timestamp == 240'000); + CHECK(near(strategy.bars.back().close, 43.0)); +} + +void test_stop_limit_activation_persists_across_trade_events() { + StopLimitStrategy strategy; + const Bar warmup[] = {flat_bar(100.0, 0)}; + CHECK(strategy.stream_begin(warmup, 1, "1", "1")); + + // The stop is crossed first, but the buy limit is below this event. The + // order must become an active limit instead of forgetting activation. + CHECK(strategy.stream_push_tick(tick(60'100, 1, 106.0))); + CHECK(near(strategy.position_size(), 0.0)); + CHECK(strategy.activated()); + + CHECK(strategy.stream_push_tick(tick(60'200, 2, 103.0))); + CHECK(near(strategy.position_size(), 1.0)); + CHECK(near(strategy.entry_price(), 103.0)); +} + +void test_quiet_clock_does_not_fill_pending_market_order() { + QuietPendingMarketStrategy strategy; + const Bar warmup[] = {flat_bar(100.0, 0)}; + CHECK(strategy.stream_begin(warmup, 1, "1", "1")); + + CHECK(strategy.stream_advance_time(120'000)); + CHECK(near(strategy.position_size(), 0.0)); + CHECK(strategy.trade_count() == 0); + + // The clock carried the mark but supplied no executable price. The first + // actual trade is therefore the pending market order's fill event. + CHECK(strategy.stream_push_tick(tick(120'100, 1, 104.0))); + CHECK(near(strategy.position_size(), 1.0)); +} + +void test_process_on_close_uses_last_trade_provenance() { + ProcessOnCloseProvenanceStrategy strategy; + const Bar warmup[] = {flat_bar(100.0, 0)}; + CHECK(strategy.stream_begin(warmup, 1, "1", "1")); + CHECK(strategy.stream_push_tick(tick(60'100, 10, 101.0))); + CHECK(strategy.stream_push_tick(tick(119'900, 11, 105.0))); + CHECK(strategy.stream_advance_time(120'000)); + + CHECK(near(strategy.position_size(), 1.0)); + CHECK(strategy.entry_time() == 119'900); +} + +void test_unsupported_profiles_reject_before_warmup() { + const Bar warmup[] = {flat_bar(100.0, 0)}; + for (UnsupportedProfileStrategy::Profile profile : { + UnsupportedProfileStrategy::Profile::EveryTick, + UnsupportedProfileStrategy::Profile::EveryHistoryTick, + UnsupportedProfileStrategy::Profile::OrderFills}) { + UnsupportedProfileStrategy strategy(profile); + CHECK(!strategy.stream_begin(warmup, 1, "1", "1")); + CHECK(strategy.last_error().find("stream profile unsupported") + != std::string::npos); + CHECK(strategy.calls == 0); + CHECK(strategy.trade_count() == 0); + } +} + +void test_replacement_gets_fresh_identity_but_keeps_priority() { + ReplacementIdentityStrategy strategy; + const Bar warmup[] = { + flat_bar(100.0, 0), + flat_bar(101.0, 60'000), + }; + CHECK(strategy.stream_begin(warmup, 2, "1", "1")); + CHECK(strategy.revisions.size() == 2); + CHECK(strategy.legs.size() == 2); + CHECK(strategy.priorities.size() == 2); + CHECK(strategy.revisions[0] != strategy.revisions[1]); + CHECK(strategy.legs[0] != strategy.legs[1]); + CHECK(strategy.priorities[0] == strategy.priorities[1]); +} + +void test_realtime_trailing_state_is_per_order_leg() { + IndependentTrailStrategy strategy; + const Bar warmup[] = { + flat_bar(100.0, 0), + flat_bar(100.0, 60'000), + }; + CHECK(strategy.stream_begin(warmup, 2, "1", "1")); + CHECK(near(strategy.position_size(), 1.0)); + + CHECK(strategy.stream_push_tick(tick(120'100, 1, 101.2))); + CHECK(strategy.stream_push_tick(tick(120'200, 2, 101.5))); + CHECK(near(strategy.position_size(), 1.0)); + + // The 0.50-offset leg fires first. The 1.00-offset sibling retains its + // own watermark and remains live until the later, deeper retrace. + CHECK(strategy.stream_push_tick(tick(120'300, 3, 100.9))); + CHECK(near(strategy.position_size(), 0.5)); + CHECK(strategy.trade_count() == 1); + CHECK(near(strategy.get_trade(0).exit_price, 100.9)); + + CHECK(strategy.stream_push_tick(tick(120'400, 4, 100.4))); + CHECK(near(strategy.position_size(), 0.0)); + CHECK(strategy.trade_count() == 2); + CHECK(near(strategy.get_trade(1).exit_price, 100.4)); +} + +void test_lifecycle_report_is_deterministic_across_batching() { + const Bar warmup[] = {flat_bar(100.0, 0)}; + const TradeTick events[] = { + tick(60'100, 1, 106.0), + tick(60'200, 2, 103.0), + }; + StopLimitStrategy one_by_one; + StopLimitStrategy batched; + CHECK(one_by_one.stream_begin(warmup, 1, "1", "1")); + CHECK(batched.stream_begin(warmup, 1, "1", "1")); + CHECK(one_by_one.stream_push_tick(events[0])); + CHECK(one_by_one.stream_push_tick(events[1])); + CHECK(batched.stream_push_ticks(events, 2)); + + ReportC a{}; + ReportC b{}; + one_by_one.fill_report(&a); + batched.fill_report(&b); + CHECK(a.order_event_count == b.order_event_count); + CHECK(a.order_event_hash == b.order_event_hash); + CHECK(a.order_event_dropped == 0); + CHECK(a.order_events_len == static_cast(a.order_event_count)); + CHECK(a.order_events_len >= 3); // create, activate, fill + + bool saw_activation = false; + bool saw_fill = false; + uint64_t leg_id = 0; + for (int64_t i = 0; i < a.order_events_len; ++i) { + const pf_order_event_t& event = a.order_events[i]; + CHECK(event.transition_sequence == static_cast(i + 1)); + CHECK(event.id != nullptr && std::string(event.id) == "SL"); + if (leg_id == 0) leg_id = event.order_leg_id; + CHECK(event.order_leg_id == leg_id); + if (event.transition == static_cast(OrderTransition::ACTIVATED)) { + saw_activation = true; + CHECK(event.event_sequence == 1); + } + if (event.transition == static_cast(OrderTransition::FILLED)) { + saw_fill = true; + CHECK(event.fill_id != 0); + CHECK(event.event_sequence == 2); + } + } + CHECK(saw_activation); + CHECK(saw_fill); + BacktestEngine::free_report(&a); + BacktestEngine::free_report(&b); +} + void test_rejects_replayed_or_out_of_order_ticks() { CaptureStrategy strategy; const Bar warmup[] = {flat_bar(100.0, 0)}; @@ -211,6 +458,14 @@ int main() { test_partial_mtf_aggregator_survives_handoff(); test_clock_materializes_quiet_bars(); test_clock_skips_out_of_session_intervals(); + test_data_driven_gap_policy_skips_tickless_intervals(); + test_stop_limit_activation_persists_across_trade_events(); + test_quiet_clock_does_not_fill_pending_market_order(); + test_process_on_close_uses_last_trade_provenance(); + test_unsupported_profiles_reject_before_warmup(); + test_replacement_gets_fresh_identity_but_keeps_priority(); + test_realtime_trailing_state_is_per_order_leg(); + test_lifecycle_report_is_deterministic_across_batching(); test_rejects_replayed_or_out_of_order_ticks(); if (failures == 0) { diff --git a/tutorial/README.md b/tutorial/README.md index e49e00b..4877d4e 100644 --- a/tutorial/README.md +++ b/tutorial/README.md @@ -64,6 +64,10 @@ example expands each live candle into a deterministic OHLC trade path. Replace that expansion with your exchange feed in production. The public API and state lifecycle remain the same. +The example also prints the canonical lifecycle count/hash and the first few +command-revision, executable-leg, and fill identities. Build JSON webhook +payloads above this simulator API and deduplicate delivery on `fill_id`. + ## Path B — Docker (no local toolchain) Mount the strategy + OHLCV into the published runtime image; get a diff --git a/tutorial/run.py b/tutorial/run.py index bd76440..749d862 100755 --- a/tutorial/run.py +++ b/tutorial/run.py @@ -84,6 +84,36 @@ class _Trace(ctypes.Structure): _fields_ = [("timestamp", ctypes.c_int64), ("bar_index", ctypes.c_int32), ("name_id", ctypes.c_int32), ("value", ctypes.c_double)] +class OrderEventC(ctypes.Structure): + _fields_ = [ + ("transition_sequence", ctypes.c_uint64), + ("command_revision_id", ctypes.c_uint64), + ("order_leg_id", ctypes.c_uint64), + ("priority_sequence", ctypes.c_uint64), + ("fill_id", ctypes.c_uint64), ("entry_lot_id", ctypes.c_uint64), + ("position_episode_id", ctypes.c_uint64), + ("event_timestamp", ctypes.c_int64), + ("event_sequence", ctypes.c_uint64), + ("input_bar_index", ctypes.c_int64), + ("script_bar_index", ctypes.c_int32), + ("command_kind", ctypes.c_int32), ("leg_kind", ctypes.c_int32), + ("state_before", ctypes.c_int32), ("state_after", ctypes.c_int32), + ("transition", ctypes.c_int32), ("reason", ctypes.c_int32), + ("side", ctypes.c_int32), ("oca_type", ctypes.c_int32), + ("requested_quantity", ctypes.c_double), + ("remaining_quantity", ctypes.c_double), + ("filled_quantity", ctypes.c_double), + ("observed_price", ctypes.c_double), ("stop_price", ctypes.c_double), + ("limit_price", ctypes.c_double), + ("trail_activation_price", ctypes.c_double), + ("trail_watermark", ctypes.c_double), ("fill_price", ctypes.c_double), + ("position_size_before", ctypes.c_double), + ("position_size_after", ctypes.c_double), + ("equity_before", ctypes.c_double), ("equity_after", ctypes.c_double), + ("id", ctypes.c_char_p), ("from_entry", ctypes.c_char_p), + ("oca_name", ctypes.c_char_p), + ] + class ReportC(ctypes.Structure): _fields_ = [("total_trades", ctypes.c_int), ("trades", ctypes.POINTER(TradeC)), ("trades_len", ctypes.c_int), @@ -107,12 +137,17 @@ class ReportC(ctypes.Structure): ("trace_names_len", ctypes.c_int), ("metrics", MetricsC), ("equity_curve", ctypes.POINTER(EquityPointC)), - ("equity_curve_len", ctypes.c_int64)] # int64, NOT c_int + ("equity_curve_len", ctypes.c_int64), + ("order_events", ctypes.POINTER(OrderEventC)), + ("order_events_len", ctypes.c_int64), + ("order_event_count", ctypes.c_uint64), + ("order_event_hash", ctypes.c_uint64), + ("order_event_dropped", ctypes.c_uint64)] # pf_report_t is caller-allocated, so a stale mirror means the runtime # writes past our buffer. Assert the .so's ABI version before any run. -EXPECTED_PF_ABI = 2 +EXPECTED_PF_ABI = 3 def check_abi(lib: ctypes.CDLL) -> None: try: diff --git a/tutorial/run_stream.py b/tutorial/run_stream.py index f4908a6..245bcf7 100644 --- a/tutorial/run_stream.py +++ b/tutorial/run_stream.py @@ -89,6 +89,8 @@ def main() -> int: ctypes.c_void_p, ctypes.POINTER(BarC), ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p] lib.strategy_stream_begin.restype = ctypes.c_int + lib.strategy_stream_set_gap_policy.argtypes = [ctypes.c_void_p, ctypes.c_int] + lib.strategy_stream_set_gap_policy.restype = ctypes.c_int lib.strategy_stream_push_ticks.argtypes = [ ctypes.c_void_p, ctypes.POINTER(TradeTickC), ctypes.c_int] lib.strategy_stream_push_ticks.restype = ctypes.c_int @@ -106,6 +108,8 @@ def main() -> int: sys.exit("strategy_create failed") report = ReportC() try: + check_call(lib, state, lib.strategy_stream_set_gap_policy( + state, 0), "set fixed-grid gap policy") check_call(lib, state, lib.strategy_stream_begin( state, bars, warmup_n, b"15", b"15"), "stream begin") @@ -135,6 +139,14 @@ def main() -> int: f"{report.script_bars_processed} script bars") print(f" trades: {report.trades_len}") print(f" net pnl: {report.net_profit:+.2f}") + print(f" order log: {report.order_event_count} transitions, " + f"hash={report.order_event_hash:016x}") + for i in range(min(5, report.order_events_len)): + event = report.order_events[i] + event_id = event.id.decode("utf-8", "replace") if event.id else "" + print(f" #{event.transition_sequence} id={event_id} " + f"rev={event.command_revision_id} leg={event.order_leg_id} " + f"transition={event.transition} fill={event.fill_id}") except RuntimeError as exc: print(f"stream error: {exc}", file=sys.stderr) return 1 From 7d45b50cb4b7b52993979b77a33a56a87f75a437 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sat, 11 Jul 2026 07:59:45 +0800 Subject: [PATCH 4/5] Add TradingView realtime probe kit --- docs/README.md | 2 + docs/design/realtime-execution-audit.md | 51 ++-- docs/probes/tradingview-realtime/DECISIONS.md | 144 ++++++++++ docs/probes/tradingview-realtime/README.md | 153 ++++++++++ .../manifest.template.json | 58 ++++ .../p1_cross_command_id.pine | 68 +++++ .../p2_replacement_priority.pine | 68 +++++ .../p3_stop_limit_reissue.pine | 93 ++++++ .../p4_trailing_reissue.pine | 142 ++++++++++ .../p5_exit_without_from_entry.pine | 93 ++++++ .../p6_from_entry_cutoff.pine | 82 ++++++ .../p7_simultaneous_priority.pine | 78 +++++ .../p8_oca_same_event.pine | 82 ++++++ .../p9_per_lot_stop_selection.pine | 161 +++++++++++ .../tradingview-realtime/results.template.md | 39 +++ .../tradingview-realtime/validate_capture.py | 267 ++++++++++++++++++ docs/tv-parity-probe-spec.md | 5 + 17 files changed, 1567 insertions(+), 19 deletions(-) create mode 100644 docs/probes/tradingview-realtime/DECISIONS.md create mode 100644 docs/probes/tradingview-realtime/README.md create mode 100644 docs/probes/tradingview-realtime/manifest.template.json create mode 100644 docs/probes/tradingview-realtime/p1_cross_command_id.pine create mode 100644 docs/probes/tradingview-realtime/p2_replacement_priority.pine create mode 100644 docs/probes/tradingview-realtime/p3_stop_limit_reissue.pine create mode 100644 docs/probes/tradingview-realtime/p4_trailing_reissue.pine create mode 100644 docs/probes/tradingview-realtime/p5_exit_without_from_entry.pine create mode 100644 docs/probes/tradingview-realtime/p6_from_entry_cutoff.pine create mode 100644 docs/probes/tradingview-realtime/p7_simultaneous_priority.pine create mode 100644 docs/probes/tradingview-realtime/p8_oca_same_event.pine create mode 100644 docs/probes/tradingview-realtime/p9_per_lot_stop_selection.pine create mode 100644 docs/probes/tradingview-realtime/results.template.md create mode 100644 docs/probes/tradingview-realtime/validate_capture.py diff --git a/docs/README.md b/docs/README.md index 65036a0..96a4f64 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,8 @@ docs/ ├── Doxyfile # Doxygen config — input, theme, options ├── build.sh # one-shot: fetch theme, run doxygen → docs/site/html/ ├── coverage.md # canonical Pine v6 coverage map (also referenced as a page) +├── probes/ # manually captured external semantic oracles +│ └── tradingview-realtime/ # P1-P9 Pine v6 JSON/webhook probe kit ├── pages/ # narrative markdown pages │ ├── index.md # @mainpage — landing │ ├── getting-started.md diff --git a/docs/design/realtime-execution-audit.md b/docs/design/realtime-execution-audit.md index f892210..d348ff5 100644 --- a/docs/design/realtime-execution-audit.md +++ b/docs/design/realtime-execution-audit.md @@ -703,25 +703,38 @@ TradingView probes". Those probes are scheduled work with a defined method, not a hand-wave; the completion gate requires every one resolved and archived. -Method: every probe below concerns deterministic broker-emulator behavior and -is observable in historical Strategy Tester exports — the corpus method this -repository already uses — so no live-session observation is required for v1. -Each probe is one minimal Pine script plus its exported trade list, archived -as a corpus probe under `corpus/validation/` with the pinned export, and its -conclusion recorded in this document before the dependent implementation -lands. - -| # | Open semantic | Prior / partial evidence | -|---|---|---| -| P1 | Public-ID collision across command kinds on placement (entry vs raw order vs exit sharing a text ID) | Exit-vs-entry independence already pinned in-repo by the `clear_existing_exit_order` regression; remaining: entry-vs-raw-order and placement-collision direction | -| P2 | Whether replacement preserves or resets broker priority (`priority_sequence`) | None | -| P3 | Stop-limit activation persistence across a same-key reissue (unchanged vs changed parameters) | No counterpart precedent exists; TradingView docs pin durability without reissue only | -| P4 | Trailing activation/watermark persistence across a reissue | PyneCore's TV-probed parameter-equality rule is the prior | -| P5 | `strategy.exit()` without `from_entry`: persistence for later entry lots | Docs give the creation-time cutoff for `from_entry`; the no-`from_entry` persistence needs an export | -| P6 | `strategy.exit(..., from_entry = X)` creation-time cutoff edge cases (same-bar entry, reissue) | Docs pin the base rule | -| P7 | Simultaneous-event priority and per-fill eligibility recomputation (carried market vs price-based entry vs exit vs trail vs OCA vs reversal vs risk/margin) | Partial coverage by existing corpus probes; needs targeted coincident-level probes | -| P8 | OCA same-event multi-trigger: does TV cancel before the second same-tick fill? | TV docs say close-priced same-group orders "may be filled"; historical kernel encodes multi-fill; decide divergence or parity | -| P9 | Per-lot stop-type working-order selection when entry-relative levels differ by lot | Three-way bracket probe pins the single-lot case | +Method: the runnable manual-capture kit lives in +`docs/probes/tradingview-realtime/`. Each Pine v6 strategy executes only on +realtime bars and emits sparse JSON trace milestones plus native TradingView +order-fill alerts. The observer must archive the webhook JSONL, TradingView +alert log, Strategy Tester trade export, chart/settings screenshot, and a +fully pinned manifest. Webhook delivery alone is not authoritative: missing or +reordered HTTP requests are reconciled against TradingView's own alert log and +trade export. + +These captures are sparse semantic evidence, not the general regression +oracle. PineForge's deterministic state digests, lifecycle invariants, +tick-to-OHLCV reconciliation, repeat-run hashes, and corpus sweeps remain the +primary verification system. Once a probe conclusion is repeatable, archive +the result and promote a minimized historical form into `corpus/validation/` +when the behavior is also observable in Strategy Tester exports. + +| # | Open semantic | Manual probe / required runs | Prior / partial evidence | +|---|---|---|---| +| P1 | Public-ID collision across command kinds on placement (entry vs raw order vs exit sharing a text ID) | `p1_cross_command_id.pine`; both placement-order modes | Exit-vs-entry independence already pinned in-repo by the `clear_existing_exit_order` regression; remaining: entry-vs-raw-order and placement-collision direction | +| P2 | Whether replacement preserves or resets broker priority (`priority_sequence`) | `p2_replacement_priority.pine`; repeat the same-price run | None | +| P3 | Stop-limit activation persistence across a same-key reissue (unchanged vs changed parameters) | `p3_stop_limit_reissue.pine`; unchanged and changed-limit modes | No counterpart precedent exists; TradingView docs pin durability without reissue only | +| P4 | Trailing activation/watermark persistence across a reissue | `p4_trailing_reissue.pine`; unchanged and changed-offset modes | PyneCore's TV-probed parameter-equality rule is the prior | +| P5 | `strategy.exit()` without `from_entry`: persistence for later entry lots | `p5_exit_without_from_entry.pine`; no-reissue run plus explicit post-B reissue control | Current official docs explicitly say the call persists for later entries until the position closes; the probe is version-pinned corroboration, not an undocumented rule | +| P6 | `strategy.exit(..., from_entry = X)` creation-time cutoff edge cases (same-bar entry, reissue) | `p6_from_entry_cutoff.pine`; all three call-order/reissue modes | Docs pin exclusion of entries created after the call bar; the same-calculation source-order/reissue edge remains useful to capture | +| P7 | Simultaneous-event priority and per-fill eligibility recomputation | `p7_simultaneous_priority.pine`; both exit/reversal source orders | Partial coverage by existing corpus probes; this first capture pins the market-reversal versus marketable-exit collision before expanding to risk/margin variants | +| P8 | OCA same-event multi-trigger: does TV cancel before the second same-tick fill? | `p8_oca_same_event.pine`; both sibling source orders | Current Pine v6 reference says same-tick OCA siblings cannot cancel/reduce one another; the probe is version-pinned realtime corroboration of documented multi-fill behavior | +| P9 | Per-lot stop-type working-order selection when entry-relative levels differ by lot | `p9_per_lot_stop_selection.pine`; two same-ID lots at different prices | Three-way bracket probe pins the single-lot case; webhook plus List of Trades is required to allocate the same-public-ID fills | + +Probe source files, the exact alert template, capture instructions, manifest, +and result template are versioned together. Manual observations are still +pending; adding the kit does not resolve P1-P9 or close the implementation +gate. Section 1's `command_key` schema is provisional until P1 concludes. diff --git a/docs/probes/tradingview-realtime/DECISIONS.md b/docs/probes/tradingview-realtime/DECISIONS.md new file mode 100644 index 0000000..bdd15e2 --- /dev/null +++ b/docs/probes/tradingview-realtime/DECISIONS.md @@ -0,0 +1,144 @@ +# Pre-registered probe decisions + +Write the chosen mode and thresholds into `results.md` before the scheduled +start. The signatures below classify only clean runs. Any invalidating sequence +means repeat the run; it is not evidence for either outcome. + +## P1 — cross-command public ID + +- Both `entry_call` and `raw_order_call` fill at the shared trigger: entry and + raw-order namespaces coexist for that placement order. +- Only the second call's tag fills, with the result reversing between the two + modes: last writer replaces across command kinds. +- Only the first call fills in both modes: first writer wins. +- Invalid: timeout/cleanup, a non-probe position, or only one placement mode. + +This probe does not re-test exit-vs-entry independence or cancel semantics, +which already have repository evidence; it resolves the remaining entry versus +raw-order collision. + +## P2 — replacement priority + +`A_initial`, then `B_unchanged`, then `A_reissued` must be observed before a +single market event makes both A and B eligible at the same price. + +- `A_reissued` precedes `B_unchanged` in TradingView's alert log on repeated + clean runs: replacement preserved A's original priority. +- `B_unchanged` precedes `A_reissued`: replacement reset A behind B. +- Invalid: `inconclusive_fill_before_reissue`, different fill prices/times, + only one sibling filling, or reliance on webhook arrival order alone. + +## P3 — stop-limit activation across reissue + +Required trace prefix: + +```text +stop_limit_placed -> stop_price_observed -> reissued_between_stop_and_limit +``` + +- Fill at the limit on the same market update represented by + `limit_observed_before_stop_recross`: activation persisted across the reissue. + The broker fill alert may precede the script trace because broker processing + and every-tick script calculation have separate event ordering. +- No fill on that first limit descent, followed by + `post_reissue_stop_recross_observed` and a later limit fill: activation reset. +- Invalid: reissue not strictly between stop and limit, fill before reissue, + missing price milestones, or a gap/tick that makes the ordering ambiguous. + +Apply this rule separately to unchanged and changed-limit reissues. + +## P4 — trailing watermark across reissue + +At `trail_reissued_on_discriminating_retrace`, preserve the emitted +`pre_reissue_watermark`, `preserved_stop_candidate`, and +`reset_stop_candidate`. + +- Fill when the preserved candidate is crossed but before the reset candidate: + activation/watermark persisted. +- No fill at the preserved candidate, then fill only when the reset candidate + is crossed: the reissue reset the watermark. +- If a reset order must reactivate, use + `post_reissue_reset_activation_tick_observed` as the reset watermark source; + the creation-close candidate is not authoritative. +- Invalid: `inconclusive_post_reissue_new_high`, fill before reissue, a gap that + crosses both candidates in one event, or missing candidate values. + +Apply this rule separately to unchanged and changed-offset reissues. + +## P5 — no-`from_entry` persistence + +The initial no-`from_entry` exit must be created while only A is open. B must +then open without reissuing that exit. + +- In `persistent_no_reissue`, the exit closes both covered lots; compare its + quantities and trade rows with `positive_control_reissue_after_B`. +- B remains uncovered despite the price crossing B's applicable exit level, + as proven by `tv-bars.csv`, while the positive control covers it: the exit + did not persist. +- Invalid: A exits before B opens, no applicable level is crossed, timeout, or + the positive control does not produce the expected covered exit. + +The current TradingView documentation already states persistence. This is a +version-pinned corroboration/control probe. + +## P6 — `from_entry` creation cutoff + +Compare all three modes using identical distances: + +- `entry_then_exit`: records whether an exit called after its entry in the same + confirmed calculation covers that pending entry. +- `exit_then_entry`: records whether the reverse source order covers it. +- `exit_then_entry_reissue_after_fill`: positive control for coverage after a + matching entry exists. + +A fill is evidence only after the corresponding profit/loss level is reached. +Use `tv-bars.csv` to prove a crossing for any negative conclusion. Timeout or +absence of a discriminating crossing is inconclusive. + +## P7 — simultaneous exit/reversal priority + +For each source-order mode, require the initial long fill, the +`collision_orders_created` trace, and fills sharing one fill-bar time. + +Record the TradingView alert-log fill order, transaction quantities, closed +trade rows, and final position. The repeated result is the rule; this probe does +not pre-assume exit-first or reversal-first behavior. Different results between +repeats are inconclusive/nondeterministic evidence. + +## P8 — OCA same-event cancellation + +The current Pine v6 reference states that OCA siblings executing on the same +tick cannot cancel/reduce one another. This probe is version-pinned realtime +corroboration, not an undocumented semantic. + +For each source-order mode, require both siblings at the same stop and the same +eligible market event. + +- Exactly one of `OCA_A` / `OCA_B` fills: OCA cancellation removes the sibling + before a second same-event fill. +- Both fill at the same fill-bar time/price: TradingView admits same-event + multi-fill before cancellation takes effect. +- Invalid: different eligibility events, timeout, or disagreement between + TradingView alert log and trade export. + +## P9 — per-lot stop-type selection + +Require two same-ID lots whose actual `strategy.opentrades.entry_price()` values +meet the configured minimum separation. Preserve the setup trace containing +each entry time/price, fixed stop, trail activation, watermark, and trailing +candidate. + +Map every exit row back to its entry time/price in the List of Trades. Lot 0 has +quantity 1 and lot 1 has quantity 2, so `strategy.order.contracts` plus the +remaining open quantity identifies the child leg. Use the `fixed_stop_fill` +versus `trailing_stop_fill` tag and the traced candidate path to identify the +selected stop type. + +- A full per-lot conclusion requires one natural child fill while the other + quantity demonstrably remains open, followed by a second natural, + discriminating child fill. Cleanup cannot establish the remaining lot's + selected stop type; after cleanup the result is limited to the first mapped + lot or remains inconclusive. Each observed fill must satisfy only the mapped + lot's candidate allocation. +- Invalid: `inconclusive_lot_prices_too_close`, no discriminating candidate + path, timeout, a reload/repaint, or inability to map the exit row to one lot. diff --git a/docs/probes/tradingview-realtime/README.md b/docs/probes/tradingview-realtime/README.md new file mode 100644 index 0000000..1217b89 --- /dev/null +++ b/docs/probes/tradingview-realtime/README.md @@ -0,0 +1,153 @@ +# TradingView realtime broker probes + +These Pine v6 strategies answer the nine open broker-emulator questions in +`docs/design/realtime-execution-audit.md`. They are sparse semantic probes, not +a general TradingView oracle. PineForge's deterministic replay and invariant +tests remain the primary regression gate. + +The probes emit two JSON-only event families: + +- `pf-tv-probe-trace-v1` from `alert()` records command creation and selected + price milestones. +- `pf-tv-probe-fill-v1` from TradingView order-fill alerts records the actual + broker fill, wrapping the order's `alert_message` object. + +TradingView references used by this protocol: + +- [Alerts](https://www.tradingview.com/pine-script-docs/concepts/alerts/) +- [Strategy order-fill alerts](https://www.tradingview.com/pine-script-docs/concepts/strategies/#strategy-alerts) +- [Execution model](https://www.tradingview.com/pine-script-docs/language/execution-model/) +- [Webhook configuration](https://www.tradingview.com/support/solutions/43000529348-how-to-configure-webhook-alerts/) + +## Fixed capture setup + +Use one probe and one alert at a time. + +1. Use a standard candlestick chart, not Heikin Ashi, Renko, or another + synthetic chart. +2. Prefer a liquid 24x7 symbol on a 1-minute chart. Record the exact + `syminfo.tickerid`; do not normalize it by hand. +3. Set the chart timezone to UTC. Do not override any strategy property after + adding the script. +4. Set `Run ID` to a unique token containing only letters, digits, `.`, `_`, or + `-`. The scripts insert it into JSON without escaping. +5. Set `Scheduled start (UTC)` at least five minutes in the future. This keeps + the chart strategy and TradingView's server-side alert snapshot synchronized. +6. Select the desired probe mode and distances before creating the alert. +7. Add the strategy, then create a strategy alert with both **Order fills and + alert() function calls** enabled. TradingView snapshots the script and its + inputs when the alert is created; after any change, delete and recreate it. +8. Configure the webhook URL and paste the fill message template below exactly. +9. Keep `Arm at scheduled time` enabled. Historical bars never place probe + orders, and commands wait until a later confirmed bar. A run is valid only + when TradingView's alert log shows `armed` before every command/fill; receiver + arrival order is not authoritative. +10. Capture until the expected fill or the probe's timeout trace. Export + **Strategy Tester -> List of Trades** as CSV after the run. + +Fill alert message template: + +```json +{"schema":"pf-tv-probe-fill-v1","probe_payload":{{strategy.order.alert_message}},"ticker":"{{ticker}}","interval":"{{interval}}","fill_bar_time":"{{time}}","server_time":"{{timenow}}","order_id":"{{strategy.order.id}}","action":"{{strategy.order.action}}","contracts":{{strategy.order.contracts}},"price":{{strategy.order.price}},"position_size":{{strategy.position_size}}} +``` + +The unexpanded template is not JSON; the delivered webhook body is. Every +probe supplies `strategy.order.alert_message` as a JSON object, not a quoted +string. `probe_payload.command_bar_*` identifies the calculation that created +or reissued the order; outer `fill_bar_time` and `server_time` identify the +actual fill notification. Use fill-bar open time—not raw `bar_index`—for +bar-level regression comparisons. + +## Required artifacts + +Store one directory per run: + +```text +P3-unchanged-2026-07-12T0100Z/ + manifest.json + webhook.jsonl + receipt.jsonl # receiver sequence/time/body hash; no TV fields added + alert-message.txt # exact fill template pasted into TradingView + tv-alert-log.csv # export/copy when available + tv-trades.csv + tv-bars.csv # exported chart OHLCV covering the observation window + deployed-source.pine # exact source used by the alert snapshot + chart.png # chart plus Strategy Properties + notes.md + results.md # completed copy of results.template.md +``` + +Copy `manifest.template.json`, fill every field, and record SHA-256 hashes after +capture. Preserve each raw TradingView body in arrival order in `webhook.jsonl`, +one delivered JSON object per line. Keep receiver sequence/timestamp/body-hash +metadata in `receipt.jsonl`; do not inject receiver fields into TradingView's +payload. Preserve an invalid raw body separately instead of normalizing it. +Also copy/export TradingView's alert log. Webhook delivery is +not by itself an oracle: a missing HTTP delivery must be reconciled against the +TradingView alert log and Strategy Tester before concluding that no event +occurred. + +Export chart OHLCV for the full observation interval as `tv-bars.csv`. P5/P6 +negative conclusions require exact evidence that an applicable level was +crossed without a fill; a screenshot is insufficient. + +Validate JSON shape, run/probe identity, event count, and the webhook hash: + +```bash +python3 docs/probes/tradingview-realtime/validate_capture.py /path/to/run-directory +``` + +The validator also requires every listed artifact and verifies recorded hashes, +receipt/body correspondence, scheduled arming, and deployed source identity. A +pass means the capture is complete and structurally consistent; it does not +replace the manual semantic cross-check in `results.md`. + +## Probe matrix + +| Probe | Script | Required runs | Deciding observation | +|---|---|---|---| +| P1 | `p1_cross_command_id.pine` | both placement-order modes | Which tagged entry/raw-order calls survive and fill under one public ID | +| P2 | `p2_replacement_priority.pine` | at least two successful collisions | Whether reissued `A` fills before or after unchanged sibling `B` at one price | +| P3 | `p3_stop_limit_reissue.pine` | `unchanged`, `changed_limit` | Whether an activated stop-limit remains limit-active after reissue without a second stop crossing | +| P4 | `p4_trailing_reissue.pine` | `unchanged`, `changed_offset` | Whether the post-reissue exit uses the pre-reissue activation/watermark | +| P5 | `p5_exit_without_from_entry.pine` | no-reissue and positive-control modes | Whether an exit created for lot A also covers later lot B; explicit post-B reissue is the control | +| P6 | `p6_from_entry_cutoff.pine` | all three modes | Whether same-calculation call order and a later reissue change coverage | +| P7 | `p7_simultaneous_priority.pine` | two successful collisions per source-order mode | Fill order and final position when a market reversal and marketable exit become eligible together | +| P8 | `p8_oca_same_event.pine` | two successful collisions per source-order mode | Whether one or both same-price OCA-cancel siblings fill | +| P9 | `p9_per_lot_stop_selection.pine` | one run with distinct lot prices | Trade export reveals the effective stop-type leg selected independently for each same-ID lot | + +For P2, P7, and P8, webhook arrival order is only supporting evidence. Confirm +the order in TradingView's alert log and the List of Trades because unrelated +network latency can reorder requests at the receiver. + +P1, P2, P5, P6, P7, and P8 use TradingView's default close-only calculation +profile. P3 and P4 use `calc_on_every_tick` only to record the price path around +orders that are still created/reissued exclusively on confirmed closes. P9 +uses it to record per-lot activation and watermark candidates. These traces +observe broker semantics; PineForge realtime v1 does not claim to implement +TradingView rollback/every-tick strategy execution. + +## Classification rule + +Record observations before writing a conclusion. A TradingView result is +accepted only when: + +- the manifest pins script hash, symbol, timeframe, chart type, strategy + properties, inputs, alert creation time, and observation window; +- trace milestones show that the intended price path occurred; +- fill events agree between webhook, TradingView alert log, and trade export; +- a repeat run reaches the same semantic conclusion when the question involves + same-event priority. + +Timeout, an early fill, a missing `armed` trace, a post-reissue path that crosses +both candidate levels at once, or a chart reload makes the run inconclusive. +Never edit/reload the script or change symbol/timeframe before exporting the +Strategy Tester result; realtime every-tick calculations can repaint after a +reload. + +Use `results.template.md` to summarize the evidence. Do not infer activation or +cancellation solely from the absence of a fill. + +Before each run, copy the matching confirming/disconfirming/invalidating +signatures from [`DECISIONS.md`](DECISIONS.md) into the result file. This keeps +the interpretation rule fixed before observing the market path. diff --git a/docs/probes/tradingview-realtime/manifest.template.json b/docs/probes/tradingview-realtime/manifest.template.json new file mode 100644 index 0000000..f83676a --- /dev/null +++ b/docs/probes/tradingview-realtime/manifest.template.json @@ -0,0 +1,58 @@ +{ + "schema": "pf-tv-probe-manifest-v1", + "probe": "P0", + "mode": "replace-me", + "run_id": "P0-mode-YYYYMMDDTHHMMSSZ", + "script_file": "p0_replace_me.pine", + "script_sha256": "replace-me", + "pine_version": "6", + "tradingview_plan": "replace-me", + "tickerid": "replace-me", + "timeframe": "1", + "chart_type": "candles", + "chart_timezone": "UTC", + "session": "replace-me", + "strategy_properties": { + "pyramiding": 1, + "process_orders_on_close": false, + "calc_on_every_tick": false, + "calc_on_order_fills": false, + "bar_magnifier": false + }, + "inputs": {}, + "scheduled_start_utc": "YYYY-MM-DDTHH:MM:SSZ", + "alert": { + "alert_id": "replace-me", + "created_at_utc": "YYYY-MM-DDTHH:MM:SSZ", + "condition": "order fills and alert() function calls", + "message_template_sha256": "replace-me", + "webhook_receiver": "redacted-host-label" + }, + "observation": { + "armed_at_utc": "YYYY-MM-DDTHH:MM:SSZ", + "started_at_utc": "YYYY-MM-DDTHH:MM:SSZ", + "ended_at_utc": "YYYY-MM-DDTHH:MM:SSZ", + "webhook_event_count": 0, + "tradingview_alert_event_count": 0, + "trade_count": 0, + "receiver_first_sequence": 0, + "receiver_last_sequence": 0, + "chart_reloaded_or_changed": false + }, + "artifacts": { + "webhook_jsonl_sha256": "replace-me", + "receipt_jsonl_sha256": "replace-me", + "tv_alert_log_sha256": "replace-me", + "tv_trades_csv_sha256": "replace-me", + "tv_bars_csv_sha256": "replace-me", + "chart_png_sha256": "replace-me", + "notes_md_sha256": "replace-me", + "results_md_sha256": "replace-me" + }, + "deployed_source_sha256": "replace-me", + "result": { + "status": "inconclusive", + "conclusion": "replace-me", + "ambiguities": [] + } +} diff --git a/docs/probes/tradingview-realtime/p1_cross_command_id.pine b/docs/probes/tradingview-realtime/p1_cross_command_id.pine new file mode 100644 index 0000000..d48ee68 --- /dev/null +++ b/docs/probes/tradingview-realtime/p1_cross_command_id.pine @@ -0,0 +1,68 @@ +//@version=6 +strategy("PF TV P1 cross-command public ID", overlay=true, pyramiding=10, + calc_on_every_tick=false, calc_on_order_fills=false, + process_orders_on_close=false) + +string PROBE = "P1" +string runId = input.string("P1-replace-me", "Run ID") +int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), + "Scheduled start (UTC; set 5+ min ahead)") +string mode = input.string("entry_then_order", "Placement order", + options=["entry_then_order", "order_then_entry"]) +int triggerTicks = input.int(1000, "Buy-stop distance (ticks)", minval=1) +int timeoutBars = input.int(120, "Timeout bars", minval=2) +bool arm = input.bool(true, "Arm at scheduled time") + +trace(string tag) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +fill(string tag) => + '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","order_tag":"' + tag + + '","command_bar_index":' + str.tostring(bar_index) + + ',"command_bar_time_ms":' + str.tostring(time) + '}' + +var int liveBars = 0 +var float trigger = na +var bool placed = false +var bool timedOut = false +varip bool armed = false +var int armedBar = na + +bool active = arm and barstate.isrealtime and timenow >= scheduledStart +if active and not armed + armed := true + armedBar := bar_index + alert(trace("armed"), alert.freq_all) + +if active and barstate.isconfirmed and bar_index > armedBar + liveBars += 1 + if not placed + placed := true + trigger := close + triggerTicks * syminfo.mintick + alert(trace("placing_same_id_commands"), alert.freq_all) + if mode == "entry_then_order" + strategy.entry("SAME", strategy.long, qty=1, stop=trigger, + alert_message=fill("entry_call")) + strategy.order("SAME", strategy.long, qty=1, stop=trigger, + alert_message=fill("raw_order_call")) + else + strategy.order("SAME", strategy.long, qty=1, stop=trigger, + alert_message=fill("raw_order_call")) + strategy.entry("SAME", strategy.long, qty=1, stop=trigger, + alert_message=fill("entry_call")) + else if not timedOut and liveBars >= timeoutBars + timedOut := true + strategy.cancel_all() + if strategy.position_size != 0 + strategy.close_all(immediately=true, + alert_message=fill("timeout_close_all")) + alert(trace("timeout_cleanup"), alert.freq_all) + +plot(trigger, "shared trigger", color=color.orange) diff --git a/docs/probes/tradingview-realtime/p2_replacement_priority.pine b/docs/probes/tradingview-realtime/p2_replacement_priority.pine new file mode 100644 index 0000000..d9a26d7 --- /dev/null +++ b/docs/probes/tradingview-realtime/p2_replacement_priority.pine @@ -0,0 +1,68 @@ +//@version=6 +strategy("PF TV P2 replacement priority", overlay=true, pyramiding=10, + calc_on_every_tick=false, calc_on_order_fills=false, + process_orders_on_close=false) + +string PROBE = "P2" +string runId = input.string("P2-replace-me", "Run ID") +int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), + "Scheduled start (UTC; set 5+ min ahead)") +int triggerTicks = input.int(5000, "Shared buy-stop distance (ticks)", minval=1) +int timeoutBars = input.int(240, "Timeout bars", minval=3) +bool arm = input.bool(true, "Arm at scheduled time") + +trace(string tag) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +fill(string tag) => + '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","order_tag":"' + tag + + '","command_bar_index":' + str.tostring(bar_index) + + ',"command_bar_time_ms":' + str.tostring(time) + '}' + +var int liveBars = 0 +var float trigger = na +var bool reissued = false +var bool timedOut = false +var bool aborted = false +varip bool armed = false +var int armedBar = na + +bool active = arm and barstate.isrealtime and timenow >= scheduledStart +if active and not armed + armed := true + armedBar := bar_index + alert(trace("armed"), alert.freq_all) + +if active and barstate.isconfirmed and bar_index > armedBar + liveBars += 1 + if liveBars == 1 + trigger := close + triggerTicks * syminfo.mintick + strategy.order("A", strategy.long, qty=1, stop=trigger, + alert_message=fill("A_initial")) + strategy.order("B", strategy.long, qty=1, stop=trigger, + alert_message=fill("B_unchanged")) + alert(trace("A_then_B_placed"), alert.freq_all) + else if liveBars == 2 and strategy.position_size == 0 + reissued := true + strategy.order("A", strategy.long, qty=1, stop=trigger, + alert_message=fill("A_reissued")) + alert(trace("A_reissued_after_B"), alert.freq_all) + else if liveBars == 2 and strategy.position_size != 0 and not aborted + aborted := true + alert(trace("inconclusive_fill_before_reissue"), alert.freq_all) + else if not timedOut and liveBars >= timeoutBars + timedOut := true + strategy.cancel_all() + if strategy.position_size != 0 + strategy.close_all(immediately=true, + alert_message=fill("timeout_close_all")) + alert(trace("timeout_cleanup"), alert.freq_all) + +plot(trigger, "shared trigger", color=reissued ? color.aqua : color.orange) diff --git a/docs/probes/tradingview-realtime/p3_stop_limit_reissue.pine b/docs/probes/tradingview-realtime/p3_stop_limit_reissue.pine new file mode 100644 index 0000000..7b8fbcb --- /dev/null +++ b/docs/probes/tradingview-realtime/p3_stop_limit_reissue.pine @@ -0,0 +1,93 @@ +//@version=6 +strategy("PF TV P3 stop-limit reissue", overlay=true, pyramiding=1, + calc_on_every_tick=true, calc_on_order_fills=false, + process_orders_on_close=false) + +string PROBE = "P3" +string runId = input.string("P3-replace-me", "Run ID") +int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), + "Scheduled start (UTC; set 5+ min ahead)") +string mode = input.string("unchanged", "Reissue mode", + options=["unchanged", "changed_limit"]) +int stopTicks = input.int(100, "Stop distance above placement (ticks)", minval=2) +int retraceTicks = input.int(50, "Limit distance below stop (ticks)", minval=1) +int changedLimitTicks = input.int(1, "Changed-limit delta (ticks)", minval=1) +int timeoutBars = input.int(240, "Timeout bars", minval=3) +bool arm = input.bool(true, "Arm at scheduled time") + +trace(string tag) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +fill(string tag) => + '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","order_tag":"' + tag + + '","command_bar_index":' + str.tostring(bar_index) + + ',"command_bar_time_ms":' + str.tostring(time) + '}' + +var int liveBars = 0 +var float stopPrice = na +var float limitPrice = na +varip bool sawStop = false +varip bool reissued = false +varip bool sawPostReissueLimit = false +varip bool sawPostReissueStopRecross = false +varip bool armed = false +varip int armedBar = na +var bool timedOut = false +var bool aborted = false + +bool active = arm and barstate.isrealtime and timenow >= scheduledStart +if active and not armed + armed := true + armedBar := bar_index + alert(trace("armed"), alert.freq_all) + +if active + if not na(stopPrice) and not sawStop and close >= stopPrice + sawStop := true + alert(trace("stop_price_observed"), alert.freq_all) + if reissued and not sawPostReissueStopRecross and close >= stopPrice + sawPostReissueStopRecross := true + alert(trace("post_reissue_stop_recross_observed"), alert.freq_all) + if reissued and sawStop and not sawPostReissueLimit and close <= limitPrice + sawPostReissueLimit := true + string limitTag = sawPostReissueStopRecross ? + "limit_observed_after_stop_recross" : + "limit_observed_before_stop_recross" + alert(trace(limitTag), alert.freq_all) + + if barstate.isconfirmed and bar_index > armedBar + liveBars += 1 + if liveBars == 1 + stopPrice := close + stopTicks * syminfo.mintick + limitPrice := stopPrice - retraceTicks * syminfo.mintick + strategy.entry("SL", strategy.long, qty=1, stop=stopPrice, + limit=limitPrice, alert_message=fill("initial_stop_limit")) + alert(trace("stop_limit_placed"), alert.freq_all) + else if sawStop and not reissued and strategy.position_size == 0 and + close < stopPrice and close > limitPrice + reissued := true + if mode == "changed_limit" + limitPrice -= changedLimitTicks * syminfo.mintick + strategy.entry("SL", strategy.long, qty=1, stop=stopPrice, + limit=limitPrice, alert_message=fill("reissued_stop_limit")) + alert(trace("reissued_between_stop_and_limit"), alert.freq_all) + else if sawStop and not reissued and strategy.position_size > 0 and not aborted + aborted := true + alert(trace("inconclusive_filled_before_reissue"), alert.freq_all) + else if not timedOut and liveBars >= timeoutBars + timedOut := true + strategy.cancel_all() + if strategy.position_size != 0 + strategy.close_all(immediately=true, + alert_message=fill("timeout_close_all")) + alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) + +plot(stopPrice, "stop", color=color.red) +plot(limitPrice, "limit", color=color.green) diff --git a/docs/probes/tradingview-realtime/p4_trailing_reissue.pine b/docs/probes/tradingview-realtime/p4_trailing_reissue.pine new file mode 100644 index 0000000..6919476 --- /dev/null +++ b/docs/probes/tradingview-realtime/p4_trailing_reissue.pine @@ -0,0 +1,142 @@ +//@version=6 +strategy("PF TV P4 trailing reissue", overlay=true, pyramiding=1, + calc_on_every_tick=true, calc_on_order_fills=false, + process_orders_on_close=false) + +string PROBE = "P4" +string runId = input.string("P4-replace-me", "Run ID") +int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), + "Scheduled start (UTC; set 5+ min ahead)") +string mode = input.string("unchanged", "Reissue mode", + options=["unchanged", "changed_offset"]) +int activationTicks = input.int(50, "Trail activation distance (ticks)", minval=1) +int offsetTicks = input.int(300, "Initial trail offset (ticks)", minval=2) +int changedOffsetTicks = input.int(100, "Changed-offset delta (ticks)", minval=1) +int timeoutBars = input.int(240, "Timeout bars", minval=4) +bool arm = input.bool(true, "Arm at scheduled time") + +trace(string tag) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +fill(string tag) => + '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","order_tag":"' + tag + + '","command_bar_index":' + str.tostring(bar_index) + + ',"command_bar_time_ms":' + str.tostring(time) + '}' + +traceCandidates(string tag, float watermark, float preservedLevel, + float resetLevel) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"pre_reissue_watermark":' + str.tostring(watermark, format.mintick) + + ',"preserved_stop_candidate":' + str.tostring(preservedLevel, format.mintick) + + ',"reset_stop_candidate":' + str.tostring(resetLevel, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +var int liveBars = 0 +var bool entryPlaced = false +varip bool trailPlaced = false +varip bool activationObserved = false +varip bool reissued = false +varip float observedWatermark = na +varip float preReissueWatermark = na +varip float preservedStopCandidate = na +varip float resetStopCandidate = na +varip bool crossedPreservedCandidate = false +varip bool crossedResetCandidate = false +varip bool postReissueNewHigh = false +varip bool resetActivationObserved = false +varip int reissueOffsetTicks = na +varip bool armed = false +varip int armedBar = na +var int trailPlacedBar = na +var bool timedOut = false +var bool aborted = false + +bool active = arm and barstate.isrealtime and timenow >= scheduledStart +if active and not armed + armed := true + armedBar := bar_index + alert(trace("armed"), alert.freq_all) + +if active + if trailPlaced and not reissued and strategy.position_size > 0 + observedWatermark := na(observedWatermark) ? close : math.max(observedWatermark, close) + float activationPrice = strategy.position_avg_price + activationTicks * syminfo.mintick + if not activationObserved and close >= activationPrice + activationObserved := true + alert(trace("trail_activation_price_observed"), alert.freq_all) + if reissued and strategy.position_size > 0 + float postReissueActivation = strategy.position_avg_price + + activationTicks * syminfo.mintick + if not resetActivationObserved and close >= postReissueActivation + resetActivationObserved := true + resetStopCandidate := close - reissueOffsetTicks * syminfo.mintick + alert(traceCandidates("post_reissue_reset_activation_tick_observed", + preReissueWatermark, preservedStopCandidate, resetStopCandidate), + alert.freq_all) + if not postReissueNewHigh and close > preReissueWatermark + postReissueNewHigh := true + alert(trace("inconclusive_post_reissue_new_high"), alert.freq_all) + if not crossedPreservedCandidate and close <= preservedStopCandidate + crossedPreservedCandidate := true + alert(traceCandidates("preserved_candidate_crossed", preReissueWatermark, + preservedStopCandidate, resetStopCandidate), alert.freq_all) + if resetActivationObserved and not crossedResetCandidate and + close <= resetStopCandidate + crossedResetCandidate := true + alert(traceCandidates("reset_candidate_crossed", preReissueWatermark, + preservedStopCandidate, resetStopCandidate), alert.freq_all) + + if barstate.isconfirmed and bar_index > armedBar + liveBars += 1 + if not entryPlaced + entryPlaced := true + strategy.entry("L", strategy.long, qty=1, + alert_message=fill("entry")) + alert(trace("market_entry_created"), alert.freq_all) + else if strategy.position_size > 0 and not trailPlaced + trailPlaced := true + trailPlacedBar := bar_index + strategy.exit("TX", "L", trail_points=activationTicks, + trail_offset=offsetTicks, + alert_message=fill("trail_initial")) + alert(trace("trail_initial_created"), alert.freq_all) + else if strategy.position_size > 0 and activationObserved and not reissued and + bar_index > trailPlacedBar and close < observedWatermark and + close > observedWatermark - offsetTicks * syminfo.mintick and + close >= strategy.position_avg_price + activationTicks * syminfo.mintick + reissued := true + int nextOffset = mode == "changed_offset" ? offsetTicks + changedOffsetTicks : offsetTicks + reissueOffsetTicks := nextOffset + preReissueWatermark := observedWatermark + preservedStopCandidate := preReissueWatermark - nextOffset * syminfo.mintick + resetStopCandidate := close - nextOffset * syminfo.mintick + strategy.exit("TX", "L", trail_points=activationTicks, + trail_offset=nextOffset, + alert_message=fill("trail_reissued")) + alert(traceCandidates("trail_reissued_on_discriminating_retrace", + preReissueWatermark, preservedStopCandidate, resetStopCandidate), + alert.freq_all) + else if trailPlaced and not reissued and strategy.position_size == 0 and not aborted + aborted := true + alert(trace("inconclusive_trail_filled_before_reissue"), alert.freq_all) + else if not timedOut and liveBars >= timeoutBars + timedOut := true + strategy.cancel("TX") + if strategy.position_size != 0 + strategy.close("L", immediately=true, + alert_message=fill("timeout_close")) + alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) + +plot(observedWatermark, "observed tick watermark", color=color.aqua) diff --git a/docs/probes/tradingview-realtime/p5_exit_without_from_entry.pine b/docs/probes/tradingview-realtime/p5_exit_without_from_entry.pine new file mode 100644 index 0000000..32d4c1e --- /dev/null +++ b/docs/probes/tradingview-realtime/p5_exit_without_from_entry.pine @@ -0,0 +1,93 @@ +//@version=6 +strategy("PF TV P5 exit without from_entry", overlay=true, pyramiding=2, + calc_on_every_tick=false, calc_on_order_fills=false, + process_orders_on_close=false) + +string PROBE = "P5" +string runId = input.string("P5-replace-me", "Run ID") +int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), + "Scheduled start (UTC; set 5+ min ahead)") +string mode = input.string("persistent_no_reissue", "Control mode", + options=["persistent_no_reissue", "positive_control_reissue_after_B"]) +int profitTicks = input.int(10000, "Exit profit distance (ticks)", minval=1) +int lossTicks = input.int(10000, "Exit loss distance (ticks)", minval=1) +int timeoutBars = input.int(360, "Timeout bars", minval=5) +bool arm = input.bool(true, "Arm at scheduled time") + +trace(string tag) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +fill(string tag) => + '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","order_tag":"' + tag + + '","command_bar_index":' + str.tostring(bar_index) + + ',"command_bar_time_ms":' + str.tostring(time) + '}' + +var int liveBars = 0 +var bool firstEntryPlaced = false +var bool exitPlaced = false +var bool secondEntryPlaced = false +var bool twoLotsObserved = false +var bool controlReissued = false +var int exitPlacedBar = na +var bool timedOut = false +var bool aborted = false +varip bool armed = false +var int armedBar = na + +bool active = arm and barstate.isrealtime and timenow >= scheduledStart +if active and not armed + armed := true + armedBar := bar_index + alert(trace("armed"), alert.freq_all) + +if active and barstate.isconfirmed and bar_index > armedBar + liveBars += 1 + if not firstEntryPlaced + firstEntryPlaced := true + strategy.entry("A", strategy.long, qty=1, + alert_message=fill("entry_A")) + alert(trace("entry_A_created"), alert.freq_all) + else if strategy.position_size >= 1 and not exitPlaced + exitPlaced := true + exitPlacedBar := bar_index + strategy.exit("X", profit=profitTicks, loss=lossTicks, + comment_profit="profit", comment_loss="loss", + alert_message=fill("exit_without_from_entry"), + alert_profit=fill("exit_profit"), alert_loss=fill("exit_loss")) + alert(trace("exit_without_from_entry_created_for_A"), alert.freq_all) + else if strategy.position_size >= 1 and exitPlaced and not secondEntryPlaced and + bar_index > exitPlacedBar + secondEntryPlaced := true + strategy.entry("B", strategy.long, qty=1, + alert_message=fill("entry_B_later")) + alert(trace("later_entry_B_created"), alert.freq_all) + else if strategy.position_size >= 2 and not twoLotsObserved + twoLotsObserved := true + if mode == "positive_control_reissue_after_B" + controlReissued := true + strategy.exit("X", profit=profitTicks, loss=lossTicks, + comment_profit="profit", comment_loss="loss", + alert_message=fill("positive_control_exit_reissued_after_B"), + alert_profit=fill("positive_control_profit"), + alert_loss=fill("positive_control_loss")) + alert(trace("positive_control_exit_reissued_after_B"), alert.freq_all) + else + alert(trace("two_lots_open_exit_not_reissued"), alert.freq_all) + else if exitPlaced and not secondEntryPlaced and strategy.position_size == 0 and + not aborted + aborted := true + alert(trace("inconclusive_A_exited_before_B"), alert.freq_all) + else if not timedOut and liveBars >= timeoutBars + timedOut := true + strategy.cancel_all() + if strategy.position_size != 0 + strategy.close_all(immediately=true, + alert_message=fill("timeout_close_all")) + alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) diff --git a/docs/probes/tradingview-realtime/p6_from_entry_cutoff.pine b/docs/probes/tradingview-realtime/p6_from_entry_cutoff.pine new file mode 100644 index 0000000..cfb5f8e --- /dev/null +++ b/docs/probes/tradingview-realtime/p6_from_entry_cutoff.pine @@ -0,0 +1,82 @@ +//@version=6 +strategy("PF TV P6 from_entry cutoff", overlay=true, pyramiding=1, + calc_on_every_tick=false, calc_on_order_fills=false, + process_orders_on_close=false) + +string PROBE = "P6" +string runId = input.string("P6-replace-me", "Run ID") +int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), + "Scheduled start (UTC; set 5+ min ahead)") +string mode = input.string("entry_then_exit", "Call-order mode", + options=["entry_then_exit", "exit_then_entry", "exit_then_entry_reissue_after_fill"]) +int profitTicks = input.int(1000, "Exit profit distance (ticks)", minval=1) +int lossTicks = input.int(1000, "Exit loss distance (ticks)", minval=1) +int timeoutBars = input.int(300, "Timeout bars", minval=4) +bool arm = input.bool(true, "Arm at scheduled time") + +trace(string tag) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +fill(string tag) => + '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","order_tag":"' + tag + + '","command_bar_index":' + str.tostring(bar_index) + + ',"command_bar_time_ms":' + str.tostring(time) + '}' + +var int liveBars = 0 +var bool initialCallsDone = false +var bool reissued = false +var bool timedOut = false +varip bool armed = false +var int armedBar = na + +bool active = arm and barstate.isrealtime and timenow >= scheduledStart +if active and not armed + armed := true + armedBar := bar_index + alert(trace("armed"), alert.freq_all) + +if active and barstate.isconfirmed and bar_index > armedBar + liveBars += 1 + if not initialCallsDone + initialCallsDone := true + if mode == "entry_then_exit" + strategy.entry("E", strategy.long, qty=1, + alert_message=fill("entry_E")) + strategy.exit("X", "E", profit=profitTicks, loss=lossTicks, + comment_profit="profit", comment_loss="loss", + alert_message=fill("exit_after_entry_same_calculation"), + alert_profit=fill("profit_after_entry_same_calculation"), + alert_loss=fill("loss_after_entry_same_calculation")) + alert(trace("entry_then_exit_calls_complete"), alert.freq_all) + else + strategy.exit("X", "E", profit=profitTicks, loss=lossTicks, + comment_profit="profit", comment_loss="loss", + alert_message=fill("exit_before_entry_same_calculation"), + alert_profit=fill("profit_before_entry_same_calculation"), + alert_loss=fill("loss_before_entry_same_calculation")) + strategy.entry("E", strategy.long, qty=1, + alert_message=fill("entry_E")) + alert(trace("exit_then_entry_calls_complete"), alert.freq_all) + else if mode == "exit_then_entry_reissue_after_fill" and + strategy.position_size > 0 and not reissued + reissued := true + strategy.exit("X", "E", profit=profitTicks, loss=lossTicks, + comment_profit="profit", comment_loss="loss", + alert_message=fill("exit_reissued_after_entry_fill"), + alert_profit=fill("profit_reissued_after_entry_fill"), + alert_loss=fill("loss_reissued_after_entry_fill")) + alert(trace("from_entry_exit_reissued_after_fill"), alert.freq_all) + else if not timedOut and liveBars >= timeoutBars + timedOut := true + strategy.cancel_all() + if strategy.position_size != 0 + strategy.close_all(immediately=true, + alert_message=fill("timeout_close_all")) + alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) diff --git a/docs/probes/tradingview-realtime/p7_simultaneous_priority.pine b/docs/probes/tradingview-realtime/p7_simultaneous_priority.pine new file mode 100644 index 0000000..5f78acf --- /dev/null +++ b/docs/probes/tradingview-realtime/p7_simultaneous_priority.pine @@ -0,0 +1,78 @@ +//@version=6 +strategy("PF TV P7 simultaneous priority", overlay=true, pyramiding=1, + calc_on_every_tick=false, calc_on_order_fills=false, + process_orders_on_close=false) + +string PROBE = "P7" +string runId = input.string("P7-replace-me", "Run ID") +int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), + "Scheduled start (UTC; set 5+ min ahead)") +string mode = input.string("exit_then_reversal", "Source-order mode", + options=["exit_then_reversal", "reversal_then_exit"]) +int marketableStopTicks = input.int(10, "Long-exit stop above close (ticks)", minval=1) +int timeoutBars = input.int(60, "Timeout bars", minval=4) +bool arm = input.bool(true, "Arm at scheduled time") + +trace(string tag) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +fill(string tag) => + '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","order_tag":"' + tag + + '","command_bar_index":' + str.tostring(bar_index) + + ',"command_bar_time_ms":' + str.tostring(time) + '}' + +var int liveBars = 0 +var bool entryPlaced = false +var bool collisionPlaced = false +var bool postCollisionObserved = false +var bool timedOut = false +var float collisionStop = na +varip bool armed = false +var int armedBar = na + +bool active = arm and barstate.isrealtime and timenow >= scheduledStart +if active and not armed + armed := true + armedBar := bar_index + alert(trace("armed"), alert.freq_all) + +if active and barstate.isconfirmed and bar_index > armedBar + liveBars += 1 + if not entryPlaced + entryPlaced := true + strategy.entry("L", strategy.long, qty=1, + alert_message=fill("initial_long")) + alert(trace("initial_long_created"), alert.freq_all) + else if strategy.position_size > 0 and not collisionPlaced + collisionPlaced := true + collisionStop := close + marketableStopTicks * syminfo.mintick + if mode == "exit_then_reversal" + strategy.exit("X", "L", qty=1, stop=collisionStop, + alert_message=fill("marketable_exit")) + strategy.entry("REV", strategy.short, qty=1, + alert_message=fill("market_reversal")) + else + strategy.entry("REV", strategy.short, qty=1, + alert_message=fill("market_reversal")) + strategy.exit("X", "L", qty=1, stop=collisionStop, + alert_message=fill("marketable_exit")) + alert(trace("collision_orders_created"), alert.freq_all) + else if collisionPlaced and not postCollisionObserved + postCollisionObserved := true + alert(trace("first_confirmed_close_after_collision"), alert.freq_all) + else if not timedOut and liveBars >= timeoutBars + timedOut := true + strategy.cancel_all() + if strategy.position_size != 0 + strategy.close_all(immediately=true, + alert_message=fill("timeout_close_all")) + alert(trace("timeout_cleanup"), alert.freq_all) + +plot(collisionStop, "marketable long-exit stop", color=color.red) diff --git a/docs/probes/tradingview-realtime/p8_oca_same_event.pine b/docs/probes/tradingview-realtime/p8_oca_same_event.pine new file mode 100644 index 0000000..2bda161 --- /dev/null +++ b/docs/probes/tradingview-realtime/p8_oca_same_event.pine @@ -0,0 +1,82 @@ +//@version=6 +strategy("PF TV P8 OCA same-event collision", overlay=true, pyramiding=1, + calc_on_every_tick=false, calc_on_order_fills=false, + process_orders_on_close=false) + +string PROBE = "P8" +string runId = input.string("P8-replace-me", "Run ID") +int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), + "Scheduled start (UTC; set 5+ min ahead)") +string mode = input.string("A_then_B", "Source-order mode", + options=["A_then_B", "B_then_A"]) +int marketableStopTicks = input.int(10, "Shared sell-stop distance above close (ticks)", minval=1) +int timeoutBars = input.int(60, "Timeout bars", minval=4) +bool arm = input.bool(true, "Arm at scheduled time") + +trace(string tag) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +fill(string tag) => + '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","order_tag":"' + tag + + '","command_bar_index":' + str.tostring(bar_index) + + ',"command_bar_time_ms":' + str.tostring(time) + '}' + +var int liveBars = 0 +var bool entryPlaced = false +var bool siblingsPlaced = false +var bool postCollisionObserved = false +var bool timedOut = false +var float sharedStop = na +varip bool armed = false +var int armedBar = na + +bool active = arm and barstate.isrealtime and timenow >= scheduledStart +if active and not armed + armed := true + armedBar := bar_index + alert(trace("armed"), alert.freq_all) + +if active and barstate.isconfirmed and bar_index > armedBar + liveBars += 1 + if not entryPlaced + entryPlaced := true + strategy.entry("L", strategy.long, qty=2, + alert_message=fill("initial_long_qty_2")) + alert(trace("initial_long_created"), alert.freq_all) + else if strategy.position_size >= 2 and not siblingsPlaced + siblingsPlaced := true + sharedStop := close + marketableStopTicks * syminfo.mintick + if mode == "A_then_B" + strategy.order("OCA_A", strategy.short, qty=1, stop=sharedStop, + oca_name="P8_GROUP", oca_type=strategy.oca.cancel, + alert_message=fill("OCA_A")) + strategy.order("OCA_B", strategy.short, qty=1, stop=sharedStop, + oca_name="P8_GROUP", oca_type=strategy.oca.cancel, + alert_message=fill("OCA_B")) + else + strategy.order("OCA_B", strategy.short, qty=1, stop=sharedStop, + oca_name="P8_GROUP", oca_type=strategy.oca.cancel, + alert_message=fill("OCA_B")) + strategy.order("OCA_A", strategy.short, qty=1, stop=sharedStop, + oca_name="P8_GROUP", oca_type=strategy.oca.cancel, + alert_message=fill("OCA_A")) + alert(trace("same_price_oca_siblings_created"), alert.freq_all) + else if siblingsPlaced and not postCollisionObserved + postCollisionObserved := true + alert(trace("first_confirmed_close_after_oca_collision"), alert.freq_all) + else if not timedOut and liveBars >= timeoutBars + timedOut := true + strategy.cancel_all() + if strategy.position_size != 0 + strategy.close_all(immediately=true, + alert_message=fill("timeout_close_all")) + alert(trace("timeout_cleanup"), alert.freq_all) + +plot(sharedStop, "shared marketable stop", color=color.red) diff --git a/docs/probes/tradingview-realtime/p9_per_lot_stop_selection.pine b/docs/probes/tradingview-realtime/p9_per_lot_stop_selection.pine new file mode 100644 index 0000000..46f197f --- /dev/null +++ b/docs/probes/tradingview-realtime/p9_per_lot_stop_selection.pine @@ -0,0 +1,161 @@ +//@version=6 +strategy("PF TV P9 per-lot stop selection", overlay=true, pyramiding=2, + calc_on_every_tick=true, calc_on_order_fills=false, + process_orders_on_close=false, close_entries_rule="ANY") + +string PROBE = "P9" +string runId = input.string("P9-replace-me", "Run ID") +int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), + "Scheduled start (UTC; set 5+ min ahead)") +int secondEntryDelayBars = input.int(3, "Bars before second same-ID entry", minval=1) +int minimumEntrySeparationTicks = input.int(100, "Minimum lot-price separation (ticks)", minval=1) +int lossTicks = input.int(1000, "Fixed stop-loss distance (ticks)", minval=2) +int trailActivationTicks = input.int(500, "Trail activation distance (ticks)", minval=1) +int trailOffsetTicks = input.int(300, "Trail offset (ticks)", minval=1) +int timeoutBars = input.int(480, "Timeout bars", minval=8) +bool arm = input.bool(true, "Arm at scheduled time") + +trace(string tag) => + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"time_close_ms":' + str.tostring(time_close) + + ',"price":' + str.tostring(close, format.mintick) + + ',"position_size":' + str.tostring(strategy.position_size) + '}' + +fill(string tag) => + '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","order_tag":"' + tag + + '","command_bar_index":' + str.tostring(bar_index) + + ',"command_bar_time_ms":' + str.tostring(time) + '}' + +traceLots(string tag, float entry0, int entryTime0, bool active0, + float watermark0, float entry1, int entryTime1, bool active1, + float watermark1) => + float fixed0 = entry0 - lossTicks * syminfo.mintick + float fixed1 = entry1 - lossTicks * syminfo.mintick + float activation0 = entry0 + trailActivationTicks * syminfo.mintick + float activation1 = entry1 + trailActivationTicks * syminfo.mintick + float trailing0 = watermark0 - trailOffsetTicks * syminfo.mintick + float trailing1 = watermark1 - trailOffsetTicks * syminfo.mintick + '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + + '","run_id":"' + runId + '","tag":"' + tag + + '","bar_index":' + str.tostring(bar_index) + + ',"bar_time_ms":' + str.tostring(time) + + ',"price":' + str.tostring(close, format.mintick) + + ',"lot0_entry_time_ms":' + str.tostring(entryTime0) + + ',"lot0_entry_price":' + str.tostring(entry0, format.mintick) + + ',"lot0_fixed_stop":' + str.tostring(fixed0, format.mintick) + + ',"lot0_activation":' + str.tostring(activation0, format.mintick) + + ',"lot0_active":' + str.tostring(active0) + + ',"lot0_watermark":' + str.tostring(watermark0, format.mintick) + + ',"lot0_trailing_candidate":' + str.tostring(trailing0, format.mintick) + + ',"lot1_entry_time_ms":' + str.tostring(entryTime1) + + ',"lot1_entry_price":' + str.tostring(entry1, format.mintick) + + ',"lot1_fixed_stop":' + str.tostring(fixed1, format.mintick) + + ',"lot1_activation":' + str.tostring(activation1, format.mintick) + + ',"lot1_active":' + str.tostring(active1) + + ',"lot1_watermark":' + str.tostring(watermark1, format.mintick) + + ',"lot1_trailing_candidate":' + str.tostring(trailing1, format.mintick) + '}' + +var int liveBars = 0 +var int firstEntryBar = na +var bool firstEntryPlaced = false +var bool secondEntryPlaced = false +var bool exitPlaced = false +var bool separationRejected = false +var bool timedOut = false +var float lot0Entry = na +var float lot1Entry = na +var int lot0EntryTime = na +var int lot1EntryTime = na +varip bool lot0Active = false +varip bool lot1Active = false +varip float lot0Watermark = na +varip float lot1Watermark = na +varip bool oneLotActivationTraced = false +varip bool armed = false +varip int armedBar = na + +bool active = arm and barstate.isrealtime and timenow >= scheduledStart +if active and not armed + armed := true + armedBar := bar_index + alert(trace("armed"), alert.freq_all) + +if active and exitPlaced and strategy.position_size > 0 + float activation0 = lot0Entry + trailActivationTicks * syminfo.mintick + float activation1 = lot1Entry + trailActivationTicks * syminfo.mintick + if not lot0Active and close >= activation0 + lot0Active := true + lot0Watermark := close + alert(traceLots("lot0_trail_activated", lot0Entry, lot0EntryTime, + lot0Active, lot0Watermark, lot1Entry, lot1EntryTime, + lot1Active, lot1Watermark), alert.freq_all) + if not lot1Active and close >= activation1 + lot1Active := true + lot1Watermark := close + alert(traceLots("lot1_trail_activated", lot0Entry, lot0EntryTime, + lot0Active, lot0Watermark, lot1Entry, lot1EntryTime, + lot1Active, lot1Watermark), alert.freq_all) + if lot0Active + lot0Watermark := math.max(lot0Watermark, close) + if lot1Active + lot1Watermark := math.max(lot1Watermark, close) + if not oneLotActivationTraced and lot0Active != lot1Active + oneLotActivationTraced := true + float finiteWatermark0 = lot0Active ? lot0Watermark : lot0Entry + float finiteWatermark1 = lot1Active ? lot1Watermark : lot1Entry + alert(traceLots("only_one_lot_trail_active", lot0Entry, lot0EntryTime, + lot0Active, finiteWatermark0, lot1Entry, lot1EntryTime, + lot1Active, finiteWatermark1), alert.freq_all) + +if active and barstate.isconfirmed and bar_index > armedBar + liveBars += 1 + if not firstEntryPlaced + firstEntryPlaced := true + firstEntryBar := bar_index + strategy.entry("L", strategy.long, qty=1, + alert_message=fill("same_id_lot_1")) + alert(trace("first_same_id_entry_created"), alert.freq_all) + else if strategy.position_size >= 1 and not secondEntryPlaced and + bar_index >= firstEntryBar + secondEntryDelayBars and + math.abs(close - strategy.position_avg_price) >= + minimumEntrySeparationTicks * syminfo.mintick + secondEntryPlaced := true + strategy.entry("L", strategy.long, qty=2, + alert_message=fill("same_id_lot_2")) + alert(trace("second_same_id_entry_created"), alert.freq_all) + else if strategy.position_size >= 3 and not exitPlaced and not separationRejected + lot0Entry := strategy.opentrades.entry_price(0) + lot1Entry := strategy.opentrades.entry_price(1) + lot0EntryTime := strategy.opentrades.entry_time(0) + lot1EntryTime := strategy.opentrades.entry_time(1) + lot0Watermark := lot0Entry + lot1Watermark := lot1Entry + if math.abs(lot0Entry - lot1Entry) < + minimumEntrySeparationTicks * syminfo.mintick + separationRejected := true + alert(traceLots("inconclusive_lot_prices_too_close", lot0Entry, + lot0EntryTime, lot0Active, lot0Watermark, lot1Entry, + lot1EntryTime, lot1Active, lot1Watermark), alert.freq_all) + else + exitPlaced := true + strategy.exit("X", "L", loss=lossTicks, + trail_points=trailActivationTicks, + trail_offset=trailOffsetTicks, + comment_loss="fixed_stop", comment_trailing="trailing_stop", + alert_message=fill("shared_exit_fixed_stop_or_trail"), + alert_loss=fill("fixed_stop_fill"), + alert_trailing=fill("trailing_stop_fill")) + alert(traceLots("shared_exit_created_for_two_lots", lot0Entry, + lot0EntryTime, lot0Active, lot0Watermark, lot1Entry, + lot1EntryTime, lot1Active, lot1Watermark), alert.freq_all) + else if not timedOut and liveBars >= timeoutBars + timedOut := true + strategy.cancel_all() + if strategy.position_size != 0 + strategy.close_all(immediately=true, + alert_message=fill("timeout_close_all")) + alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) diff --git a/docs/probes/tradingview-realtime/results.template.md b/docs/probes/tradingview-realtime/results.template.md new file mode 100644 index 0000000..0e819ee --- /dev/null +++ b/docs/probes/tradingview-realtime/results.template.md @@ -0,0 +1,39 @@ +# TradingView realtime probe results + +| Field | Value | +|---|---| +| Probe / mode | | +| Run ID | | +| Webhook / alert-log / trade counts | | +| Repeat run | | + +## Pre-registered decision rule + +| Item | Signature | +|---|---| +| Tested statement | | +| Confirming sequence | | +| Disconfirming sequence | | +| Invalidating/contaminating sequence | | + +## Intended event sequence + +Write the command and price sequence before examining the result. + +## Observed event sequence + +List trace and fill events in TradingView alert-log order. Include bar open time, +command bar, order tag, order ID, fill-bar open time, server time, fill price, +quantity, and resulting position. + +## Cross-check + +- Webhook versus TradingView alert log: +- Alert log versus Strategy Tester List of Trades: +- Intended price milestones present: +- Missing or duplicated events: + +## Conclusion + +Choose one: `confirmed`, `disconfirmed`, or `inconclusive`. State only the +semantic rule established by the evidence and list every remaining ambiguity. diff --git a/docs/probes/tradingview-realtime/validate_capture.py b/docs/probes/tradingview-realtime/validate_capture.py new file mode 100644 index 0000000..7f4c345 --- /dev/null +++ b/docs/probes/tradingview-realtime/validate_capture.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Validate completeness and JSON structure of a TradingView probe capture. + +Semantic agreement between the TradingView alert log, trade export, and the +pre-registered decision rule still requires manual review. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime +import hashlib +import json +from pathlib import Path + + +TRACE_SCHEMA = "pf-tv-probe-trace-v1" +FILL_SCHEMA = "pf-tv-probe-fill-v1" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def line_sha256(raw: str) -> str: + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def fail(message: str) -> None: + raise ValueError(message) + + +def parse_utc(value: object, label: str, errors: list[str]) -> datetime | None: + if not isinstance(value, str) or value.startswith("YYYY-"): + errors.append(f"{label} is not populated") + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + errors.append(f"{label} is not ISO-8601: {value!r}") + return None + if parsed.tzinfo is None: + errors.append(f"{label} must include a timezone") + return None + return parsed + + +def verify_hash(path: Path, expected: object, label: str, + errors: list[str]) -> str: + actual = sha256(path) + if not isinstance(expected, str) or expected in {"", "replace-me"}: + errors.append(f"{label} hash is not populated") + elif expected != actual: + errors.append(f"{label} hash {expected} != actual {actual}") + return actual + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("run_dir", type=Path) + args = parser.parse_args() + + run_dir = args.run_dir.resolve() + manifest_path = run_dir / "manifest.json" + if not manifest_path.is_file(): + fail(f"missing {manifest_path}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("schema") != "pf-tv-probe-manifest-v1": + fail("manifest schema must be pf-tv-probe-manifest-v1") + + required_files = { + "webhook": run_dir / "webhook.jsonl", + "receipt": run_dir / "receipt.jsonl", + "alert_message": run_dir / "alert-message.txt", + "tv_alert_log": run_dir / "tv-alert-log.csv", + "tv_trades": run_dir / "tv-trades.csv", + "tv_bars": run_dir / "tv-bars.csv", + "chart": run_dir / "chart.png", + "source": run_dir / "deployed-source.pine", + "notes": run_dir / "notes.md", + "results": run_dir / "results.md", + } + missing = [str(path) for path in required_files.values() if not path.is_file()] + if missing: + fail("missing required artifacts: " + ", ".join(missing)) + + errors: list[str] = [] + probe = manifest.get("probe") + run_id = manifest.get("run_id") + if not isinstance(probe, str) or not probe.startswith("P"): + errors.append("manifest probe must be a P-number string") + if not isinstance(run_id, str) or not run_id: + errors.append("manifest run_id must be non-empty") + if manifest.get("mode") in {None, "", "replace-me"}: + errors.append("manifest mode is not populated") + if manifest.get("alert", {}).get("alert_id") in {None, "", "replace-me"}: + errors.append("manifest alert.alert_id is not populated") + + scheduled = parse_utc( + manifest.get("scheduled_start_utc"), "scheduled_start_utc", errors) + created = parse_utc( + manifest.get("alert", {}).get("created_at_utc"), + "alert.created_at_utc", errors) + armed = parse_utc( + manifest.get("observation", {}).get("armed_at_utc"), + "observation.armed_at_utc", errors) + if scheduled and created and created >= scheduled: + errors.append("alert must be created before scheduled start") + if scheduled and armed and armed < scheduled: + errors.append("armed time precedes scheduled start") + + webhook_path = required_files["webhook"] + raw_lines = [raw for raw in webhook_path.read_text( + encoding="utf-8").splitlines() if raw.strip()] + events: list[dict] = [] + for line_number, raw in enumerate(raw_lines, 1): + try: + event = json.loads(raw) + except json.JSONDecodeError as exc: + errors.append(f"webhook line {line_number}: invalid JSON: {exc}") + continue + schema = event.get("schema") + if schema == TRACE_SCHEMA: + event_probe = event.get("probe") + event_run_id = event.get("run_id") + if not isinstance(event.get("tag"), str): + errors.append(f"webhook line {line_number}: trace tag missing") + elif schema == FILL_SCHEMA: + payload = event.get("probe_payload") + if not isinstance(payload, dict): + errors.append( + f"webhook line {line_number}: fill probe_payload is not an object") + continue + event_probe = payload.get("probe") + event_run_id = payload.get("run_id") + if not isinstance(payload.get("order_tag"), str): + errors.append(f"webhook line {line_number}: fill order_tag missing") + if not isinstance(event.get("fill_bar_time"), str): + errors.append(f"webhook line {line_number}: fill_bar_time missing") + if not isinstance(event.get("server_time"), str): + errors.append(f"webhook line {line_number}: server_time missing") + else: + errors.append(f"webhook line {line_number}: unknown schema {schema!r}") + continue + if event_probe != probe: + errors.append( + f"webhook line {line_number}: probe {event_probe!r} != {probe!r}") + if event_run_id != run_id: + errors.append( + f"webhook line {line_number}: run_id {event_run_id!r} != {run_id!r}") + events.append(event) + + armed_events = [event for event in events + if event.get("schema") == TRACE_SCHEMA + and event.get("tag") == "armed"] + if len(armed_events) != 1: + errors.append(f"expected exactly one armed trace, got {len(armed_events)}") + + observation = manifest.get("observation", {}) + declared_count = observation.get("webhook_event_count") + if not isinstance(declared_count, int) or declared_count != len(events): + errors.append( + f"manifest webhook_event_count {declared_count!r} != parsed {len(events)}") + tv_alert_count = observation.get("tradingview_alert_event_count") + if not isinstance(tv_alert_count, int) or tv_alert_count <= 0: + errors.append("tradingview_alert_event_count must be positive") + if isinstance(tv_alert_count, int) and tv_alert_count < len(events): + errors.append("TradingView alert count cannot be smaller than webhook count") + if observation.get("chart_reloaded_or_changed") is not False: + errors.append("chart/script changed or reloaded during capture") + + receipts: list[dict] = [] + for line_number, raw in enumerate(required_files["receipt"].read_text( + encoding="utf-8").splitlines(), 1): + if not raw.strip(): + continue + try: + record = json.loads(raw) + except json.JSONDecodeError as exc: + errors.append(f"receipt line {line_number}: invalid JSON: {exc}") + continue + receipts.append(record) + if len(receipts) != len(raw_lines): + errors.append( + f"receipt count {len(receipts)} != raw webhook body count {len(raw_lines)}") + sequences = [record.get("sequence") for record in receipts] + if not sequences: + errors.append("receipt.jsonl is empty") + elif not all(isinstance(value, int) for value in sequences): + errors.append("every receipt sequence must be an integer") + elif sequences != list(range(sequences[0], sequences[0] + len(sequences))): + errors.append("receipt sequences must be contiguous and arrival-ordered") + for index, (record, raw) in enumerate(zip(receipts, raw_lines), 1): + if record.get("body_sha256") != line_sha256(raw): + errors.append(f"receipt {index}: body_sha256 mismatch") + parse_utc(record.get("received_at_utc"), + f"receipt {index}.received_at_utc", errors) + if sequences: + if observation.get("receiver_first_sequence") != sequences[0]: + errors.append("receiver_first_sequence does not match receipts") + if observation.get("receiver_last_sequence") != sequences[-1]: + errors.append("receiver_last_sequence does not match receipts") + + artifacts = manifest.get("artifacts", {}) + actual_hashes = { + "webhook_jsonl_sha256": verify_hash( + required_files["webhook"], artifacts.get("webhook_jsonl_sha256"), + "webhook.jsonl", errors), + "receipt_jsonl_sha256": verify_hash( + required_files["receipt"], artifacts.get("receipt_jsonl_sha256"), + "receipt.jsonl", errors), + "tv_alert_log_sha256": verify_hash( + required_files["tv_alert_log"], artifacts.get("tv_alert_log_sha256"), + "tv-alert-log.csv", errors), + "tv_trades_csv_sha256": verify_hash( + required_files["tv_trades"], artifacts.get("tv_trades_csv_sha256"), + "tv-trades.csv", errors), + "tv_bars_csv_sha256": verify_hash( + required_files["tv_bars"], artifacts.get("tv_bars_csv_sha256"), + "tv-bars.csv", errors), + "chart_png_sha256": verify_hash( + required_files["chart"], artifacts.get("chart_png_sha256"), + "chart.png", errors), + "notes_md_sha256": verify_hash( + required_files["notes"], artifacts.get("notes_md_sha256"), + "notes.md", errors), + "results_md_sha256": verify_hash( + required_files["results"], artifacts.get("results_md_sha256"), + "results.md", errors), + } + verify_hash( + required_files["alert_message"], + manifest.get("alert", {}).get("message_template_sha256"), + "alert-message.txt", errors) + source_hash = verify_hash( + required_files["source"], manifest.get("deployed_source_sha256"), + "deployed-source.pine", errors) + if manifest.get("script_sha256") != source_hash: + errors.append("script_sha256 does not match deployed-source.pine") + + results_text = required_files["results"].read_text(encoding="utf-8") + if "## Pre-registered decision rule" not in results_text: + errors.append("results.md lacks the pre-registered decision rule") + + summary = { + "run_dir": str(run_dir), + "probe": probe, + "run_id": run_id, + "events": len(events), + "trace_events": sum( + event.get("schema") == TRACE_SCHEMA for event in events), + "fill_events": sum( + event.get("schema") == FILL_SCHEMA for event in events), + "artifact_hashes": actual_hashes, + "semantic_review_required": True, + "errors": errors, + } + print(json.dumps(summary, indent=2, sort_keys=True)) + return 1 if errors else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/tv-parity-probe-spec.md b/docs/tv-parity-probe-spec.md index d150c03..3707a15 100644 --- a/docs/tv-parity-probe-spec.md +++ b/docs/tv-parity-probe-spec.md @@ -1,5 +1,10 @@ # TV-Parity Probe Spec (handoff) +For the realtime broker-emulator questions P1-P9, use the runnable JSON/webhook +kit in [`docs/probes/tradingview-realtime/`](probes/tradingview-realtime/README.md). +This document continues to cover historical export probes for instruments, +margin, TA, higher timeframes, and report fields. + > These probes need a **TradingView export** (`tv_trades.csv`) as ground truth, > which cannot be generated from this repo. Each entry is a clean-room > `strategy.pine` + the exact capture recipe + expected outcome, ready for From b0a8a3ea8b068c25699c78617fb2c40c929aa2cd Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sat, 11 Jul 2026 17:20:35 +0800 Subject: [PATCH 5/5] Instrument TradingView probe webhooks --- docs/probes/tradingview-realtime/PAYLOAD.md | 133 +++++++++++ docs/probes/tradingview-realtime/README.md | 46 ++-- .../fill-alert-message.template.txt | 1 + .../manifest.template.json | 2 + .../p1_cross_command_id.pine | 74 ++++-- .../p2_replacement_priority.pine | 68 ++++-- .../p3_stop_limit_reissue.pine | 65 ++++-- .../p4_trailing_reissue.pine | 91 ++++++-- .../p5_exit_without_from_entry.pine | 98 ++++++-- .../p6_from_entry_cutoff.pine | 118 ++++++++-- .../p7_simultaneous_priority.pine | 82 +++++-- .../p8_oca_same_event.pine | 82 +++++-- .../p9_per_lot_stop_selection.pine | 120 +++++++--- .../tradingview-realtime/validate_capture.py | 218 +++++++++++++++--- 14 files changed, 964 insertions(+), 234 deletions(-) create mode 100644 docs/probes/tradingview-realtime/PAYLOAD.md create mode 100644 docs/probes/tradingview-realtime/fill-alert-message.template.txt diff --git a/docs/probes/tradingview-realtime/PAYLOAD.md b/docs/probes/tradingview-realtime/PAYLOAD.md new file mode 100644 index 0000000..84d7422 --- /dev/null +++ b/docs/probes/tradingview-realtime/PAYLOAD.md @@ -0,0 +1,133 @@ +# Webhook payload contract + +Every delivered body uses `pf-tv-probe-event-v2`. There are two event types: + +- `trace`: emitted directly by Pine's `alert()` for arming, command creation, + price milestones, cancellations, invalidation, and cleanup. +- `order_fill`: emitted by TradingView's broker emulator. The fill envelope is + configured with `fill-alert-message.template.txt`; its `command_context` + comes from the exact order call's `alert_message` (or `alert_profit`, + `alert_loss`, or `alert_trailing`). + +## Trace event + +```json +{ + "schema": "pf-tv-probe-event-v2", + "source": "tradingview", + "event_type": "trace", + "event_key_hint": "P3|run-123|trace|reissued_between_stop_and_limit|12345|1740000000000|1740000060123", + "probe": {"id": "P3", "run_id": "run-123", "mode": "unchanged"}, + "event": {"name": "reissued_between_stop_and_limit"}, + "market": { + "tickerid": "BINANCE:ETHUSDT.P", + "interval": "1", + "bar_index": 12345, + "bar_time_ms": 1740000000000, + "bar_close_ms": 1740000060000, + "server_time_ms": 1740000060123, + "price": 2712.34 + }, + "strategy": {"position_size": 0} +} +``` + +## Order-fill event + +```json +{ + "schema": "pf-tv-probe-event-v2", + "source": "tradingview", + "event_type": "order_fill", + "event_key_hint": "BINANCE:ETHUSDT.P|1|2026-07-12T01:01:00Z|SL|buy|2712.34|1|2026-07-12T01:01:03Z", + "command_context": { + "schema": "pf-tv-probe-command-v2", + "probe": {"id": "P3", "run_id": "run-123", "mode": "unchanged"}, + "command": { + "tag": "reissued_stop_limit", + "api": "strategy.entry", + "action": "replace", + "order_id": "SL", + "source_order": 1, + "side": "long", + "order_type": "stop_limit", + "from_entry": null, + "qty": 1, + "stop_price": 2713.00, + "limit_price": 2712.50, + "profit_ticks": null, + "loss_ticks": null, + "trail_points": null, + "trail_offset": null, + "oca_name": null, + "oca_type": null, + "debug_intent": "test activated stop-limit persistence after reissue" + }, + "message_evaluation": { + "bar_index": 12345, + "bar_time_ms": 1740000000000, + "server_time_ms": 1740000000123, + "position_size": 0 + } + }, + "market": { + "ticker": "ETHUSDT.P", + "exchange": "BINANCE", + "interval": "1", + "fill_bar_time": "2026-07-12T01:01:00Z", + "server_time": "2026-07-12T01:01:03Z" + }, + "fill": { + "order_id": "SL", + "action": "buy", + "contracts": 1, + "price": 2712.34, + "position_size": 1, + "market_position": "long", + "market_position_size": 1, + "previous_market_position": "flat", + "previous_market_position_size": 0 + } +} +``` + +`command_context.command` identifies which source action produced the filled +order. It includes absolute prices, relative profit/loss ticks, trailing +parameters, and OCA settings where applicable. TradingView documents variables +inside `alert_message` as evaluated when the order executes. Consequently, +`message_evaluation` is diagnostic fill-time context and must not be used as +the command-creation timestamp. Correlate `command.tag` with the adjacent trace +event named `command__issued` to establish creation/reissue timing. That +trace is emitted even when the order never fills. + +## Receiver behavior + +1. Accept only `POST` with a JSON object and `schema == pf-tv-probe-event-v2`. +2. Assign a receiver-global monotonically increasing `receipt_id`, a contiguous + capture-local `sequence`, and UTC `received_at_utc` before responding. Global + receipt IDs may have gaps inside one capture; capture sequences may not. +3. Persist the exact raw body and its SHA-256. Store receiver metadata + separately in `receipt.jsonl`. +4. Return a 2xx response immediately; perform semantic processing + asynchronously. +5. Route traces by `probe.id`; route fills by `command_context.probe.id`. + Persist but quarantine an inactive/unknown `run_id` during asynchronous + processing; do not use that check to delay the immediate 2xx response. +6. Preserve arrival order, but never treat it as broker order. TradingView's + alert log is authoritative for relative fill ordering. +7. `event_key_hint` is a correlation hint, not an idempotency key. Do not + collapse equal-looking events: same-tick multi-fill is an intentional probe + surface. Flag possible retries for manual reconciliation instead. +8. Never execute trades from this endpoint. The payload is diagnostic evidence + only. + +One receipt line has this receiver-owned shape: + +```json +{"receipt_id":4812,"sequence":7,"received_at_utc":"2026-07-12T01:01:03.412Z","body_sha256":"64 lowercase hex characters"} +``` + +Cancellation commands do not generate TradingView order-fill alerts and accept +no `alert_message`. Each probe therefore emits an explicit trace such as +`command_cancel_all_timeout` immediately beside every `strategy.cancel*()` +call. diff --git a/docs/probes/tradingview-realtime/README.md b/docs/probes/tradingview-realtime/README.md index 1217b89..a5ed372 100644 --- a/docs/probes/tradingview-realtime/README.md +++ b/docs/probes/tradingview-realtime/README.md @@ -5,12 +5,14 @@ These Pine v6 strategies answer the nine open broker-emulator questions in a general TradingView oracle. PineForge's deterministic replay and invariant tests remain the primary regression gate. -The probes emit two JSON-only event families: +The probes emit one versioned JSON-only event envelope with two event types: -- `pf-tv-probe-trace-v1` from `alert()` records command creation and selected - price milestones. -- `pf-tv-probe-fill-v1` from TradingView order-fill alerts records the actual - broker fill, wrapping the order's `alert_message` object. +- `trace` from `alert()` records arming, command creation, cancellation, and + selected price milestones. +- `order_fill` from TradingView's broker emulator records the actual fill and + embeds the exact order call's `pf-tv-probe-command-v2` diagnostic message. + +See [`PAYLOAD.md`](PAYLOAD.md) for the complete payload and receiver contract. TradingView references used by this protocol: @@ -37,7 +39,8 @@ Use one probe and one alert at a time. 7. Add the strategy, then create a strategy alert with both **Order fills and alert() function calls** enabled. TradingView snapshots the script and its inputs when the alert is created; after any change, delete and recreate it. -8. Configure the webhook URL and paste the fill message template below exactly. +8. Configure the webhook URL and paste + [`fill-alert-message.template.txt`](fill-alert-message.template.txt) exactly. 9. Keep `Arm at scheduled time` enabled. Historical bars never place probe orders, and commands wait until a later confirmed bar. A run is valid only when TradingView's alert log shows `armed` before every command/fill; receiver @@ -45,18 +48,25 @@ Use one probe and one alert at a time. 10. Capture until the expected fill or the probe's timeout trace. Export **Strategy Tester -> List of Trades** as CSV after the run. -Fill alert message template: - -```json -{"schema":"pf-tv-probe-fill-v1","probe_payload":{{strategy.order.alert_message}},"ticker":"{{ticker}}","interval":"{{interval}}","fill_bar_time":"{{time}}","server_time":"{{timenow}}","order_id":"{{strategy.order.id}}","action":"{{strategy.order.action}}","contracts":{{strategy.order.contracts}},"price":{{strategy.order.price}},"position_size":{{strategy.position_size}}} -``` - The unexpanded template is not JSON; the delivered webhook body is. Every probe supplies `strategy.order.alert_message` as a JSON object, not a quoted -string. `probe_payload.command_bar_*` identifies the calculation that created -or reissued the order; outer `fill_bar_time` and `server_time` identify the -actual fill notification. Use fill-bar open time—not raw `bar_index`—for -bar-level regression comparisons. +string. Each order action has a distinct `command.tag`, API, create/replace/ +cleanup action, order ID, source-call order, order type and parameters, and +plain-language `debug_intent`. TradingView documents variables in +`alert_message` as evaluated when the order executes, so `message_evaluation` +is fill-time diagnostic context—not proof of the command creation bar. Use the +adjacent `trace` event to establish command creation, and outer +`market.fill_bar_time` / `market.server_time` for the fill. Use fill-bar open +time—not raw `bar_index`—for bar-level regression comparisons. + +Every order-generating call also emits a `command__issued` trace immediately +before the call. This preserves creation evidence even when the order is later +replaced, rejected, cancelled, or never filled and therefore never exposes its +`alert_message` through an order-fill alert. + +TradingView's cancellation APIs accept no `alert_message` and produce no fill +alert. Every `strategy.cancel*()` in these probes therefore has an immediately +adjacent `command_cancel*` trace. Do not infer cancellation from silence. ## Required artifacts @@ -66,8 +76,8 @@ Store one directory per run: P3-unchanged-2026-07-12T0100Z/ manifest.json webhook.jsonl - receipt.jsonl # receiver sequence/time/body hash; no TV fields added - alert-message.txt # exact fill template pasted into TradingView + receipt.jsonl # receipt_id/sequence/UTC time/body hash; separate metadata + alert-message.txt # copied fill-alert-message.template.txt tv-alert-log.csv # export/copy when available tv-trades.csv tv-bars.csv # exported chart OHLCV covering the observation window diff --git a/docs/probes/tradingview-realtime/fill-alert-message.template.txt b/docs/probes/tradingview-realtime/fill-alert-message.template.txt new file mode 100644 index 0000000..8637b8a --- /dev/null +++ b/docs/probes/tradingview-realtime/fill-alert-message.template.txt @@ -0,0 +1 @@ +{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"order_fill","event_key_hint":"{{ticker}}|{{interval}}|{{time}}|{{strategy.order.id}}|{{strategy.order.action}}|{{strategy.order.price}}|{{strategy.order.contracts}}|{{timenow}}","command_context":{{strategy.order.alert_message}},"market":{"ticker":"{{ticker}}","exchange":"{{exchange}}","interval":"{{interval}}","fill_bar_time":"{{time}}","server_time":"{{timenow}}"},"fill":{"order_id":"{{strategy.order.id}}","action":"{{strategy.order.action}}","contracts":{{strategy.order.contracts}},"price":{{strategy.order.price}},"position_size":{{strategy.position_size}},"market_position":"{{strategy.market_position}}","market_position_size":{{strategy.market_position_size}},"previous_market_position":"{{strategy.prev_market_position}}","previous_market_position_size":{{strategy.prev_market_position_size}}}} diff --git a/docs/probes/tradingview-realtime/manifest.template.json b/docs/probes/tradingview-realtime/manifest.template.json index f83676a..5d1bfac 100644 --- a/docs/probes/tradingview-realtime/manifest.template.json +++ b/docs/probes/tradingview-realtime/manifest.template.json @@ -25,6 +25,8 @@ "alert_id": "replace-me", "created_at_utc": "YYYY-MM-DDTHH:MM:SSZ", "condition": "order fills and alert() function calls", + "event_schema": "pf-tv-probe-event-v2", + "command_schema": "pf-tv-probe-command-v2", "message_template_sha256": "replace-me", "webhook_receiver": "redacted-host-label" }, diff --git a/docs/probes/tradingview-realtime/p1_cross_command_id.pine b/docs/probes/tradingview-realtime/p1_cross_command_id.pine index d48ee68..3d9cd56 100644 --- a/docs/probes/tradingview-realtime/p1_cross_command_id.pine +++ b/docs/probes/tradingview-realtime/p1_cross_command_id.pine @@ -12,21 +12,44 @@ string mode = input.string("entry_then_order", "Placement order", int triggerTicks = input.int(1000, "Buy-stop distance (ticks)", minval=1) int timeoutBars = input.int(120, "Timeout bars", minval=2) bool arm = input.bool(true, "Arm at scheduled time") +string MODE = mode trace(string tag) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + '}}' -fill(string tag) => - '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","order_tag":"' + tag + - '","command_bar_index":' + str.tostring(bar_index) + - ',"command_bar_time_ms":' + str.tostring(time) + '}' +jsonNumber(float value) => na(value) ? "null" : str.tostring(value) +jsonStringOrNull(string value) => value == "" ? "null" : '"' + value + '"' + +commandMessage(string tag, string api, string action, string orderId, + int sourceOrder, string side, string orderType, string fromEntry, + float qty, float stopPrice, float limitPrice, float trailPoints, + float trailOffset, string ocaName, string ocaType, string intent) => + '{"schema":"pf-tv-probe-command-v2","probe":{"id":"' + PROBE + + '","run_id":"' + runId + '","mode":"' + MODE + + '"},"command":{"tag":"' + tag + '","api":"' + api + + '","action":"' + action + '","order_id":"' + orderId + + '","source_order":' + str.tostring(sourceOrder) + ',"side":"' + side + + '","order_type":"' + orderType + '","from_entry":' + + jsonStringOrNull(fromEntry) + ',"qty":' + jsonNumber(qty) + + ',"stop_price":' + jsonNumber(stopPrice) + ',"limit_price":' + + jsonNumber(limitPrice) + ',"profit_ticks":null,"loss_ticks":null,"trail_points":' + + jsonNumber(trailPoints) + + ',"trail_offset":' + jsonNumber(trailOffset) + ',"oca_name":' + + jsonStringOrNull(ocaName) + ',"oca_type":' + jsonStringOrNull(ocaType) + + ',"debug_intent":"' + intent + '"},"message_evaluation":{"bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"server_time_ms":' + str.tostring(timenow) + + ',"position_size":' + str.tostring(strategy.position_size) + '}}' var int liveBars = 0 var float trigger = na @@ -48,21 +71,38 @@ if active and barstate.isconfirmed and bar_index > armedBar trigger := close + triggerTicks * syminfo.mintick alert(trace("placing_same_id_commands"), alert.freq_all) if mode == "entry_then_order" + alert(trace("command_entry_call_issued"), alert.freq_all) strategy.entry("SAME", strategy.long, qty=1, stop=trigger, - alert_message=fill("entry_call")) + alert_message=commandMessage("entry_call", "strategy.entry", + "create", "SAME", 1, "long", "stop", "", 1, trigger, na, + na, na, "", "", "test entry and raw-order ID collision")) + alert(trace("command_raw_order_call_issued"), alert.freq_all) strategy.order("SAME", strategy.long, qty=1, stop=trigger, - alert_message=fill("raw_order_call")) + alert_message=commandMessage("raw_order_call", "strategy.order", + "create", "SAME", 2, "long", "stop", "", 1, trigger, na, + na, na, "", "", "test raw order placed after same-ID entry")) else + alert(trace("command_raw_order_call_issued"), alert.freq_all) strategy.order("SAME", strategy.long, qty=1, stop=trigger, - alert_message=fill("raw_order_call")) + alert_message=commandMessage("raw_order_call", "strategy.order", + "create", "SAME", 1, "long", "stop", "", 1, trigger, na, + na, na, "", "", "test raw order and entry ID collision")) + alert(trace("command_entry_call_issued"), alert.freq_all) strategy.entry("SAME", strategy.long, qty=1, stop=trigger, - alert_message=fill("entry_call")) + alert_message=commandMessage("entry_call", "strategy.entry", + "create", "SAME", 2, "long", "stop", "", 1, trigger, na, + na, na, "", "", "test entry placed after same-ID raw order")) else if not timedOut and liveBars >= timeoutBars timedOut := true + alert(trace("command_cancel_all_timeout"), alert.freq_all) strategy.cancel_all() if strategy.position_size != 0 + alert(trace("command_timeout_close_all_issued"), alert.freq_all) strategy.close_all(immediately=true, - alert_message=fill("timeout_close_all")) + alert_message=commandMessage("timeout_close_all", + "strategy.close_all", "cleanup", "__close_all__", 1, + "flat", "market_close", "", na, na, na, na, na, "", "", + "cleanup residual position after P1 timeout")) alert(trace("timeout_cleanup"), alert.freq_all) plot(trigger, "shared trigger", color=color.orange) diff --git a/docs/probes/tradingview-realtime/p2_replacement_priority.pine b/docs/probes/tradingview-realtime/p2_replacement_priority.pine index d9a26d7..7e90581 100644 --- a/docs/probes/tradingview-realtime/p2_replacement_priority.pine +++ b/docs/probes/tradingview-realtime/p2_replacement_priority.pine @@ -10,21 +10,43 @@ int scheduledStart = input.time(timestamp("01 Jan 2030 00:00 +0000"), int triggerTicks = input.int(5000, "Shared buy-stop distance (ticks)", minval=1) int timeoutBars = input.int(240, "Timeout bars", minval=3) bool arm = input.bool(true, "Arm at scheduled time") +string MODE = "reissue_A_after_B" trace(string tag) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + '}}' -fill(string tag) => - '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","order_tag":"' + tag + - '","command_bar_index":' + str.tostring(bar_index) + - ',"command_bar_time_ms":' + str.tostring(time) + '}' +jsonNumber(float value) => na(value) ? "null" : str.tostring(value) +jsonStringOrNull(string value) => value == "" ? "null" : '"' + value + '"' +commandMessage(string tag, string api, string action, string orderId, + int sourceOrder, string side, string orderType, string fromEntry, + float qty, float stopPrice, float limitPrice, float trailPoints, + float trailOffset, string ocaName, string ocaType, string intent) => + '{"schema":"pf-tv-probe-command-v2","probe":{"id":"' + PROBE + + '","run_id":"' + runId + '","mode":"' + MODE + + '"},"command":{"tag":"' + tag + '","api":"' + api + + '","action":"' + action + '","order_id":"' + orderId + + '","source_order":' + str.tostring(sourceOrder) + ',"side":"' + side + + '","order_type":"' + orderType + '","from_entry":' + + jsonStringOrNull(fromEntry) + ',"qty":' + jsonNumber(qty) + + ',"stop_price":' + jsonNumber(stopPrice) + ',"limit_price":' + + jsonNumber(limitPrice) + ',"profit_ticks":null,"loss_ticks":null,"trail_points":' + + jsonNumber(trailPoints) + + ',"trail_offset":' + jsonNumber(trailOffset) + ',"oca_name":' + + jsonStringOrNull(ocaName) + ',"oca_type":' + jsonStringOrNull(ocaType) + + ',"debug_intent":"' + intent + '"},"message_evaluation":{"bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"server_time_ms":' + str.tostring(timenow) + + ',"position_size":' + str.tostring(strategy.position_size) + '}}' var int liveBars = 0 var float trigger = na @@ -44,25 +66,39 @@ if active and barstate.isconfirmed and bar_index > armedBar liveBars += 1 if liveBars == 1 trigger := close + triggerTicks * syminfo.mintick + alert(trace("command_A_initial_issued"), alert.freq_all) strategy.order("A", strategy.long, qty=1, stop=trigger, - alert_message=fill("A_initial")) + alert_message=commandMessage("A_initial", "strategy.order", + "create", "A", 1, "long", "stop", "", 1, trigger, na, na, na, + "", "", "establish A ahead of B before replacement")) + alert(trace("command_B_unchanged_issued"), alert.freq_all) strategy.order("B", strategy.long, qty=1, stop=trigger, - alert_message=fill("B_unchanged")) + alert_message=commandMessage("B_unchanged", "strategy.order", + "create", "B", 2, "long", "stop", "", 1, trigger, na, na, na, + "", "", "unchanged sibling used as broker-priority control")) alert(trace("A_then_B_placed"), alert.freq_all) else if liveBars == 2 and strategy.position_size == 0 reissued := true + alert(trace("command_A_reissued_issued"), alert.freq_all) strategy.order("A", strategy.long, qty=1, stop=trigger, - alert_message=fill("A_reissued")) + alert_message=commandMessage("A_reissued", "strategy.order", + "replace", "A", 1, "long", "stop", "", 1, trigger, na, na, na, + "", "", "measure whether same-ID replacement resets A priority")) alert(trace("A_reissued_after_B"), alert.freq_all) else if liveBars == 2 and strategy.position_size != 0 and not aborted aborted := true alert(trace("inconclusive_fill_before_reissue"), alert.freq_all) else if not timedOut and liveBars >= timeoutBars timedOut := true + alert(trace("command_cancel_all_timeout"), alert.freq_all) strategy.cancel_all() if strategy.position_size != 0 + alert(trace("command_timeout_close_all_issued"), alert.freq_all) strategy.close_all(immediately=true, - alert_message=fill("timeout_close_all")) + alert_message=commandMessage("timeout_close_all", + "strategy.close_all", "cleanup", "__close_all__", 1, + "flat", "market_close", "", na, na, na, na, na, "", "", + "cleanup residual position after P2 timeout")) alert(trace("timeout_cleanup"), alert.freq_all) plot(trigger, "shared trigger", color=reissued ? color.aqua : color.orange) diff --git a/docs/probes/tradingview-realtime/p3_stop_limit_reissue.pine b/docs/probes/tradingview-realtime/p3_stop_limit_reissue.pine index 7b8fbcb..b402f2e 100644 --- a/docs/probes/tradingview-realtime/p3_stop_limit_reissue.pine +++ b/docs/probes/tradingview-realtime/p3_stop_limit_reissue.pine @@ -14,21 +14,43 @@ int retraceTicks = input.int(50, "Limit distance below stop (ticks)", minval=1) int changedLimitTicks = input.int(1, "Changed-limit delta (ticks)", minval=1) int timeoutBars = input.int(240, "Timeout bars", minval=3) bool arm = input.bool(true, "Arm at scheduled time") +string MODE = mode trace(string tag) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + '}}' -fill(string tag) => - '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","order_tag":"' + tag + - '","command_bar_index":' + str.tostring(bar_index) + - ',"command_bar_time_ms":' + str.tostring(time) + '}' +jsonNumber(float value) => na(value) ? "null" : str.tostring(value) +jsonStringOrNull(string value) => value == "" ? "null" : '"' + value + '"' +commandMessage(string tag, string api, string action, string orderId, + int sourceOrder, string side, string orderType, string fromEntry, + float qty, float stopValue, float limitValue, float trailPoints, + float trailOffset, string ocaName, string ocaType, string intent) => + '{"schema":"pf-tv-probe-command-v2","probe":{"id":"' + PROBE + + '","run_id":"' + runId + '","mode":"' + MODE + + '"},"command":{"tag":"' + tag + '","api":"' + api + + '","action":"' + action + '","order_id":"' + orderId + + '","source_order":' + str.tostring(sourceOrder) + ',"side":"' + side + + '","order_type":"' + orderType + '","from_entry":' + + jsonStringOrNull(fromEntry) + ',"qty":' + jsonNumber(qty) + + ',"stop_price":' + jsonNumber(stopValue) + ',"limit_price":' + + jsonNumber(limitValue) + ',"profit_ticks":null,"loss_ticks":null,"trail_points":' + + jsonNumber(trailPoints) + + ',"trail_offset":' + jsonNumber(trailOffset) + ',"oca_name":' + + jsonStringOrNull(ocaName) + ',"oca_type":' + jsonStringOrNull(ocaType) + + ',"debug_intent":"' + intent + '"},"message_evaluation":{"bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"server_time_ms":' + str.tostring(timenow) + + ',"position_size":' + str.tostring(strategy.position_size) + '}}' var int liveBars = 0 var float stopPrice = na @@ -67,26 +89,39 @@ if active if liveBars == 1 stopPrice := close + stopTicks * syminfo.mintick limitPrice := stopPrice - retraceTicks * syminfo.mintick + alert(trace("command_initial_stop_limit_issued"), alert.freq_all) strategy.entry("SL", strategy.long, qty=1, stop=stopPrice, - limit=limitPrice, alert_message=fill("initial_stop_limit")) + limit=limitPrice, alert_message=commandMessage( + "initial_stop_limit", "strategy.entry", "create", "SL", 1, + "long", "stop_limit", "", 1, stopPrice, limitPrice, na, na, + "", "", "establish stop-limit before activation and reissue")) alert(trace("stop_limit_placed"), alert.freq_all) else if sawStop and not reissued and strategy.position_size == 0 and close < stopPrice and close > limitPrice reissued := true if mode == "changed_limit" limitPrice -= changedLimitTicks * syminfo.mintick + alert(trace("command_reissued_stop_limit_issued"), alert.freq_all) strategy.entry("SL", strategy.long, qty=1, stop=stopPrice, - limit=limitPrice, alert_message=fill("reissued_stop_limit")) + limit=limitPrice, alert_message=commandMessage( + "reissued_stop_limit", "strategy.entry", "replace", "SL", 1, + "long", "stop_limit", "", 1, stopPrice, limitPrice, na, na, + "", "", "test activated stop-limit persistence after reissue")) alert(trace("reissued_between_stop_and_limit"), alert.freq_all) else if sawStop and not reissued and strategy.position_size > 0 and not aborted aborted := true alert(trace("inconclusive_filled_before_reissue"), alert.freq_all) else if not timedOut and liveBars >= timeoutBars timedOut := true + alert(trace("command_cancel_all_timeout"), alert.freq_all) strategy.cancel_all() if strategy.position_size != 0 + alert(trace("command_timeout_close_all_issued"), alert.freq_all) strategy.close_all(immediately=true, - alert_message=fill("timeout_close_all")) + alert_message=commandMessage("timeout_close_all", + "strategy.close_all", "cleanup", "__close_all__", 1, + "flat", "market_close", "", na, na, na, na, na, "", "", + "cleanup residual position after P3 timeout")) alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) plot(stopPrice, "stop", color=color.red) diff --git a/docs/probes/tradingview-realtime/p4_trailing_reissue.pine b/docs/probes/tradingview-realtime/p4_trailing_reissue.pine index 6919476..af7535c 100644 --- a/docs/probes/tradingview-realtime/p4_trailing_reissue.pine +++ b/docs/probes/tradingview-realtime/p4_trailing_reissue.pine @@ -14,34 +14,61 @@ int offsetTicks = input.int(300, "Initial trail offset (ticks)", minval=2) int changedOffsetTicks = input.int(100, "Changed-offset delta (ticks)", minval=1) int timeoutBars = input.int(240, "Timeout bars", minval=4) bool arm = input.bool(true, "Arm at scheduled time") +string MODE = mode trace(string tag) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + '}}' -fill(string tag) => - '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","order_tag":"' + tag + - '","command_bar_index":' + str.tostring(bar_index) + - ',"command_bar_time_ms":' + str.tostring(time) + '}' +jsonNumber(float value) => na(value) ? "null" : str.tostring(value) +jsonStringOrNull(string value) => value == "" ? "null" : '"' + value + '"' +commandMessage(string tag, string api, string action, string orderId, + int sourceOrder, string side, string orderType, string fromEntry, + float qty, float stopValue, float limitValue, float trailPoints, + float trailOffset, string ocaName, string ocaType, string intent) => + '{"schema":"pf-tv-probe-command-v2","probe":{"id":"' + PROBE + + '","run_id":"' + runId + '","mode":"' + MODE + + '"},"command":{"tag":"' + tag + '","api":"' + api + + '","action":"' + action + '","order_id":"' + orderId + + '","source_order":' + str.tostring(sourceOrder) + ',"side":"' + side + + '","order_type":"' + orderType + '","from_entry":' + + jsonStringOrNull(fromEntry) + ',"qty":' + jsonNumber(qty) + + ',"stop_price":' + jsonNumber(stopValue) + ',"limit_price":' + + jsonNumber(limitValue) + ',"profit_ticks":null,"loss_ticks":null,"trail_points":' + + jsonNumber(trailPoints) + + ',"trail_offset":' + jsonNumber(trailOffset) + ',"oca_name":' + + jsonStringOrNull(ocaName) + ',"oca_type":' + jsonStringOrNull(ocaType) + + ',"debug_intent":"' + intent + '"},"message_evaluation":{"bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"server_time_ms":' + str.tostring(timenow) + + ',"position_size":' + str.tostring(strategy.position_size) + '}}' traceCandidates(string tag, float watermark, float preservedLevel, float resetLevel) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"pre_reissue_watermark":' + str.tostring(watermark, format.mintick) + + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + + '},"debug":{"pre_reissue_watermark":' + str.tostring(watermark, format.mintick) + ',"preserved_stop_candidate":' + str.tostring(preservedLevel, format.mintick) + ',"reset_stop_candidate":' + str.tostring(resetLevel, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '}}' var int liveBars = 0 var bool entryPlaced = false @@ -102,15 +129,22 @@ if active liveBars += 1 if not entryPlaced entryPlaced := true + alert(trace("command_entry_issued"), alert.freq_all) strategy.entry("L", strategy.long, qty=1, - alert_message=fill("entry")) + alert_message=commandMessage("entry", "strategy.entry", + "create", "L", 1, "long", "market", "", 1, na, na, + na, na, "", "", "open position for trailing reissue probe")) alert(trace("market_entry_created"), alert.freq_all) else if strategy.position_size > 0 and not trailPlaced trailPlaced := true trailPlacedBar := bar_index + alert(trace("command_trail_initial_issued"), alert.freq_all) strategy.exit("TX", "L", trail_points=activationTicks, trail_offset=offsetTicks, - alert_message=fill("trail_initial")) + alert_message=commandMessage("trail_initial", "strategy.exit", + "create", "TX", 1, "exit_long", "trailing_exit", "L", na, + na, na, activationTicks, offsetTicks, "", "", + "create initial trailing exit before activation")) alert(trace("trail_initial_created"), alert.freq_all) else if strategy.position_size > 0 and activationObserved and not reissued and bar_index > trailPlacedBar and close < observedWatermark and @@ -122,9 +156,13 @@ if active preReissueWatermark := observedWatermark preservedStopCandidate := preReissueWatermark - nextOffset * syminfo.mintick resetStopCandidate := close - nextOffset * syminfo.mintick + alert(trace("command_trail_reissued_issued"), alert.freq_all) strategy.exit("TX", "L", trail_points=activationTicks, trail_offset=nextOffset, - alert_message=fill("trail_reissued")) + alert_message=commandMessage("trail_reissued", "strategy.exit", + "replace", "TX", 1, "exit_long", "trailing_exit", "L", na, + na, na, activationTicks, nextOffset, "", "", + "test whether reissue preserves the pre-reissue watermark")) alert(traceCandidates("trail_reissued_on_discriminating_retrace", preReissueWatermark, preservedStopCandidate, resetStopCandidate), alert.freq_all) @@ -133,10 +171,15 @@ if active alert(trace("inconclusive_trail_filled_before_reissue"), alert.freq_all) else if not timedOut and liveBars >= timeoutBars timedOut := true + alert(trace("command_cancel_TX_timeout"), alert.freq_all) strategy.cancel("TX") if strategy.position_size != 0 + alert(trace("command_timeout_close_issued"), alert.freq_all) strategy.close("L", immediately=true, - alert_message=fill("timeout_close")) + alert_message=commandMessage("timeout_close", + "strategy.close", "cleanup", "L", 1, "flat", + "market_close", "L", na, na, na, na, na, "", "", + "cleanup residual long position after P4 timeout")) alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) plot(observedWatermark, "observed tick watermark", color=color.aqua) diff --git a/docs/probes/tradingview-realtime/p5_exit_without_from_entry.pine b/docs/probes/tradingview-realtime/p5_exit_without_from_entry.pine index 32d4c1e..658cd52 100644 --- a/docs/probes/tradingview-realtime/p5_exit_without_from_entry.pine +++ b/docs/probes/tradingview-realtime/p5_exit_without_from_entry.pine @@ -13,21 +13,47 @@ int profitTicks = input.int(10000, "Exit profit distance (ticks)", minval=1) int lossTicks = input.int(10000, "Exit loss distance (ticks)", minval=1) int timeoutBars = input.int(360, "Timeout bars", minval=5) bool arm = input.bool(true, "Arm at scheduled time") +string MODE = mode trace(string tag) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + '}}' -fill(string tag) => - '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","order_tag":"' + tag + - '","command_bar_index":' + str.tostring(bar_index) + - ',"command_bar_time_ms":' + str.tostring(time) + '}' +jsonNumber(float value) => na(value) ? "null" : str.tostring(value) +jsonStringOrNull(string value) => value == "" ? "null" : '"' + value + '"' +commandMessage(string tag, string api, string action, string orderId, + int sourceOrder, string side, string orderType, string fromEntry, + float qty, float stopValue, float limitValue, float trailPoints, + float trailOffset, string ocaName, string ocaType, string intent) => + float profitValue = orderType == "relative_bracket" or + orderType == "profit_exit" ? profitTicks : na + float lossValue = orderType == "relative_bracket" or + orderType == "stop_loss" ? lossTicks : na + '{"schema":"pf-tv-probe-command-v2","probe":{"id":"' + PROBE + + '","run_id":"' + runId + '","mode":"' + MODE + + '"},"command":{"tag":"' + tag + '","api":"' + api + + '","action":"' + action + '","order_id":"' + orderId + + '","source_order":' + str.tostring(sourceOrder) + ',"side":"' + side + + '","order_type":"' + orderType + '","from_entry":' + + jsonStringOrNull(fromEntry) + ',"qty":' + jsonNumber(qty) + + ',"stop_price":' + jsonNumber(stopValue) + ',"limit_price":' + + jsonNumber(limitValue) + ',"profit_ticks":' + jsonNumber(profitValue) + + ',"loss_ticks":' + jsonNumber(lossValue) + ',"trail_points":' + jsonNumber(trailPoints) + + ',"trail_offset":' + jsonNumber(trailOffset) + ',"oca_name":' + + jsonStringOrNull(ocaName) + ',"oca_type":' + jsonStringOrNull(ocaType) + + ',"debug_intent":"' + intent + '"},"message_evaluation":{"bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"server_time_ms":' + str.tostring(timenow) + + ',"position_size":' + str.tostring(strategy.position_size) + '}}' var int liveBars = 0 var bool firstEntryPlaced = false @@ -51,32 +77,59 @@ if active and barstate.isconfirmed and bar_index > armedBar liveBars += 1 if not firstEntryPlaced firstEntryPlaced := true + alert(trace("command_entry_A_issued"), alert.freq_all) strategy.entry("A", strategy.long, qty=1, - alert_message=fill("entry_A")) + alert_message=commandMessage("entry_A", "strategy.entry", + "create", "A", 1, "long", "market", "", 1, na, na, na, na, + "", "", "open first lot before creating the unscoped exit")) alert(trace("entry_A_created"), alert.freq_all) else if strategy.position_size >= 1 and not exitPlaced exitPlaced := true exitPlacedBar := bar_index + alert(trace("command_exit_without_from_entry_issued"), alert.freq_all) strategy.exit("X", profit=profitTicks, loss=lossTicks, comment_profit="profit", comment_loss="loss", - alert_message=fill("exit_without_from_entry"), - alert_profit=fill("exit_profit"), alert_loss=fill("exit_loss")) + alert_message=commandMessage("exit_without_from_entry", + "strategy.exit", "create", "X", 1, "exit_long", + "relative_bracket", "", na, na, na, na, na, "", "", + "unscoped bracket created while only entry A exists"), + alert_profit=commandMessage("exit_profit", "strategy.exit", + "create", "X", 1, "exit_long", "profit_exit", "", na, na, na, + na, na, "", "", "profit child of exit X for original scope"), + alert_loss=commandMessage("exit_loss", "strategy.exit", "create", + "X", 1, "exit_long", "stop_loss", "", na, na, na, na, na, + "", "", "loss child of exit X for original scope")) alert(trace("exit_without_from_entry_created_for_A"), alert.freq_all) else if strategy.position_size >= 1 and exitPlaced and not secondEntryPlaced and bar_index > exitPlacedBar secondEntryPlaced := true + alert(trace("command_entry_B_later_issued"), alert.freq_all) strategy.entry("B", strategy.long, qty=1, - alert_message=fill("entry_B_later")) + alert_message=commandMessage("entry_B_later", "strategy.entry", + "create", "B", 1, "long", "market", "", 1, na, na, na, na, + "", "", "open later lot B after unscoped exit X already exists")) alert(trace("later_entry_B_created"), alert.freq_all) else if strategy.position_size >= 2 and not twoLotsObserved twoLotsObserved := true if mode == "positive_control_reissue_after_B" controlReissued := true + alert(trace("command_positive_control_exit_reissued_after_B_issued"), + alert.freq_all) strategy.exit("X", profit=profitTicks, loss=lossTicks, comment_profit="profit", comment_loss="loss", - alert_message=fill("positive_control_exit_reissued_after_B"), - alert_profit=fill("positive_control_profit"), - alert_loss=fill("positive_control_loss")) + alert_message=commandMessage( + "positive_control_exit_reissued_after_B", "strategy.exit", + "replace", "X", 1, "exit_long", "relative_bracket", "", na, + na, na, na, na, "", "", + "positive control reissues X after both A and B exist"), + alert_profit=commandMessage("positive_control_profit", + "strategy.exit", "replace", "X", 1, "exit_long", + "profit_exit", "", na, na, na, na, na, "", "", + "profit child of reissued positive-control exit X"), + alert_loss=commandMessage("positive_control_loss", + "strategy.exit", "replace", "X", 1, "exit_long", + "stop_loss", "", na, na, na, na, na, "", "", + "loss child of reissued positive-control exit X")) alert(trace("positive_control_exit_reissued_after_B"), alert.freq_all) else alert(trace("two_lots_open_exit_not_reissued"), alert.freq_all) @@ -86,8 +139,13 @@ if active and barstate.isconfirmed and bar_index > armedBar alert(trace("inconclusive_A_exited_before_B"), alert.freq_all) else if not timedOut and liveBars >= timeoutBars timedOut := true + alert(trace("command_cancel_all_timeout"), alert.freq_all) strategy.cancel_all() if strategy.position_size != 0 + alert(trace("command_timeout_close_all_issued"), alert.freq_all) strategy.close_all(immediately=true, - alert_message=fill("timeout_close_all")) + alert_message=commandMessage("timeout_close_all", + "strategy.close_all", "cleanup", "__close_all__", 1, + "flat", "market_close", "", na, na, na, na, na, "", "", + "cleanup residual lots after P5 timeout")) alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) diff --git a/docs/probes/tradingview-realtime/p6_from_entry_cutoff.pine b/docs/probes/tradingview-realtime/p6_from_entry_cutoff.pine index cfb5f8e..a6fda90 100644 --- a/docs/probes/tradingview-realtime/p6_from_entry_cutoff.pine +++ b/docs/probes/tradingview-realtime/p6_from_entry_cutoff.pine @@ -13,21 +13,47 @@ int profitTicks = input.int(1000, "Exit profit distance (ticks)", minval=1) int lossTicks = input.int(1000, "Exit loss distance (ticks)", minval=1) int timeoutBars = input.int(300, "Timeout bars", minval=4) bool arm = input.bool(true, "Arm at scheduled time") +string MODE = mode trace(string tag) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + '}}' -fill(string tag) => - '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","order_tag":"' + tag + - '","command_bar_index":' + str.tostring(bar_index) + - ',"command_bar_time_ms":' + str.tostring(time) + '}' +jsonNumber(float value) => na(value) ? "null" : str.tostring(value) +jsonStringOrNull(string value) => value == "" ? "null" : '"' + value + '"' +commandMessage(string tag, string api, string action, string orderId, + int sourceOrder, string side, string orderType, string fromEntry, + float qty, float stopValue, float limitValue, float trailPoints, + float trailOffset, string ocaName, string ocaType, string intent) => + float profitValue = orderType == "relative_bracket" or + orderType == "profit_exit" ? profitTicks : na + float lossValue = orderType == "relative_bracket" or + orderType == "stop_loss" ? lossTicks : na + '{"schema":"pf-tv-probe-command-v2","probe":{"id":"' + PROBE + + '","run_id":"' + runId + '","mode":"' + MODE + + '"},"command":{"tag":"' + tag + '","api":"' + api + + '","action":"' + action + '","order_id":"' + orderId + + '","source_order":' + str.tostring(sourceOrder) + ',"side":"' + side + + '","order_type":"' + orderType + '","from_entry":' + + jsonStringOrNull(fromEntry) + ',"qty":' + jsonNumber(qty) + + ',"stop_price":' + jsonNumber(stopValue) + ',"limit_price":' + + jsonNumber(limitValue) + ',"profit_ticks":' + jsonNumber(profitValue) + + ',"loss_ticks":' + jsonNumber(lossValue) + ',"trail_points":' + jsonNumber(trailPoints) + + ',"trail_offset":' + jsonNumber(trailOffset) + ',"oca_name":' + + jsonStringOrNull(ocaName) + ',"oca_type":' + jsonStringOrNull(ocaType) + + ',"debug_intent":"' + intent + '"},"message_evaluation":{"bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"server_time_ms":' + str.tostring(timenow) + + ',"position_size":' + str.tostring(strategy.position_size) + '}}' var int liveBars = 0 var bool initialCallsDone = false @@ -47,36 +73,80 @@ if active and barstate.isconfirmed and bar_index > armedBar if not initialCallsDone initialCallsDone := true if mode == "entry_then_exit" + alert(trace("command_entry_E_issued"), alert.freq_all) strategy.entry("E", strategy.long, qty=1, - alert_message=fill("entry_E")) + alert_message=commandMessage("entry_E", "strategy.entry", + "create", "E", 1, "long", "market", "", 1, na, na, na, na, + "", "", "source call 1 opens entry E")) + alert(trace("command_exit_after_entry_same_calculation_issued"), + alert.freq_all) strategy.exit("X", "E", profit=profitTicks, loss=lossTicks, comment_profit="profit", comment_loss="loss", - alert_message=fill("exit_after_entry_same_calculation"), - alert_profit=fill("profit_after_entry_same_calculation"), - alert_loss=fill("loss_after_entry_same_calculation")) + alert_message=commandMessage("exit_after_entry_same_calculation", + "strategy.exit", "create", "X", 2, "exit_long", + "relative_bracket", "E", na, na, na, na, na, "", "", + "source call 2 scopes X to E created earlier this calculation"), + alert_profit=commandMessage( + "profit_after_entry_same_calculation", "strategy.exit", + "create", "X", 2, "exit_long", "profit_exit", "E", na, na, + na, na, na, "", "", "profit child when entry precedes exit"), + alert_loss=commandMessage("loss_after_entry_same_calculation", + "strategy.exit", "create", "X", 2, "exit_long", "stop_loss", + "E", na, na, na, na, na, "", "", + "loss child when entry precedes exit")) alert(trace("entry_then_exit_calls_complete"), alert.freq_all) else + alert(trace("command_exit_before_entry_same_calculation_issued"), + alert.freq_all) strategy.exit("X", "E", profit=profitTicks, loss=lossTicks, comment_profit="profit", comment_loss="loss", - alert_message=fill("exit_before_entry_same_calculation"), - alert_profit=fill("profit_before_entry_same_calculation"), - alert_loss=fill("loss_before_entry_same_calculation")) + alert_message=commandMessage("exit_before_entry_same_calculation", + "strategy.exit", "create", "X", 1, "exit_long", + "relative_bracket", "E", na, na, na, na, na, "", "", + "source call 1 scopes X to E before E exists"), + alert_profit=commandMessage( + "profit_before_entry_same_calculation", "strategy.exit", + "create", "X", 1, "exit_long", "profit_exit", "E", na, na, + na, na, na, "", "", "profit child when exit precedes entry"), + alert_loss=commandMessage("loss_before_entry_same_calculation", + "strategy.exit", "create", "X", 1, "exit_long", "stop_loss", + "E", na, na, na, na, na, "", "", + "loss child when exit precedes entry")) + alert(trace("command_entry_E_issued"), alert.freq_all) strategy.entry("E", strategy.long, qty=1, - alert_message=fill("entry_E")) + alert_message=commandMessage("entry_E", "strategy.entry", + "create", "E", 2, "long", "market", "", 1, na, na, na, na, + "", "", "source call 2 opens E after scoped exit X call")) alert(trace("exit_then_entry_calls_complete"), alert.freq_all) else if mode == "exit_then_entry_reissue_after_fill" and strategy.position_size > 0 and not reissued reissued := true + alert(trace("command_exit_reissued_after_entry_fill_issued"), + alert.freq_all) strategy.exit("X", "E", profit=profitTicks, loss=lossTicks, comment_profit="profit", comment_loss="loss", - alert_message=fill("exit_reissued_after_entry_fill"), - alert_profit=fill("profit_reissued_after_entry_fill"), - alert_loss=fill("loss_reissued_after_entry_fill")) + alert_message=commandMessage("exit_reissued_after_entry_fill", + "strategy.exit", "replace", "X", 1, "exit_long", + "relative_bracket", "E", na, na, na, na, na, "", "", + "positive control reissues X after entry E has filled"), + alert_profit=commandMessage("profit_reissued_after_entry_fill", + "strategy.exit", "replace", "X", 1, "exit_long", "profit_exit", + "E", na, na, na, na, na, "", "", + "profit child of post-fill reissued X"), + alert_loss=commandMessage("loss_reissued_after_entry_fill", + "strategy.exit", "replace", "X", 1, "exit_long", "stop_loss", + "E", na, na, na, na, na, "", "", + "loss child of post-fill reissued X")) alert(trace("from_entry_exit_reissued_after_fill"), alert.freq_all) else if not timedOut and liveBars >= timeoutBars timedOut := true + alert(trace("command_cancel_all_timeout"), alert.freq_all) strategy.cancel_all() if strategy.position_size != 0 + alert(trace("command_timeout_close_all_issued"), alert.freq_all) strategy.close_all(immediately=true, - alert_message=fill("timeout_close_all")) + alert_message=commandMessage("timeout_close_all", + "strategy.close_all", "cleanup", "__close_all__", 1, + "flat", "market_close", "", na, na, na, na, na, "", "", + "cleanup residual position after P6 timeout")) alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) diff --git a/docs/probes/tradingview-realtime/p7_simultaneous_priority.pine b/docs/probes/tradingview-realtime/p7_simultaneous_priority.pine index 5f78acf..5a81cb0 100644 --- a/docs/probes/tradingview-realtime/p7_simultaneous_priority.pine +++ b/docs/probes/tradingview-realtime/p7_simultaneous_priority.pine @@ -12,21 +12,43 @@ string mode = input.string("exit_then_reversal", "Source-order mode", int marketableStopTicks = input.int(10, "Long-exit stop above close (ticks)", minval=1) int timeoutBars = input.int(60, "Timeout bars", minval=4) bool arm = input.bool(true, "Arm at scheduled time") +string MODE = mode trace(string tag) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + '}}' -fill(string tag) => - '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","order_tag":"' + tag + - '","command_bar_index":' + str.tostring(bar_index) + - ',"command_bar_time_ms":' + str.tostring(time) + '}' +jsonNumber(float value) => na(value) ? "null" : str.tostring(value) +jsonStringOrNull(string value) => value == "" ? "null" : '"' + value + '"' +commandMessage(string tag, string api, string action, string orderId, + int sourceOrder, string side, string orderType, string fromEntry, + float qty, float stopValue, float limitValue, float trailPoints, + float trailOffset, string ocaName, string ocaType, string intent) => + '{"schema":"pf-tv-probe-command-v2","probe":{"id":"' + PROBE + + '","run_id":"' + runId + '","mode":"' + MODE + + '"},"command":{"tag":"' + tag + '","api":"' + api + + '","action":"' + action + '","order_id":"' + orderId + + '","source_order":' + str.tostring(sourceOrder) + ',"side":"' + side + + '","order_type":"' + orderType + '","from_entry":' + + jsonStringOrNull(fromEntry) + ',"qty":' + jsonNumber(qty) + + ',"stop_price":' + jsonNumber(stopValue) + ',"limit_price":' + + jsonNumber(limitValue) + ',"profit_ticks":null,"loss_ticks":null,"trail_points":' + + jsonNumber(trailPoints) + + ',"trail_offset":' + jsonNumber(trailOffset) + ',"oca_name":' + + jsonStringOrNull(ocaName) + ',"oca_type":' + jsonStringOrNull(ocaType) + + ',"debug_intent":"' + intent + '"},"message_evaluation":{"bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"server_time_ms":' + str.tostring(timenow) + + ',"position_size":' + str.tostring(strategy.position_size) + '}}' var int liveBars = 0 var bool entryPlaced = false @@ -47,32 +69,56 @@ if active and barstate.isconfirmed and bar_index > armedBar liveBars += 1 if not entryPlaced entryPlaced := true + alert(trace("command_initial_long_issued"), alert.freq_all) strategy.entry("L", strategy.long, qty=1, - alert_message=fill("initial_long")) + alert_message=commandMessage("initial_long", "strategy.entry", + "create", "L", 1, "long", "market", "", 1, na, na, na, na, + "", "", "open long position before collision commands")) alert(trace("initial_long_created"), alert.freq_all) else if strategy.position_size > 0 and not collisionPlaced collisionPlaced := true collisionStop := close + marketableStopTicks * syminfo.mintick if mode == "exit_then_reversal" + alert(trace("command_marketable_exit_issued"), alert.freq_all) strategy.exit("X", "L", qty=1, stop=collisionStop, - alert_message=fill("marketable_exit")) + alert_message=commandMessage("marketable_exit", + "strategy.exit", "create", "X", 1, "exit_long", "stop_exit", + "L", 1, collisionStop, na, na, na, "", "", + "source call 1 creates marketable long exit")) + alert(trace("command_market_reversal_issued"), alert.freq_all) strategy.entry("REV", strategy.short, qty=1, - alert_message=fill("market_reversal")) + alert_message=commandMessage("market_reversal", + "strategy.entry", "create", "REV", 2, "short", + "market_reversal", "", 1, na, na, na, na, "", "", + "source call 2 creates short reversal")) else + alert(trace("command_market_reversal_issued"), alert.freq_all) strategy.entry("REV", strategy.short, qty=1, - alert_message=fill("market_reversal")) + alert_message=commandMessage("market_reversal", + "strategy.entry", "create", "REV", 1, "short", + "market_reversal", "", 1, na, na, na, na, "", "", + "source call 1 creates short reversal")) + alert(trace("command_marketable_exit_issued"), alert.freq_all) strategy.exit("X", "L", qty=1, stop=collisionStop, - alert_message=fill("marketable_exit")) + alert_message=commandMessage("marketable_exit", + "strategy.exit", "create", "X", 2, "exit_long", "stop_exit", + "L", 1, collisionStop, na, na, na, "", "", + "source call 2 creates marketable long exit")) alert(trace("collision_orders_created"), alert.freq_all) else if collisionPlaced and not postCollisionObserved postCollisionObserved := true alert(trace("first_confirmed_close_after_collision"), alert.freq_all) else if not timedOut and liveBars >= timeoutBars timedOut := true + alert(trace("command_cancel_all_timeout"), alert.freq_all) strategy.cancel_all() if strategy.position_size != 0 + alert(trace("command_timeout_close_all_issued"), alert.freq_all) strategy.close_all(immediately=true, - alert_message=fill("timeout_close_all")) + alert_message=commandMessage("timeout_close_all", + "strategy.close_all", "cleanup", "__close_all__", 1, + "flat", "market_close", "", na, na, na, na, na, "", "", + "cleanup residual position after P7 timeout")) alert(trace("timeout_cleanup"), alert.freq_all) plot(collisionStop, "marketable long-exit stop", color=color.red) diff --git a/docs/probes/tradingview-realtime/p8_oca_same_event.pine b/docs/probes/tradingview-realtime/p8_oca_same_event.pine index 2bda161..2ebdfbc 100644 --- a/docs/probes/tradingview-realtime/p8_oca_same_event.pine +++ b/docs/probes/tradingview-realtime/p8_oca_same_event.pine @@ -12,21 +12,43 @@ string mode = input.string("A_then_B", "Source-order mode", int marketableStopTicks = input.int(10, "Shared sell-stop distance above close (ticks)", minval=1) int timeoutBars = input.int(60, "Timeout bars", minval=4) bool arm = input.bool(true, "Arm at scheduled time") +string MODE = mode trace(string tag) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + '}}' -fill(string tag) => - '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","order_tag":"' + tag + - '","command_bar_index":' + str.tostring(bar_index) + - ',"command_bar_time_ms":' + str.tostring(time) + '}' +jsonNumber(float value) => na(value) ? "null" : str.tostring(value) +jsonStringOrNull(string value) => value == "" ? "null" : '"' + value + '"' +commandMessage(string tag, string api, string action, string orderId, + int sourceOrder, string side, string orderType, string fromEntry, + float qty, float stopValue, float limitValue, float trailPoints, + float trailOffset, string ocaName, string ocaType, string intent) => + '{"schema":"pf-tv-probe-command-v2","probe":{"id":"' + PROBE + + '","run_id":"' + runId + '","mode":"' + MODE + + '"},"command":{"tag":"' + tag + '","api":"' + api + + '","action":"' + action + '","order_id":"' + orderId + + '","source_order":' + str.tostring(sourceOrder) + ',"side":"' + side + + '","order_type":"' + orderType + '","from_entry":' + + jsonStringOrNull(fromEntry) + ',"qty":' + jsonNumber(qty) + + ',"stop_price":' + jsonNumber(stopValue) + ',"limit_price":' + + jsonNumber(limitValue) + ',"profit_ticks":null,"loss_ticks":null,"trail_points":' + + jsonNumber(trailPoints) + + ',"trail_offset":' + jsonNumber(trailOffset) + ',"oca_name":' + + jsonStringOrNull(ocaName) + ',"oca_type":' + jsonStringOrNull(ocaType) + + ',"debug_intent":"' + intent + '"},"message_evaluation":{"bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"server_time_ms":' + str.tostring(timenow) + + ',"position_size":' + str.tostring(strategy.position_size) + '}}' var int liveBars = 0 var bool entryPlaced = false @@ -47,36 +69,60 @@ if active and barstate.isconfirmed and bar_index > armedBar liveBars += 1 if not entryPlaced entryPlaced := true + alert(trace("command_initial_long_qty_2_issued"), alert.freq_all) strategy.entry("L", strategy.long, qty=2, - alert_message=fill("initial_long_qty_2")) + alert_message=commandMessage("initial_long_qty_2", + "strategy.entry", "create", "L", 1, "long", "market", "", 2, + na, na, na, na, "", "", "open quantity 2 for OCA collision")) alert(trace("initial_long_created"), alert.freq_all) else if strategy.position_size >= 2 and not siblingsPlaced siblingsPlaced := true sharedStop := close + marketableStopTicks * syminfo.mintick if mode == "A_then_B" + alert(trace("command_OCA_A_issued"), alert.freq_all) strategy.order("OCA_A", strategy.short, qty=1, stop=sharedStop, oca_name="P8_GROUP", oca_type=strategy.oca.cancel, - alert_message=fill("OCA_A")) + alert_message=commandMessage("OCA_A", "strategy.order", + "create", "OCA_A", 1, "short", "stop", "", 1, sharedStop, + na, na, na, "P8_GROUP", "cancel", + "source call 1 creates OCA cancel sibling A")) + alert(trace("command_OCA_B_issued"), alert.freq_all) strategy.order("OCA_B", strategy.short, qty=1, stop=sharedStop, oca_name="P8_GROUP", oca_type=strategy.oca.cancel, - alert_message=fill("OCA_B")) + alert_message=commandMessage("OCA_B", "strategy.order", + "create", "OCA_B", 2, "short", "stop", "", 1, sharedStop, + na, na, na, "P8_GROUP", "cancel", + "source call 2 creates OCA cancel sibling B")) else + alert(trace("command_OCA_B_issued"), alert.freq_all) strategy.order("OCA_B", strategy.short, qty=1, stop=sharedStop, oca_name="P8_GROUP", oca_type=strategy.oca.cancel, - alert_message=fill("OCA_B")) + alert_message=commandMessage("OCA_B", "strategy.order", + "create", "OCA_B", 1, "short", "stop", "", 1, sharedStop, + na, na, na, "P8_GROUP", "cancel", + "source call 1 creates OCA cancel sibling B")) + alert(trace("command_OCA_A_issued"), alert.freq_all) strategy.order("OCA_A", strategy.short, qty=1, stop=sharedStop, oca_name="P8_GROUP", oca_type=strategy.oca.cancel, - alert_message=fill("OCA_A")) + alert_message=commandMessage("OCA_A", "strategy.order", + "create", "OCA_A", 2, "short", "stop", "", 1, sharedStop, + na, na, na, "P8_GROUP", "cancel", + "source call 2 creates OCA cancel sibling A")) alert(trace("same_price_oca_siblings_created"), alert.freq_all) else if siblingsPlaced and not postCollisionObserved postCollisionObserved := true alert(trace("first_confirmed_close_after_oca_collision"), alert.freq_all) else if not timedOut and liveBars >= timeoutBars timedOut := true + alert(trace("command_cancel_all_timeout"), alert.freq_all) strategy.cancel_all() if strategy.position_size != 0 + alert(trace("command_timeout_close_all_issued"), alert.freq_all) strategy.close_all(immediately=true, - alert_message=fill("timeout_close_all")) + alert_message=commandMessage("timeout_close_all", + "strategy.close_all", "cleanup", "__close_all__", 1, + "flat", "market_close", "", na, na, na, na, na, "", "", + "cleanup residual position after P8 timeout")) alert(trace("timeout_cleanup"), alert.freq_all) plot(sharedStop, "shared marketable stop", color=color.red) diff --git a/docs/probes/tradingview-realtime/p9_per_lot_stop_selection.pine b/docs/probes/tradingview-realtime/p9_per_lot_stop_selection.pine index 46f197f..74258f8 100644 --- a/docs/probes/tradingview-realtime/p9_per_lot_stop_selection.pine +++ b/docs/probes/tradingview-realtime/p9_per_lot_stop_selection.pine @@ -14,21 +14,45 @@ int trailActivationTicks = input.int(500, "Trail activation distance (ticks)", m int trailOffsetTicks = input.int(300, "Trail offset (ticks)", minval=1) int timeoutBars = input.int(480, "Timeout bars", minval=8) bool arm = input.bool(true, "Arm at scheduled time") +string MODE = "qty1_qty2_any" trace(string tag) => - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"time_close_ms":' + str.tostring(time_close) + - ',"price":' + str.tostring(close, format.mintick) + - ',"position_size":' + str.tostring(strategy.position_size) + '}' + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + '}}' -fill(string tag) => - '{"schema":"pf-tv-probe-order-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","order_tag":"' + tag + - '","command_bar_index":' + str.tostring(bar_index) + - ',"command_bar_time_ms":' + str.tostring(time) + '}' +jsonNumber(float value) => na(value) ? "null" : str.tostring(value) +jsonStringOrNull(string value) => value == "" ? "null" : '"' + value + '"' +commandMessage(string tag, string api, string action, string orderId, + int sourceOrder, string side, string orderType, string fromEntry, + float qty, float stopValue, float limitValue, float trailPoints, + float trailOffset, string ocaName, string ocaType, string intent) => + float lossValue = orderType == "relative_stop_or_trail" or + orderType == "stop_loss" ? lossTicks : na + '{"schema":"pf-tv-probe-command-v2","probe":{"id":"' + PROBE + + '","run_id":"' + runId + '","mode":"' + MODE + + '"},"command":{"tag":"' + tag + '","api":"' + api + + '","action":"' + action + '","order_id":"' + orderId + + '","source_order":' + str.tostring(sourceOrder) + ',"side":"' + side + + '","order_type":"' + orderType + '","from_entry":' + + jsonStringOrNull(fromEntry) + ',"qty":' + jsonNumber(qty) + + ',"stop_price":' + jsonNumber(stopValue) + ',"limit_price":' + + jsonNumber(limitValue) + ',"profit_ticks":null,"loss_ticks":' + + jsonNumber(lossValue) + ',"trail_points":' + jsonNumber(trailPoints) + + ',"trail_offset":' + jsonNumber(trailOffset) + ',"oca_name":' + + jsonStringOrNull(ocaName) + ',"oca_type":' + jsonStringOrNull(ocaType) + + ',"debug_intent":"' + intent + '"},"message_evaluation":{"bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"server_time_ms":' + str.tostring(timenow) + + ',"position_size":' + str.tostring(strategy.position_size) + '}}' traceLots(string tag, float entry0, int entryTime0, bool active0, float watermark0, float entry1, int entryTime1, bool active1, @@ -39,25 +63,31 @@ traceLots(string tag, float entry0, int entryTime0, bool active0, float activation1 = entry1 + trailActivationTicks * syminfo.mintick float trailing0 = watermark0 - trailOffsetTicks * syminfo.mintick float trailing1 = watermark1 - trailOffsetTicks * syminfo.mintick - '{"schema":"pf-tv-probe-trace-v1","probe":"' + PROBE + - '","run_id":"' + runId + '","tag":"' + tag + - '","bar_index":' + str.tostring(bar_index) + - ',"bar_time_ms":' + str.tostring(time) + - ',"price":' + str.tostring(close, format.mintick) + - ',"lot0_entry_time_ms":' + str.tostring(entryTime0) + - ',"lot0_entry_price":' + str.tostring(entry0, format.mintick) + - ',"lot0_fixed_stop":' + str.tostring(fixed0, format.mintick) + - ',"lot0_activation":' + str.tostring(activation0, format.mintick) + + '{"schema":"pf-tv-probe-event-v2","source":"tradingview","event_type":"trace","event_key_hint":"' + + PROBE + '|' + runId + '|trace|' + tag + '|' + str.tostring(bar_index) + + '|' + str.tostring(time) + '|' + str.tostring(timenow) + + '","probe":{"id":"' + PROBE + '","run_id":"' + runId + + '","mode":"' + MODE + '"},"event":{"name":"' + tag + + '"},"market":{"tickerid":"' + syminfo.tickerid + + '","interval":"' + timeframe.period + '","bar_index":' + + str.tostring(bar_index) + ',"bar_time_ms":' + str.tostring(time) + + ',"bar_close_ms":' + str.tostring(time_close) + ',"server_time_ms":' + + str.tostring(timenow) + ',"price":' + str.tostring(close, format.mintick) + + '},"strategy":{"position_size":' + str.tostring(strategy.position_size) + + '},"debug":{"lot0_entry_time_ms":' + str.tostring(entryTime0) + + ',"lot0_entry_price":' + jsonNumber(entry0) + + ',"lot0_fixed_stop":' + jsonNumber(fixed0) + + ',"lot0_activation":' + jsonNumber(activation0) + ',"lot0_active":' + str.tostring(active0) + - ',"lot0_watermark":' + str.tostring(watermark0, format.mintick) + - ',"lot0_trailing_candidate":' + str.tostring(trailing0, format.mintick) + + ',"lot0_watermark":' + jsonNumber(watermark0) + + ',"lot0_trailing_candidate":' + jsonNumber(trailing0) + ',"lot1_entry_time_ms":' + str.tostring(entryTime1) + - ',"lot1_entry_price":' + str.tostring(entry1, format.mintick) + - ',"lot1_fixed_stop":' + str.tostring(fixed1, format.mintick) + - ',"lot1_activation":' + str.tostring(activation1, format.mintick) + + ',"lot1_entry_price":' + jsonNumber(entry1) + + ',"lot1_fixed_stop":' + jsonNumber(fixed1) + + ',"lot1_activation":' + jsonNumber(activation1) + ',"lot1_active":' + str.tostring(active1) + - ',"lot1_watermark":' + str.tostring(watermark1, format.mintick) + - ',"lot1_trailing_candidate":' + str.tostring(trailing1, format.mintick) + '}' + ',"lot1_watermark":' + jsonNumber(watermark1) + + ',"lot1_trailing_candidate":' + jsonNumber(trailing1) + '}}' var int liveBars = 0 var int firstEntryBar = na @@ -116,16 +146,22 @@ if active and barstate.isconfirmed and bar_index > armedBar if not firstEntryPlaced firstEntryPlaced := true firstEntryBar := bar_index + alert(trace("command_same_id_lot_1_issued"), alert.freq_all) strategy.entry("L", strategy.long, qty=1, - alert_message=fill("same_id_lot_1")) + alert_message=commandMessage("same_id_lot_1", "strategy.entry", + "create", "L", 1, "long", "market", "", 1, na, na, na, na, + "", "", "create first same-ID lot with quantity 1")) alert(trace("first_same_id_entry_created"), alert.freq_all) else if strategy.position_size >= 1 and not secondEntryPlaced and bar_index >= firstEntryBar + secondEntryDelayBars and math.abs(close - strategy.position_avg_price) >= minimumEntrySeparationTicks * syminfo.mintick secondEntryPlaced := true + alert(trace("command_same_id_lot_2_issued"), alert.freq_all) strategy.entry("L", strategy.long, qty=2, - alert_message=fill("same_id_lot_2")) + alert_message=commandMessage("same_id_lot_2", "strategy.entry", + "add", "L", 1, "long", "market", "", 2, na, na, na, na, + "", "", "add second same-ID lot with quantity 2")) alert(trace("second_same_id_entry_created"), alert.freq_all) else if strategy.position_size >= 3 and not exitPlaced and not separationRejected lot0Entry := strategy.opentrades.entry_price(0) @@ -142,20 +178,36 @@ if active and barstate.isconfirmed and bar_index > armedBar lot1EntryTime, lot1Active, lot1Watermark), alert.freq_all) else exitPlaced := true + alert(trace("command_shared_exit_fixed_stop_or_trail_issued"), + alert.freq_all) strategy.exit("X", "L", loss=lossTicks, trail_points=trailActivationTicks, trail_offset=trailOffsetTicks, comment_loss="fixed_stop", comment_trailing="trailing_stop", - alert_message=fill("shared_exit_fixed_stop_or_trail"), - alert_loss=fill("fixed_stop_fill"), - alert_trailing=fill("trailing_stop_fill")) + alert_message=commandMessage("shared_exit_fixed_stop_or_trail", + "strategy.exit", "create", "X", 1, "exit_long", + "relative_stop_or_trail", "L", na, na, na, + trailActivationTicks, trailOffsetTicks, "", "", + "shared exit X chooses fixed loss or trailing stop per lot"), + alert_loss=commandMessage("fixed_stop_fill", "strategy.exit", + "create", "X", 1, "exit_long", "stop_loss", "L", na, na, + na, na, na, "", "", "fixed-stop child selected for a lot"), + alert_trailing=commandMessage("trailing_stop_fill", + "strategy.exit", "create", "X", 1, "exit_long", + "trailing_exit", "L", na, na, na, trailActivationTicks, + trailOffsetTicks, "", "", "trailing-stop child selected for a lot")) alert(traceLots("shared_exit_created_for_two_lots", lot0Entry, lot0EntryTime, lot0Active, lot0Watermark, lot1Entry, lot1EntryTime, lot1Active, lot1Watermark), alert.freq_all) else if not timedOut and liveBars >= timeoutBars timedOut := true + alert(trace("command_cancel_all_timeout"), alert.freq_all) strategy.cancel_all() if strategy.position_size != 0 + alert(trace("command_timeout_close_all_issued"), alert.freq_all) strategy.close_all(immediately=true, - alert_message=fill("timeout_close_all")) + alert_message=commandMessage("timeout_close_all", + "strategy.close_all", "cleanup", "__close_all__", 1, + "flat", "market_close", "", na, na, na, na, na, "", "", + "cleanup residual lots after P9 timeout")) alert(trace("timeout_cleanup_inconclusive"), alert.freq_all) diff --git a/docs/probes/tradingview-realtime/validate_capture.py b/docs/probes/tradingview-realtime/validate_capture.py index 7f4c345..81bcd13 100644 --- a/docs/probes/tradingview-realtime/validate_capture.py +++ b/docs/probes/tradingview-realtime/validate_capture.py @@ -12,10 +12,24 @@ import hashlib import json from pathlib import Path +import re -TRACE_SCHEMA = "pf-tv-probe-trace-v1" -FILL_SCHEMA = "pf-tv-probe-fill-v1" +EVENT_SCHEMA = "pf-tv-probe-event-v2" +COMMAND_SCHEMA = "pf-tv-probe-command-v2" +RUN_ID_RE = re.compile(r"^[A-Za-z0-9._-]+$") + + +def reject_json_constant(value: str) -> None: + raise ValueError(f"non-standard JSON number {value}") + + +def strict_json_loads(raw: str) -> object: + return json.loads(raw, parse_constant=reject_json_constant) + + +def is_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) def sha256(path: Path) -> str: @@ -68,7 +82,12 @@ def main() -> int: manifest_path = run_dir / "manifest.json" if not manifest_path.is_file(): fail(f"missing {manifest_path}") - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + try: + manifest = strict_json_loads(manifest_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, ValueError) as exc: + fail(f"invalid manifest JSON: {exc}") + if not isinstance(manifest, dict): + fail("manifest root must be an object") if manifest.get("schema") != "pf-tv-probe-manifest-v1": fail("manifest schema must be pf-tv-probe-manifest-v1") @@ -91,14 +110,21 @@ def main() -> int: errors: list[str] = [] probe = manifest.get("probe") run_id = manifest.get("run_id") + mode = manifest.get("mode") if not isinstance(probe, str) or not probe.startswith("P"): errors.append("manifest probe must be a P-number string") if not isinstance(run_id, str) or not run_id: errors.append("manifest run_id must be non-empty") - if manifest.get("mode") in {None, "", "replace-me"}: + elif not RUN_ID_RE.fullmatch(run_id): + errors.append("manifest run_id must match [A-Za-z0-9._-]+") + if mode is None or mode == "" or mode == "replace-me": errors.append("manifest mode is not populated") if manifest.get("alert", {}).get("alert_id") in {None, "", "replace-me"}: errors.append("manifest alert.alert_id is not populated") + if manifest.get("alert", {}).get("event_schema") != EVENT_SCHEMA: + errors.append(f"manifest alert.event_schema must be {EVENT_SCHEMA}") + if manifest.get("alert", {}).get("command_schema") != COMMAND_SCHEMA: + errors.append(f"manifest alert.command_schema must be {COMMAND_SCHEMA}") scheduled = parse_utc( manifest.get("scheduled_start_utc"), "scheduled_start_utc", errors) @@ -119,32 +145,149 @@ def main() -> int: events: list[dict] = [] for line_number, raw in enumerate(raw_lines, 1): try: - event = json.loads(raw) - except json.JSONDecodeError as exc: + event = strict_json_loads(raw) + except (json.JSONDecodeError, ValueError) as exc: errors.append(f"webhook line {line_number}: invalid JSON: {exc}") continue + if not isinstance(event, dict): + errors.append(f"webhook line {line_number}: root must be an object") + continue schema = event.get("schema") - if schema == TRACE_SCHEMA: - event_probe = event.get("probe") - event_run_id = event.get("run_id") - if not isinstance(event.get("tag"), str): - errors.append(f"webhook line {line_number}: trace tag missing") - elif schema == FILL_SCHEMA: - payload = event.get("probe_payload") + if schema != EVENT_SCHEMA: + errors.append(f"webhook line {line_number}: unknown schema {schema!r}") + continue + if event.get("source") != "tradingview": + errors.append(f"webhook line {line_number}: source must be tradingview") + if not isinstance(event.get("event_key_hint"), str) or not event.get( + "event_key_hint"): + errors.append(f"webhook line {line_number}: event_key_hint missing") + event_type = event.get("event_type") + if event_type == "trace": + identity = event.get("probe") + if not isinstance(identity, dict): + errors.append(f"webhook line {line_number}: trace probe is not an object") + continue + event_probe = identity.get("id") + event_run_id = identity.get("run_id") + event_mode = identity.get("mode") + if not isinstance(event_mode, str): + errors.append(f"webhook line {line_number}: trace mode missing") + trace_event = event.get("event") + if not isinstance(trace_event, dict) or not isinstance( + trace_event.get("name"), str): + errors.append(f"webhook line {line_number}: trace event.name missing") + market = event.get("market") + if not isinstance(market, dict): + errors.append(f"webhook line {line_number}: trace market missing") + else: + for field in ("tickerid", "interval"): + if not isinstance(market.get(field), str) or not market.get(field): + errors.append( + f"webhook line {line_number}: market.{field} missing") + for field in ("bar_index", "bar_time_ms", "bar_close_ms", + "server_time_ms"): + if not isinstance(market.get(field), int) or isinstance( + market.get(field), bool): + errors.append( + f"webhook line {line_number}: market.{field} must be int") + if not is_number(market.get("price")): + errors.append( + f"webhook line {line_number}: market.price must be numeric") + strategy = event.get("strategy") + if not isinstance(strategy, dict) or not is_number( + strategy.get("position_size")): + errors.append( + f"webhook line {line_number}: strategy.position_size missing") + elif event_type == "order_fill": + payload = event.get("command_context") if not isinstance(payload, dict): errors.append( - f"webhook line {line_number}: fill probe_payload is not an object") + f"webhook line {line_number}: command_context is not an object") + continue + if payload.get("schema") != COMMAND_SCHEMA: + errors.append( + f"webhook line {line_number}: command_context schema invalid") + identity = payload.get("probe") + if not isinstance(identity, dict): + errors.append( + f"webhook line {line_number}: command_context.probe is not an object") continue - event_probe = payload.get("probe") - event_run_id = payload.get("run_id") - if not isinstance(payload.get("order_tag"), str): - errors.append(f"webhook line {line_number}: fill order_tag missing") - if not isinstance(event.get("fill_bar_time"), str): - errors.append(f"webhook line {line_number}: fill_bar_time missing") - if not isinstance(event.get("server_time"), str): - errors.append(f"webhook line {line_number}: server_time missing") + event_probe = identity.get("id") + event_run_id = identity.get("run_id") + event_mode = identity.get("mode") + if not isinstance(event_mode, str): + errors.append(f"webhook line {line_number}: command mode missing") + command = payload.get("command") + required_string_fields = ( + "tag", "api", "action", "order_id", "side", "order_type", + "debug_intent") + if not isinstance(command, dict): + errors.append(f"webhook line {line_number}: command missing") + else: + for field in required_string_fields: + if not isinstance(command.get(field), str) or not command.get(field): + errors.append( + f"webhook line {line_number}: command.{field} must be nonempty string") + for field in ("from_entry", "oca_name", "oca_type"): + value = command.get(field) + if field not in command or ( + value is not None and not isinstance(value, str)): + errors.append( + f"webhook line {line_number}: command.{field} invalid") + if not isinstance(command.get("source_order"), int) or isinstance( + command.get("source_order"), bool) or command.get( + "source_order") < 1: + errors.append( + f"webhook line {line_number}: command.source_order must be positive int") + numeric_fields = ("qty", "stop_price", "limit_price", + "profit_ticks", "loss_ticks", "trail_points", + "trail_offset") + for field in numeric_fields: + value = command.get(field) + if field not in command or ( + value is not None and not is_number(value)): + errors.append( + f"webhook line {line_number}: command.{field} invalid") + evaluation = payload.get("message_evaluation") + if not isinstance(evaluation, dict): + errors.append( + f"webhook line {line_number}: message_evaluation missing") + else: + for field in ("bar_index", "bar_time_ms", "server_time_ms"): + if not isinstance(evaluation.get(field), int) or isinstance( + evaluation.get(field), bool): + errors.append( + f"webhook line {line_number}: message_evaluation.{field} must be int") + if not is_number(evaluation.get("position_size")): + errors.append( + f"webhook line {line_number}: message_evaluation.position_size invalid") + market = event.get("market") + if not isinstance(market, dict): + errors.append(f"webhook line {line_number}: fill market missing") + else: + for field in ("ticker", "exchange", "interval", + "fill_bar_time", "server_time"): + if not isinstance(market.get(field), str) or not market.get(field): + errors.append( + f"webhook line {line_number}: market.{field} missing") + fill = event.get("fill") + if not isinstance(fill, dict): + errors.append(f"webhook line {line_number}: fill object missing") + else: + for field in ("order_id", "action", "market_position", + "previous_market_position"): + if not isinstance(fill.get(field), str) or not fill.get(field): + errors.append( + f"webhook line {line_number}: fill.{field} missing") + for field in ("contracts", "price", "position_size", + "market_position_size", + "previous_market_position_size"): + if not is_number(fill.get(field)): + errors.append( + f"webhook line {line_number}: fill.{field} must be numeric") else: - errors.append(f"webhook line {line_number}: unknown schema {schema!r}") + errors.append( + f"webhook line {line_number}: unknown event_type {event_type!r}") continue if event_probe != probe: errors.append( @@ -152,11 +295,14 @@ def main() -> int: if event_run_id != run_id: errors.append( f"webhook line {line_number}: run_id {event_run_id!r} != {run_id!r}") + if event_mode != mode: + errors.append( + f"webhook line {line_number}: mode {event_mode!r} != {mode!r}") events.append(event) armed_events = [event for event in events - if event.get("schema") == TRACE_SCHEMA - and event.get("tag") == "armed"] + if event.get("event_type") == "trace" + and event.get("event", {}).get("name") == "armed"] if len(armed_events) != 1: errors.append(f"expected exactly one armed trace, got {len(armed_events)}") @@ -179,10 +325,13 @@ def main() -> int: if not raw.strip(): continue try: - record = json.loads(raw) - except json.JSONDecodeError as exc: + record = strict_json_loads(raw) + except (json.JSONDecodeError, ValueError) as exc: errors.append(f"receipt line {line_number}: invalid JSON: {exc}") continue + if not isinstance(record, dict): + errors.append(f"receipt line {line_number}: root must be an object") + continue receipts.append(record) if len(receipts) != len(raw_lines): errors.append( @@ -190,15 +339,24 @@ def main() -> int: sequences = [record.get("sequence") for record in receipts] if not sequences: errors.append("receipt.jsonl is empty") - elif not all(isinstance(value, int) for value in sequences): + elif not all(isinstance(value, int) and not isinstance(value, bool) + for value in sequences): errors.append("every receipt sequence must be an integer") elif sequences != list(range(sequences[0], sequences[0] + len(sequences))): errors.append("receipt sequences must be contiguous and arrival-ordered") for index, (record, raw) in enumerate(zip(receipts, raw_lines), 1): + if not isinstance(record.get("receipt_id"), int) or isinstance( + record.get("receipt_id"), bool): + errors.append(f"receipt {index}: receipt_id must be an integer") if record.get("body_sha256") != line_sha256(raw): errors.append(f"receipt {index}: body_sha256 mismatch") parse_utc(record.get("received_at_utc"), f"receipt {index}.received_at_utc", errors) + receipt_ids = [record.get("receipt_id") for record in receipts] + if receipt_ids and all(isinstance(value, int) and not isinstance(value, bool) + for value in receipt_ids): + if any(right <= left for left, right in zip(receipt_ids, receipt_ids[1:])): + errors.append("receipt_id values must increase in arrival order") if sequences: if observation.get("receiver_first_sequence") != sequences[0]: errors.append("receiver_first_sequence does not match receipts") @@ -252,9 +410,9 @@ def main() -> int: "run_id": run_id, "events": len(events), "trace_events": sum( - event.get("schema") == TRACE_SCHEMA for event in events), + event.get("event_type") == "trace" for event in events), "fill_events": sum( - event.get("schema") == FILL_SCHEMA for event in events), + event.get("event_type") == "order_fill" for event in events), "artifact_hashes": actual_hashes, "semantic_review_required": True, "errors": errors,