From 8e3a9a7c45ec0cedce09d2bbb83054c67ffc32fb Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Tue, 7 Jul 2026 18:02:35 +0800 Subject: [PATCH] fix(codegen): parse "YYYY-MM-DD HH:MM UTC" timestamps + accept ignore_invalid_symbol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two additive transpile fixes that unblock valid community Pine (each recovered a data/ strategy from transpile-error to EXCELLENT, 100% match / cntAbs 0): 1. timestamp(dateString): _parse_pine_datestring_ms couldn't read a trailing tz WORD ("2024-01-01 00:00 UTC", "1 Jan 2020 09:30 GMT+2") or the space-separated "YYYY-MM-DD HH:MM[:SS]" form. Peel the UTC/GMT[±H[:MM]] word into an explicit offset and add the Y-M-D strptime formats. Purely additive — strings that already parsed are unchanged (the regex is a no-op without a trailing tz word). Recovers yukozb-gold-ny-orb. 2. request.security ignore_invalid_symbol: was hard-rejected ("reject loudly"), but it is a GUARANTEED no-op here — codegen forces request.security onto the current chart symbol (the "symbol must reference current chart symbol" check), which is always valid, so the flag can never change the result. Move it into SECURITY_ALLOWED_PARAMS (accept + drop), alongside backadjustment/adjustment. `currency` stays hard-rejected (it changes values via FX conversion). Recovers officialjack-helios. Also fixes a pre-existing stale KI-35 assertion in test_compile_smoke.py (test_regression_year_function_call_form_compiles asserted the old gmtime_r inline lambda; year()/hour() now lower to pine_() engine helpers — PR #42 leftover that PR #43 missed). Tests: new timestamp UTC/GMT-word cases; ignore_invalid_symbol now asserted accepted; locked-params snapshot updated. Full suite green. Co-Authored-By: Claude Opus 4.8 (1M context) --- pineforge_codegen/codegen/visit_call.py | 17 +++++++++++++++-- pineforge_codegen/support_checker.py | 8 ++++++-- tests/test_codegen_audit_fixes.py | 14 ++++++++++++++ tests/test_compile_smoke.py | 13 ++++++++----- tests/test_support_checker.py | 9 +++++++-- 5 files changed, 50 insertions(+), 11 deletions(-) diff --git a/pineforge_codegen/codegen/visit_call.py b/pineforge_codegen/codegen/visit_call.py index a35f71f..5ec9af5 100644 --- a/pineforge_codegen/codegen/visit_call.py +++ b/pineforge_codegen/codegen/visit_call.py @@ -176,9 +176,21 @@ def _parse_pine_datestring_ms(text: str) -> int | None: "MMM DD YYYY ..." forms. A dateString without a time zone is GMT+0 per the Pine reference. Returns None when the string cannot be parsed. """ - from datetime import datetime, timezone + import re + from datetime import datetime, timezone, timedelta txt = text.strip() + # Pine dateStrings may carry a trailing timezone WORD ("2024-01-01 00:00 UTC", + # "1 Jan 2020 09:30 GMT+2") that neither fromisoformat nor the strptime forms + # below can read. Peel it off and fold it into an explicit offset (UTC/GMT + # with an optional ±H[:MM] suffix; the bare word is +00:00). + tzoff = None + m = re.search(r"\s+(?:UTC|GMT)([+-]\d{1,2})?(?::?(\d{2}))?$", txt, re.I) + if m: + txt = txt[: m.start()].strip() + h = int(m.group(1) or 0) + mm = int(m.group(2) or 0) + tzoff = timezone(timedelta(hours=h, minutes=(mm if h >= 0 else -mm))) dt = None try: dt = datetime.fromisoformat(txt) @@ -188,6 +200,7 @@ def _parse_pine_datestring_ms(text: str) -> int | None: "%d %b %Y %H:%M:%S", "%d %b %Y %H:%M", "%d %b %Y", "%b %d %Y %H:%M:%S %z", "%b %d %Y %H:%M %z", "%b %d %Y %H:%M:%S", "%b %d %Y %H:%M", "%b %d %Y", + "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d", ): try: dt = datetime.strptime(txt, fmt) @@ -197,7 +210,7 @@ def _parse_pine_datestring_ms(text: str) -> int | None: if dt is None: return None if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) + dt = dt.replace(tzinfo=(tzoff or timezone.utc)) return int(dt.timestamp() * 1000) diff --git a/pineforge_codegen/support_checker.py b/pineforge_codegen/support_checker.py index f9cffb1..b17a32b 100644 --- a/pineforge_codegen/support_checker.py +++ b/pineforge_codegen/support_checker.py @@ -342,10 +342,14 @@ # request.security parameter rules. # Codegen supports symbol/timeframe/expression plus gaps/lookahead (read in # _eval_security_* emission and forwarded to register_security_eval). -# ignore_invalid_symbol/currency are accepted by the parser but silently -# dropped by codegen — reject loudly to surface unsupported behavior. +# `currency` is still rejected loudly (it changes the returned values via FX +# conversion, which codegen drops — silently wrong). `ignore_invalid_symbol` is +# a guaranteed no-op here: codegen forces request.security onto the current +# chart symbol (see the "symbol must reference current chart symbol" check), +# which is always valid, so the flag can never change the result — accept+ignore. SECURITY_ALLOWED_PARAMS: frozenset[str] = frozenset( {"symbol", "timeframe", "expression", "gaps", "lookahead", + "ignore_invalid_symbol", # Data-adjustment constants — silently accepted and ignored by codegen; # the underlying engine uses a fixed unadjusted data source. "backadjustment", "settlement_as_close", "adjustment"} diff --git a/tests/test_codegen_audit_fixes.py b/tests/test_codegen_audit_fixes.py index 2f7dfa8..c6bb695 100644 --- a/tests/test_codegen_audit_fixes.py +++ b/tests/test_codegen_audit_fixes.py @@ -108,6 +108,20 @@ def test_timestamp_datestring_dd_mmm_yyyy_parsed(): assert "1626728400000LL" in cpp +def test_timestamp_datestring_ymd_utc_word_parsed(): + # "YYYY-MM-DD HH:MM UTC" — space-separated with a trailing tz WORD (not a + # numeric offset). 2024-01-01 00:00 UTC = 1704067200000 ms. + cpp = _gen('t = timestamp("2024-01-01 00:00 UTC")\nplot(close)\n') + assert "1704067200000LL" in cpp + + +def test_timestamp_datestring_gmt_offset_word_parsed(): + # "GMT+2" trailing word = +2h offset. 1 Jan 2020 09:30 GMT+2 = 07:30 UTC + # = 1577863800000 ms. + cpp = _gen('t = timestamp("1 Jan 2020 09:30 GMT+2")\nplot(close)\n') + assert "1577863800000LL" in cpp + + def test_timestamp_unparseable_datestring_rejected(): with pytest.raises(CompileError, match="could not parse"): _gen('t = timestamp("not a date")\nplot(close)\n') diff --git a/tests/test_compile_smoke.py b/tests/test_compile_smoke.py index 0990bcc..c69cffe 100644 --- a/tests/test_compile_smoke.py +++ b/tests/test_compile_smoke.py @@ -517,11 +517,14 @@ def test_regression_year_function_call_form_compiles(): cpp = transpile(src) # Spot-check the lowering shape so anyone deleting the special case # in visit_call.py learns about it from the assertion, not from a - # mysterious clang error. - assert "gmtime_r" in cpp, ( - "year(time) etc. should lower to a gmtime_r-based extraction lambda; " - "if you intentionally moved the implementation to a runtime helper, " - "update this assertion to match the new emission shape." + # mysterious clang error. The bar-time builtins now lower to the engine's + # cached pine_() helpers (session_time.cpp) instead of an inline + # gmtime_r/localtime_r lambda — the per-call tzset churn caused a macOS + # notifyd storm (KI-35). The UTC/tz decode lives inside the helper now. + assert "pine_year(" in cpp and "pine_hour(" in cpp, ( + "year(time)/hour(time) etc. should lower to the engine pine_() " + "helpers; if you intentionally changed the emission, update this " + "assertion to match the new shape." ) compile_cpp(cpp, label="regression_year_function_form") diff --git a/tests/test_support_checker.py b/tests/test_support_checker.py index 1470c0f..b3a2310 100644 --- a/tests/test_support_checker.py +++ b/tests/test_support_checker.py @@ -349,11 +349,14 @@ def test_request_security_currency_kwarg_rejected(): _expect_error(src, "currency") -def test_request_security_ignore_invalid_symbol_kwarg_rejected(): +def test_request_security_ignore_invalid_symbol_kwarg_accepted(): + # ignore_invalid_symbol is a guaranteed no-op: codegen forces the current + # chart symbol (always valid), so the flag can never change the result. + # Accept it (codegen drops it) rather than reject a valid, harmless script. src = (PRELUDE + 'a = request.security(syminfo.tickerid, "60", close, ' 'ignore_invalid_symbol=true)\n') - _expect_error(src, "ignore_invalid_symbol") + assert _errors(src) == [] def test_request_security_positional_gaps_lookahead_passes(): @@ -626,9 +629,11 @@ def test_security_allowed_params_locked(): G2 sprint: backadjustment / settlement_as_close / adjustment added so scripts using those request.security kwargs compile instead of being hard-rejected. Codegen silently drops them (engine uses fixed unadjusted data). + ignore_invalid_symbol added: a guaranteed no-op on the forced chart symbol. """ assert SECURITY_ALLOWED_PARAMS == frozenset( {"symbol", "timeframe", "expression", "gaps", "lookahead", + "ignore_invalid_symbol", "backadjustment", "settlement_as_close", "adjustment"} )