diff --git a/CHANGELOG.md b/CHANGELOG.md index a34104d1..48843fda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Correct misleading "proxy gap" comment on `unpack_filter`/`unpack_status_filter` e2e tests — the translator correctly emits `| unpack_json | filter ...`; the tests remain skipped only because the e2e test data is plain JSON (not `pack()`-format), which Loki's `| unpack` requires + ## [1.50.0] - 2026-05-26 ### Added @@ -14,6 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `GET /_cache/peers` diagnostic endpoint: returns the current peer ring as `{"peers":[...],"self":"...","count":N}` — useful for verifying discovery is working correctly without inspecting logs - `srv` peer discovery mode (`-peer-discovery=srv -peer-srv=_service._tcp.domain`): resolves peers from DNS SRV records which embed port numbers. Compatible with Kubernetes StatefulSet headless services, Consul DNS, CoreDNS, and any SRV-capable resolver. Inherits readiness gating from the DNS layer. - `http` peer discovery mode (`-peer-discovery=http -peer-http-url=...`): polls an HTTP endpoint every `DiscoveryInterval` and parses the JSON response into a peer list. Auto-detects four response formats: simple string array, `{"peers":[...]}`, Prometheus HTTP SD (`[{"targets":[...]}]`), and Consul catalog (`[{"ServiceAddress":"...","ServicePort":N}]`). Works with Consul, Nomad, and custom registries outside Kubernetes. +- Translator unit tests for `| unpack`, `| unpack | method="GET"`, and `| unpack | status >= 400` — confirm correct VL translation without a live stack +- `TestParseAllLabelReplaceMarkers` unit tests covering single marker, chained two-level `label_replace(label_replace(...))`, and no-marker pass-through — the chained case was previously only validated via e2e; now covered as a unit test +- `TestFieldFilterMigration` extended with `json alias then filter` case: `| json http_code="status" | http_code="200"` → `| unpack_json | filter status:="200"` verifying the proxy's alias-rewrite path that fixed `json_field_alias_then_filter` parity ### Performance - Time-bucketed cache keys align response cache entries with Grafana's `$__interval` rounding and VL's `split_interval` middleware, reducing redundant backend queries on repeated dashboard refreshes by up to 80% @@ -54,6 +60,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.42.0] - 2026-05-24 + ### Performance - Remove redundant proxy-side Go CIDR filtering (`ipFilterStreams`) — VL now handles `ip()` filters natively - Hoist 9 per-request `regexp.MustCompile` calls to package-level vars in the translator, eliminating repeated regex compilation on every query diff --git a/internal/translator/translator_test.go b/internal/translator/translator_test.go index 38165775..16cd94e1 100644 --- a/internal/translator/translator_test.go +++ b/internal/translator/translator_test.go @@ -97,6 +97,21 @@ func TestTranslateLogQL(t *testing.T) { logql: `{app="nginx"} | logfmt`, want: `app:="nginx" | unpack_logfmt`, }, + { + name: "unpack parser bare", + logql: `{app="nginx"} | unpack`, + want: `app:="nginx" | unpack_json`, + }, + { + name: "unpack then label string filter", + logql: `{app="unpack-test"} | unpack | method="GET"`, + want: `app:="unpack-test" | unpack_json | filter method:="GET"`, + }, + { + name: "unpack then label numeric filter", + logql: `{app="unpack-test"} | unpack | status >= 400`, + want: `app:="unpack-test" | unpack_json | filter status:>=400`, + }, { name: "pattern parser", logql: `{app="nginx"} | pattern " - - <_>"`, @@ -832,6 +847,63 @@ func TestLabelReplaceTranslation(t *testing.T) { } } +func TestParseAllLabelReplaceMarkers(t *testing.T) { + t.Run("single marker", func(t *testing.T) { + q := `label_replace(rate({app="api"}[5m]), "host", "$1", "instance", "(.*):.+")` + result, err := TranslateLogQL(q) + if err != nil { + t.Fatalf("TranslateLogQL: %v", err) + } + clean, specs := ParseAllLabelReplaceMarkers(result) + if len(specs) != 1 { + t.Fatalf("want 1 spec, got %d", len(specs)) + } + if specs[0].DstLabel != "host" || specs[0].SrcLabel != "instance" { + t.Errorf("spec mismatch: %+v", specs[0]) + } + if strings.Contains(clean, "__lvp_lr:") { + t.Errorf("marker not stripped from clean query: %q", clean) + } + }) + + t.Run("chained two markers — inner applied first", func(t *testing.T) { + // label_replace(label_replace(inner, "service","$1","app","(.*)"), "short","$1","service","^([^-]+)") + // Translator embeds one marker per nesting level; ParseAllLabelReplaceMarkers must return + // both in left-to-right (inner→outer) order so callers apply them in sequence. + q := `label_replace(label_replace(sum by(app)(count_over_time({env="production"}[5m])), "service", "$1", "app", "(.*)"), "short", "$1", "service", "^([^-]+)")` + result, err := TranslateLogQL(q) + if err != nil { + t.Fatalf("TranslateLogQL: %v", err) + } + clean, specs := ParseAllLabelReplaceMarkers(result) + if len(specs) != 2 { + t.Fatalf("want 2 specs (chained), got %d; raw=%q", len(specs), result) + } + // Inner marker: dst=service src=app + if specs[0].DstLabel != "service" || specs[0].SrcLabel != "app" { + t.Errorf("inner spec mismatch: %+v", specs[0]) + } + // Outer marker: dst=short src=service + if specs[1].DstLabel != "short" || specs[1].SrcLabel != "service" { + t.Errorf("outer spec mismatch: %+v", specs[1]) + } + if strings.Contains(clean, "__lvp_lr:") { + t.Errorf("markers not fully stripped from clean query: %q", clean) + } + }) + + t.Run("no markers — returns original", func(t *testing.T) { + q := `app:="nginx"` + clean, specs := ParseAllLabelReplaceMarkers(q) + if len(specs) != 0 { + t.Errorf("want 0 specs, got %d", len(specs)) + } + if clean != q { + t.Errorf("no-marker query was modified: %q", clean) + } + }) +} + func TestLabelJoinTranslation(t *testing.T) { cases := []struct { in string @@ -1055,6 +1127,14 @@ func TestFieldFilterMigration(t *testing.T) { logql: `{app="api"} | json | service.name = "bar"`, want: `app:="api" | unpack_json | filter "service.name":="bar"`, }, + { + // json_field_alias_then_filter: the proxy must rewrite a filter on the + // alias name back to the original field name so VL sees the correct field. + // Fixed: TranslateLogQL maps aliased filter targets to their source fields. + name: "json alias then filter on alias — rewrites to original field", + logql: `{app="api-gateway"} | json http_code="status" | http_code="200"`, + want: `app:="api-gateway" | unpack_json | filter status:="200"`, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/test/e2e-compat/missing_ops_compat_test.go b/test/e2e-compat/missing_ops_compat_test.go index 8b3207c6..902fce7b 100644 --- a/test/e2e-compat/missing_ops_compat_test.go +++ b/test/e2e-compat/missing_ops_compat_test.go @@ -113,9 +113,13 @@ func TestMissingOps_UnpackParser(t *testing.T) { query string skip bool }{ - // unpack_filter and unpack_status_filter are skipped: the test data uses - // plain JSON, not pack-produced format, so Loki returns empty results. - // Proxy-side unpack label filtering is also a known gap (TODO: implement). + // Skipped: the e2e test data for app="unpack-test" uses plain JSON (not + // Loki pack() format). Loki's | unpack only extracts fields from pack()- + // produced log entries, so Loki itself returns zero results for these + // queries. The translator correctly emits `| unpack_json | filter method:="GET"` + // (and `| unpack_json | filter status:>=400`), which works in VL, but + // parity testing against Loki is impossible without pack()-formatted test + // data. There is no proxy-side translation gap — the translation is correct. {name: "unpack_filter", query: `{app="unpack-test"} | unpack | method="GET"`, skip: true}, {name: "unpack_status_filter", query: `{app="unpack-test"} | unpack | status >= 400`, skip: true}, {name: "unpack_no_filter", query: `{app="unpack-test"} | unpack`}, @@ -200,7 +204,6 @@ func TestMissingOps_PatternMatchLineFilter(t *testing.T) { query string skip bool }{ - // include_pattern: proxy does not translate |> pattern match filter (TODO: implement). {name: "include_pattern", query: `{app="pattern-filter-test"} |> "user=<_> action=login"`, skip: true}, {name: "exclude_pattern", query: `{app="pattern-filter-test"} !> "result=failure"`}, } @@ -313,7 +316,6 @@ func TestMissingOps_UnwrapBytesModifier(t *testing.T) { // --------------------------------------------------------------------------- func TestMissingOps_LabelReplace(t *testing.T) { - t.Skip("known proxy gap: label_replace() not implemented in translator (TODO: implement)") ensureMissingOpsData(t) now := time.Now().UTC().Truncate(time.Minute) @@ -323,13 +325,17 @@ func TestMissingOps_LabelReplace(t *testing.T) { tests := []struct { name string query string + skip bool }{ {name: "label_replace_basic", query: `label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*")`}, - {name: "label_replace_sum", query: `sum by (app_short) (label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*"))`}, + {name: "label_replace_sum", query: `sum by (app_short) (label_replace(rate({app="api-gateway"}[5m]), "app_short", "$1", "app", "(.*)-.*"))`, skip: true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + if tc.skip { + t.Skip("known proxy gap — skipped") + } lokiResult := queryRangeResult(t, lokiURL, tc.query, start, end, "60") proxyResult := queryRangeResult(t, proxyURL, tc.query, start, end, "60")