diff --git a/parser/internal/BUILD b/parser/internal/BUILD index af815588e..1263a3f47 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -13,12 +13,29 @@ # limitations under the License. load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") load("//bazel:antlr.bzl", "antlr_cc_library") package(default_visibility = ["//visibility:public"]) licenses(["notice"]) +cc_library( + name = "ast_factory_interface", + hdrs = ["ast_factory_interface.h"], +) + +cc_library( + name = "ast_factory", + hdrs = ["ast_factory.h"], + deps = [ + ":ast_factory_interface", + "//common:constant", + "//common:expr", + "@com_google_absl//absl/strings:string_view", + ], +) + cc_library( name = "options", hdrs = ["options.h"], @@ -29,3 +46,123 @@ antlr_cc_library( src = "Cel.g4", package = "cel_parser_internal", ) + +cc_library( + name = "lexer", + srcs = ["lexer.cc"], + hdrs = ["lexer.h"], + deps = [ + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_library( + name = "pratt_parser_worker", + srcs = ["pratt_parser_worker.cc"], + hdrs = ["pratt_parser_worker.h"], + deps = [ + ":ast_factory_interface", + ":lexer", + "//common:operators", + "//common:source", + "//internal:lexis", + "//internal:strings", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_library( + name = "pratt_parser", + srcs = ["pratt_parser.cc"], + hdrs = ["pratt_parser.h"], + deps = [ + ":ast_factory", + ":pratt_parser_worker", + "//common:ast", + "//common:expr", + "//common:source", + "//internal:status_macros", + "//parser:macro", + "//parser:macro_registry", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + ], +) + +cc_test( + name = "ast_factory_test", + srcs = ["ast_factory_test.cc"], + deps = [ + ":ast_factory", + ":ast_factory_interface", + "//common:expr", + "//internal:testing", + ], +) + +cc_test( + name = "lexer_test", + srcs = ["lexer_test.cc"], + deps = [ + ":lexer", + "//internal:testing", + ], +) + +cc_test( + name = "pratt_parser_test", + srcs = ["pratt_parser_test.cc"], + deps = [ + ":pratt_parser", + "//common:ast", + "//common:constant", + "//common:expr", + "//common:source", + "//internal:status_macros", + "//internal:testing", + "//parser:options", + "//parser:parser_interface", + "//testutil:expr_printer", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_test( + name = "lexer_fuzzer", + size = "large", + srcs = ["lexer_fuzzer.cc"], + deps = [ + ":lexer", + "//internal:testing", + "//testing/fuzzing:fuzztest", + ], +) diff --git a/parser/internal/ast_factory.h b/parser/internal/ast_factory.h new file mode 100644 index 000000000..96a02bcf8 --- /dev/null +++ b/parser/internal/ast_factory.h @@ -0,0 +1,236 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ + +#include +#include +#include +#include +#include + +#include "common/constant.h" +#include "common/expr.h" +#include "parser/internal/ast_factory_interface.h" + +namespace cel::parser_internal { + +// Explicit specialization of `AstFactoryInterface` for `cel::Expr` AST nodes. +template <> +class AstFactoryInterface { + public: + AstFactoryInterface() = default; + + AstFactoryInterface(const AstFactoryInterface&) = delete; + AstFactoryInterface(AstFactoryInterface&&) = delete; + AstFactoryInterface& operator=(const AstFactoryInterface&) = delete; + AstFactoryInterface& operator=(AstFactoryInterface&&) = delete; + + ~AstFactoryInterface() = default; + + // Node inspection and encapsulation API + int64_t GetId(const cel::Expr& expr) const { return expr.id(); } + + bool IsEmpty(const cel::Expr& expr) const { return expr.id() == 0; } + + bool IsConst(const cel::Expr& expr) const { return expr.has_const_expr(); } + + bool IsIdent(const cel::Expr& expr) const { return expr.has_ident_expr(); } + + std::string_view GetIdentName(const cel::Expr& expr) const { + return expr.has_ident_expr() ? std::string_view(expr.ident_expr().name()) + : std::string_view(); + } + + bool IsSelect(const cel::Expr& expr) const { return expr.has_select_expr(); } + + bool IsPresenceTest(const cel::Expr& expr) const { + return expr.has_select_expr() && expr.select_expr().test_only(); + } + + const cel::Expr* GetSelectOperand(const cel::Expr& expr) const { + return expr.has_select_expr() ? &expr.select_expr().operand() : nullptr; + } + + std::string_view GetSelectField(const cel::Expr& expr) const { + return expr.has_select_expr() ? std::string_view(expr.select_expr().field()) + : std::string_view(); + } + + // Node creation API + cel::Expr NewUnspecified(int64_t id) { + cel::Expr expr; + expr.set_id(id); + return expr; + } + + cel::Expr NewNullConst(int64_t id) { + cel::Constant constant; + constant.set_null_value(); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewBoolConst(int64_t id, bool value) { + cel::Constant constant; + constant.set_bool_value(value); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewIntConst(int64_t id, int64_t value) { + cel::Constant constant; + constant.set_int_value(value); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewUintConst(int64_t id, uint64_t value) { + cel::Constant constant; + constant.set_uint_value(value); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewDoubleConst(int64_t id, double value) { + cel::Constant constant; + constant.set_double_value(value); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewBytesConst(int64_t id, std::string value) { + cel::Constant constant; + constant.set_bytes_value(std::move(value)); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewStringConst(int64_t id, std::string value) { + cel::Constant constant; + constant.set_string_value(std::move(value)); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewIdent(int64_t id, std::string name) { + cel::Expr expr; + expr.set_id(id); + cel::IdentExpr& ident_expr = expr.mutable_ident_expr(); + ident_expr.set_name(std::move(name)); + return expr; + } + + cel::Expr NewSelect(int64_t id, cel::Expr operand, std::string field) { + cel::Expr expr; + expr.set_id(id); + cel::SelectExpr& select_expr = expr.mutable_select_expr(); + select_expr.set_operand(std::move(operand)); + select_expr.set_field(std::move(field)); + select_expr.set_test_only(false); + return expr; + } + + cel::Expr NewPresenceTest(int64_t id, cel::Expr operand, std::string field) { + cel::Expr expr; + expr.set_id(id); + cel::SelectExpr& select_expr = expr.mutable_select_expr(); + select_expr.set_operand(std::move(operand)); + select_expr.set_field(std::move(field)); + select_expr.set_test_only(true); + return expr; + } + + cel::Expr NewCall(int64_t id, std::string function, + std::vector args) { + cel::Expr expr; + expr.set_id(id); + cel::CallExpr& call_expr = expr.mutable_call_expr(); + call_expr.set_function(std::move(function)); + call_expr.set_args(std::move(args)); + return expr; + } + + cel::Expr NewMemberCall(int64_t id, std::string function, cel::Expr target, + std::vector args) { + cel::Expr expr; + expr.set_id(id); + cel::CallExpr& call_expr = expr.mutable_call_expr(); + call_expr.set_function(std::move(function)); + call_expr.set_target(std::move(target)); + call_expr.set_args(std::move(args)); + return expr; + } + + cel::Expr NewList(int64_t id) { + cel::Expr expr; + expr.set_id(id); + expr.mutable_list_expr(); + return expr; + } + + void AddListElement(cel::Expr& list_expr, cel::Expr element, + bool optional = false) { + cel::ListExpr& list_val = list_expr.mutable_list_expr(); + cel::ListExprElement expr_element; + expr_element.set_expr(std::move(element)); + expr_element.set_optional(optional); + list_val.mutable_elements().push_back(std::move(expr_element)); + } + + cel::Expr NewStruct(int64_t id, std::string name) { + cel::Expr expr; + expr.set_id(id); + cel::StructExpr& struct_expr = expr.mutable_struct_expr(); + struct_expr.set_name(std::move(name)); + return expr; + } + + void AddStructField(cel::Expr& struct_expr, int64_t id, std::string name, + cel::Expr value, bool optional = false) { + cel::StructExpr& struct_val = struct_expr.mutable_struct_expr(); + cel::StructExprField field; + field.set_id(id); + field.set_name(std::move(name)); + field.set_value(std::move(value)); + field.set_optional(optional); + struct_val.mutable_fields().push_back(std::move(field)); + } + + cel::Expr NewMap(int64_t id) { + cel::Expr expr; + expr.set_id(id); + expr.mutable_map_expr(); + return expr; + } + + void AddMapEntry(cel::Expr& map_expr, int64_t id, cel::Expr key, + cel::Expr value, bool optional = false) { + cel::MapExpr& map_val = map_expr.mutable_map_expr(); + cel::MapExprEntry entry; + entry.set_id(id); + entry.set_key(std::move(key)); + entry.set_value(std::move(value)); + entry.set_optional(optional); + map_val.mutable_entries().push_back(std::move(entry)); + } + + private: + cel::Expr NewConst(int64_t id, cel::Constant value) { + cel::Expr expr; + expr.set_id(id); + expr.mutable_const_expr() = std::move(value); + return expr; + } +}; + +using AstFactory = AstFactoryInterface; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ diff --git a/parser/internal/ast_factory_interface.h b/parser/internal/ast_factory_interface.h new file mode 100644 index 000000000..7052c0f87 --- /dev/null +++ b/parser/internal/ast_factory_interface.h @@ -0,0 +1,86 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ + +#include +#include +#include +#include + +namespace cel::parser_internal { + +// Interface for decoupling parser logic from the underlying AST node +// data structures. +// +// By parameterizing the parser and factory on `ExprNode`, alternative AST node +// representations (such as `cel::Expr`) can be constructed without modifying +// parser rules. +// +// To implement AST construction using an alternative AST structure: +// 1. Define or specify your custom node type `MyNode`. +// 2. Implement a concrete factory specialization `AstFactoryInterface` +// that provides inspection (`GetId`, `IsSelect`, etc.) and creation +// (`NewCall`, `AddListElement`, etc.) operations for `MyNode`. +// 3. Instantiate the parser worker with your node type: +// `PrattParserWorker`. +template +class AstFactoryInterface { + public: + AstFactoryInterface() = default; + AstFactoryInterface(const AstFactoryInterface&) = delete; + AstFactoryInterface(AstFactoryInterface&&) = delete; + AstFactoryInterface& operator=(const AstFactoryInterface&) = delete; + AstFactoryInterface& operator=(AstFactoryInterface&&) = delete; + + int64_t GetId(const ExprNode& expr) const; + bool IsEmpty(const ExprNode& expr) const; + bool IsConst(const ExprNode& expr) const; + bool IsIdent(const ExprNode& expr) const; + std::string_view GetIdentName(const ExprNode& expr) const; + bool IsSelect(const ExprNode& expr) const; + bool IsPresenceTest(const ExprNode& expr) const; + const ExprNode* GetSelectOperand(const ExprNode& expr) const; + std::string_view GetSelectField(const ExprNode& expr) const; + + ExprNode NewUnspecified(int64_t id); + ExprNode NewNullConst(int64_t id); + ExprNode NewBoolConst(int64_t id, bool value); + ExprNode NewIntConst(int64_t id, int64_t value); + ExprNode NewUintConst(int64_t id, uint64_t value); + ExprNode NewDoubleConst(int64_t id, double value); + ExprNode NewBytesConst(int64_t id, std::string value); + ExprNode NewStringConst(int64_t id, std::string value); + ExprNode NewIdent(int64_t id, std::string name); + ExprNode NewSelect(int64_t id, ExprNode operand, std::string field); + ExprNode NewPresenceTest(int64_t id, ExprNode operand, std::string field); + ExprNode NewCall(int64_t id, std::string function, + std::vector args); + ExprNode NewMemberCall(int64_t id, std::string function, ExprNode target, + std::vector args); + ExprNode NewList(int64_t id); + void AddListElement(ExprNode& list_expr, ExprNode element, + bool optional = false); + ExprNode NewStruct(int64_t id, std::string name); + void AddStructField(ExprNode& struct_expr, int64_t id, std::string name, + ExprNode value, bool optional = false); + ExprNode NewMap(int64_t id); + void AddMapEntry(ExprNode& map_expr, int64_t id, ExprNode key, ExprNode value, + bool optional = false); +}; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ diff --git a/parser/internal/ast_factory_test.cc b/parser/internal/ast_factory_test.cc new file mode 100644 index 000000000..706f13e88 --- /dev/null +++ b/parser/internal/ast_factory_test.cc @@ -0,0 +1,197 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/ast_factory.h" + +#include +#include +#include + +#include "common/expr.h" +#include "internal/testing.h" + +namespace cel::parser_internal { +namespace { + +TEST(AstFactoryInterfaceTest, AstFactoryUnspecified) { + AstFactory factory; + cel::Expr expr = factory.NewUnspecified(1); + EXPECT_EQ(factory.GetId(expr), 1); + EXPECT_FALSE(factory.IsEmpty(expr)); + EXPECT_FALSE(factory.IsConst(expr)); + EXPECT_FALSE(factory.IsIdent(expr)); + EXPECT_FALSE(factory.IsSelect(expr)); + + cel::Expr empty_expr = factory.NewUnspecified(0); + EXPECT_TRUE(factory.IsEmpty(empty_expr)); +} + +TEST(AstFactoryInterfaceTest, AstFactoryConstNodes) { + AstFactory factory; + + cel::Expr null_expr = factory.NewNullConst(10); + EXPECT_EQ(factory.GetId(null_expr), 10); + EXPECT_TRUE(factory.IsConst(null_expr)); + ASSERT_TRUE(null_expr.has_const_expr()); + EXPECT_TRUE(null_expr.const_expr().has_null_value()); + + cel::Expr bool_expr = factory.NewBoolConst(11, true); + EXPECT_EQ(factory.GetId(bool_expr), 11); + EXPECT_TRUE(factory.IsConst(bool_expr)); + ASSERT_TRUE(bool_expr.has_const_expr()); + EXPECT_TRUE(bool_expr.const_expr().bool_value()); + + cel::Expr int_expr = factory.NewIntConst(12, -42); + EXPECT_EQ(factory.GetId(int_expr), 12); + EXPECT_TRUE(factory.IsConst(int_expr)); + ASSERT_TRUE(int_expr.has_const_expr()); + EXPECT_EQ(int_expr.const_expr().int_value(), -42); + + cel::Expr uint_expr = factory.NewUintConst(13, 100u); + EXPECT_EQ(factory.GetId(uint_expr), 13); + EXPECT_TRUE(factory.IsConst(uint_expr)); + ASSERT_TRUE(uint_expr.has_const_expr()); + EXPECT_EQ(uint_expr.const_expr().uint_value(), 100u); + + cel::Expr double_expr = factory.NewDoubleConst(14, 3.14159); + EXPECT_EQ(factory.GetId(double_expr), 14); + EXPECT_TRUE(factory.IsConst(double_expr)); + ASSERT_TRUE(double_expr.has_const_expr()); + EXPECT_DOUBLE_EQ(double_expr.const_expr().double_value(), 3.14159); + + cel::Expr bytes_expr = factory.NewBytesConst(15, "bytes_val"); + EXPECT_EQ(factory.GetId(bytes_expr), 15); + EXPECT_TRUE(factory.IsConst(bytes_expr)); + ASSERT_TRUE(bytes_expr.has_const_expr()); + EXPECT_EQ(bytes_expr.const_expr().bytes_value(), "bytes_val"); + + cel::Expr string_expr = factory.NewStringConst(16, "string_val"); + EXPECT_EQ(factory.GetId(string_expr), 16); + EXPECT_TRUE(factory.IsConst(string_expr)); + ASSERT_TRUE(string_expr.has_const_expr()); + EXPECT_EQ(string_expr.const_expr().string_value(), "string_val"); +} + +TEST(AstFactoryInterfaceTest, AstFactoryIdentAndSelect) { + AstFactory factory; + + cel::Expr ident_expr = factory.NewIdent(20, "foo"); + EXPECT_EQ(factory.GetId(ident_expr), 20); + EXPECT_TRUE(factory.IsIdent(ident_expr)); + EXPECT_EQ(factory.GetIdentName(ident_expr), "foo"); + + cel::Expr select_expr = + factory.NewSelect(21, factory.NewIdent(20, "foo"), "bar"); + EXPECT_EQ(factory.GetId(select_expr), 21); + EXPECT_TRUE(factory.IsSelect(select_expr)); + EXPECT_FALSE(factory.IsPresenceTest(select_expr)); + EXPECT_EQ(factory.GetSelectField(select_expr), "bar"); + ASSERT_NE(factory.GetSelectOperand(select_expr), nullptr); + EXPECT_EQ(factory.GetIdentName(*factory.GetSelectOperand(select_expr)), + "foo"); + + cel::Expr presence_expr = + factory.NewPresenceTest(22, factory.NewIdent(20, "foo"), "bar"); + EXPECT_EQ(factory.GetId(presence_expr), 22); + EXPECT_TRUE(factory.IsSelect(presence_expr)); + EXPECT_TRUE(factory.IsPresenceTest(presence_expr)); + EXPECT_EQ(factory.GetSelectField(presence_expr), "bar"); +} + +TEST(AstFactoryInterfaceTest, AstFactoryCalls) { + AstFactory factory; + + std::vector call_args; + call_args.push_back(factory.NewIntConst(30, 1)); + call_args.push_back(factory.NewIntConst(31, 2)); + cel::Expr call_expr = factory.NewCall(32, "_+_", std::move(call_args)); + EXPECT_EQ(factory.GetId(call_expr), 32); + ASSERT_TRUE(call_expr.has_call_expr()); + EXPECT_EQ(call_expr.call_expr().function(), "_+_"); + EXPECT_FALSE(call_expr.call_expr().has_target()); + EXPECT_EQ(call_expr.call_expr().args().size(), 2); + + std::vector member_args; + member_args.push_back(factory.NewStringConst(33, "suffix")); + cel::Expr member_call_expr = factory.NewMemberCall( + 34, "endsWith", factory.NewIdent(35, "str_var"), std::move(member_args)); + EXPECT_EQ(factory.GetId(member_call_expr), 34); + ASSERT_TRUE(member_call_expr.has_call_expr()); + EXPECT_EQ(member_call_expr.call_expr().function(), "endsWith"); + EXPECT_TRUE(member_call_expr.call_expr().has_target()); + EXPECT_EQ(member_call_expr.call_expr().target().ident_expr().name(), + "str_var"); + EXPECT_EQ(member_call_expr.call_expr().args().size(), 1); +} + +TEST(AstFactoryInterfaceTest, AstFactoryList) { + AstFactory factory; + + cel::Expr list_expr = factory.NewList(42); + factory.AddListElement(list_expr, factory.NewIntConst(40, 1), false); + factory.AddListElement(list_expr, factory.NewIntConst(41, 2), true); + EXPECT_EQ(factory.GetId(list_expr), 42); + ASSERT_TRUE(list_expr.has_list_expr()); + ASSERT_EQ(list_expr.list_expr().elements().size(), 2); + EXPECT_FALSE(list_expr.list_expr().elements()[0].optional()); + EXPECT_EQ(list_expr.list_expr().elements()[0].expr().const_expr().int_value(), + 1); + EXPECT_TRUE(list_expr.list_expr().elements()[1].optional()); + EXPECT_EQ(list_expr.list_expr().elements()[1].expr().const_expr().int_value(), + 2); +} + +TEST(AstFactoryInterfaceTest, AstFactoryStruct) { + AstFactory factory; + + cel::Expr struct_expr = factory.NewStruct(54, "MyMessage"); + factory.AddStructField(struct_expr, 50, "field1", + factory.NewIntConst(51, 100), false); + factory.AddStructField(struct_expr, 52, "field2", + factory.NewIntConst(53, 200), true); + EXPECT_EQ(factory.GetId(struct_expr), 54); + ASSERT_TRUE(struct_expr.has_struct_expr()); + EXPECT_EQ(struct_expr.struct_expr().name(), "MyMessage"); + ASSERT_EQ(struct_expr.struct_expr().fields().size(), 2); + EXPECT_EQ(struct_expr.struct_expr().fields()[0].id(), 50); + EXPECT_EQ(struct_expr.struct_expr().fields()[0].name(), "field1"); + EXPECT_FALSE(struct_expr.struct_expr().fields()[0].optional()); + EXPECT_EQ(struct_expr.struct_expr().fields()[1].id(), 52); + EXPECT_EQ(struct_expr.struct_expr().fields()[1].name(), "field2"); + EXPECT_TRUE(struct_expr.struct_expr().fields()[1].optional()); +} + +TEST(AstFactoryInterfaceTest, AstFactoryMap) { + AstFactory factory; + + cel::Expr map_expr = factory.NewMap(66); + factory.AddMapEntry(map_expr, 60, factory.NewStringConst(61, "key1"), + factory.NewIntConst(62, 10), false); + factory.AddMapEntry(map_expr, 63, factory.NewStringConst(64, "key2"), + factory.NewIntConst(65, 20), true); + EXPECT_EQ(factory.GetId(map_expr), 66); + ASSERT_TRUE(map_expr.has_map_expr()); + ASSERT_EQ(map_expr.map_expr().entries().size(), 2); + EXPECT_EQ(map_expr.map_expr().entries()[0].id(), 60); + EXPECT_EQ(map_expr.map_expr().entries()[0].key().const_expr().string_value(), + "key1"); + EXPECT_EQ(map_expr.map_expr().entries()[0].value().const_expr().int_value(), + 10); + EXPECT_FALSE(map_expr.map_expr().entries()[0].optional()); + EXPECT_EQ(map_expr.map_expr().entries()[1].id(), 63); + EXPECT_TRUE(map_expr.map_expr().entries()[1].optional()); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc new file mode 100644 index 000000000..7b8da71d1 --- /dev/null +++ b/parser/internal/lexer.cc @@ -0,0 +1,694 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" +#include "absl/container/flat_hash_map.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace cel::parser_internal { + +namespace { + +[[nodiscard]] bool IsIdentTrailing(unsigned char c) { + return absl::ascii_isdigit(c) || absl::ascii_isalpha(c) || c == '_'; +} + +[[nodiscard]] bool IsPlusOrMinus(unsigned char c) { + return c == '+' || c == '-'; +} + +[[nodiscard]] const absl::flat_hash_map& +Keywords() { + static const absl::NoDestructor< + absl::flat_hash_map> + kKeywords({ + {"false", TokenType::kFalse}, + {"true", TokenType::kTrue}, + {"null", TokenType::kNull}, + {"in", TokenType::kIn}, + {"as", TokenType::kReservedWord}, + {"break", TokenType::kReservedWord}, + {"const", TokenType::kReservedWord}, + {"continue", TokenType::kReservedWord}, + {"else", TokenType::kReservedWord}, + {"for", TokenType::kReservedWord}, + {"function", TokenType::kReservedWord}, + {"if", TokenType::kReservedWord}, + {"import", TokenType::kReservedWord}, + {"let", TokenType::kReservedWord}, + {"loop", TokenType::kReservedWord}, + {"package", TokenType::kReservedWord}, + {"namespace", TokenType::kReservedWord}, + {"return", TokenType::kReservedWord}, + {"var", TokenType::kReservedWord}, + {"void", TokenType::kReservedWord}, + {"while", TokenType::kReservedWord}, + }); + return *kKeywords; +} + +} // namespace + +std::string_view TokenTypeToString(TokenType type) { + switch (type) { + case TokenType::kError: + return "error"; + case TokenType::kEnd: + return "end"; + case TokenType::kWhitespace: + return "whitespace"; + case TokenType::kComment: + return "comment"; + case TokenType::kNull: + return "null"; + case TokenType::kFalse: + return "false"; + case TokenType::kTrue: + return "true"; + case TokenType::kIn: + return "in"; + case TokenType::kReservedWord: + return "reserved_word"; + case TokenType::kInt: + return "int"; + case TokenType::kUint: + return "uint"; + case TokenType::kFloat: + return "float"; + case TokenType::kString: + return "string"; + case TokenType::kBytes: + return "bytes"; + case TokenType::kIdent: + return "ident"; + case TokenType::kLeftBracket: + return "["; + case TokenType::kRightBracket: + return "]"; + case TokenType::kLeftBrace: + return "{"; + case TokenType::kRightBrace: + return "}"; + case TokenType::kLeftParen: + return "("; + case TokenType::kRightParen: + return ")"; + case TokenType::kDot: + return "."; + case TokenType::kComma: + return ","; + case TokenType::kMinus: + return "-"; + case TokenType::kPlus: + return "+"; + case TokenType::kAsterisk: + return "*"; + case TokenType::kSlash: + return "/"; + case TokenType::kPercent: + return "%"; + case TokenType::kQuestion: + return "?"; + case TokenType::kColon: + return ":"; + case TokenType::kExclamation: + return "!"; + case TokenType::kEqual: + return "="; + case TokenType::kEqualEqual: + return "=="; + case TokenType::kExclamationEqual: + return "!="; + case TokenType::kLess: + return "<"; + case TokenType::kLessEqual: + return "<="; + case TokenType::kGreater: + return ">"; + case TokenType::kGreaterEqual: + return ">="; + case TokenType::kLogicalAnd: + return "&&"; + case TokenType::kLogicalOr: + return "||"; + default: + return ""; + } +} + +Token Lexer::Lex() { + if (ABSL_PREDICT_FALSE(at_error_)) { + return MakeToken(TokenType::kError, error_.start, error_.end); + } + int32_t start = GetPosition(); + if (ABSL_PREDICT_FALSE(text_ == text_end_)) { + at_end_ = true; + done_ = true; + return MakeToken(TokenType::kEnd, start, start); + } + char c = *text_; + switch (c) { + case '\v': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\n': + ABSL_FALLTHROUGH_INTENDED; + case ' ': { + ConsumeWhitespace(); + return MakeToken(TokenType::kWhitespace, start, GetPosition()); + } + case '.': { + // Check if this looks like a double literal and break if it is, we handle + // that after this switch. + if (text_ + 1 < text_end_ && absl::ascii_isdigit(text_[1])) { + break; + } + Advance(1); + return MakeToken(TokenType::kDot, start, GetPosition()); + } + case ',': { + Advance(1); + return MakeToken(TokenType::kComma, start, GetPosition()); + } + case '!': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kExclamationEqual, start, GetPosition()); + } + return MakeToken(TokenType::kExclamation, start, GetPosition()); + } + case '?': { + Advance(1); + return MakeToken(TokenType::kQuestion, start, GetPosition()); + } + case '(': { + Advance(1); + return MakeToken(TokenType::kLeftParen, start, GetPosition()); + } + case ')': { + Advance(1); + return MakeToken(TokenType::kRightParen, start, GetPosition()); + } + case '{': { + Advance(1); + return MakeToken(TokenType::kLeftBrace, start, GetPosition()); + } + case '}': { + Advance(1); + return MakeToken(TokenType::kRightBrace, start, GetPosition()); + } + case '[': { + Advance(1); + return MakeToken(TokenType::kLeftBracket, start, GetPosition()); + } + case ']': { + Advance(1); + return MakeToken(TokenType::kRightBracket, start, GetPosition()); + } + case '=': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kEqualEqual, start, GetPosition()); + } + return MakeToken(TokenType::kEqual, start, GetPosition()); + } + case '<': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kLessEqual, start, GetPosition()); + } + return MakeToken(TokenType::kLess, start, GetPosition()); + } + case '>': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kGreaterEqual, start, GetPosition()); + } + return MakeToken(TokenType::kGreater, start, GetPosition()); + } + case ':': { + Advance(1); + return MakeToken(TokenType::kColon, start, GetPosition()); + } + case '%': { + Advance(1); + return MakeToken(TokenType::kPercent, start, GetPosition()); + } + case '+': { + Advance(1); + return MakeToken(TokenType::kPlus, start, GetPosition()); + } + case '-': { + Advance(1); + return MakeToken(TokenType::kMinus, start, GetPosition()); + } + case '*': { + Advance(1); + return MakeToken(TokenType::kAsterisk, start, GetPosition()); + } + case '/': { + Advance(1); + if (Consume('/')) { + ConsumeLine(); + return MakeToken(TokenType::kComment, start, GetPosition()); + } + return MakeToken(TokenType::kSlash, start, GetPosition()); + } + case '&': { + Advance(1); + if (Consume('&')) { + return MakeToken(TokenType::kLogicalAnd, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '&', expected '&&'"); + } + case '|': { + Advance(1); + if (Consume('|')) { + return MakeToken(TokenType::kLogicalOr, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '|', expected '||'"); + } + case '`': { + // Report the entire match including the backticks because the parser + // forbids quoted identifiers in certain places and needs to be able to + // detect them. + Advance(1); + if (!ConsumeUntilAfter('`')) { + return SetError(start, GetPosition(), "unterminated quoted identifier"); + } + return MakeToken(TokenType::kIdent, start, GetPosition()); + } + case '\'': { + Advance(1); + if (ConsumeString("''")) { + if (!ConsumeUntilAfterString("'''")) { + return SetError(start, GetPosition(), "unterminated string literal"); + } + return MakeToken(TokenType::kString, start, GetPosition()); + } + if (!ConsumeUntilAfterUnescaped('\'')) { + return SetError(start, GetPosition(), "unterminated string literal"); + } + return MakeToken(TokenType::kString, start, GetPosition()); + } + case '"': { + Advance(1); + if (ConsumeString("\"\"")) { + if (!ConsumeUntilAfterString("\"\"\"")) { + return SetError(start, GetPosition(), "unterminated string literal"); + } + return MakeToken(TokenType::kString, start, GetPosition()); + } + if (!ConsumeUntilAfterUnescaped('"')) { + return SetError(start, GetPosition(), "unterminated string literal"); + } + return MakeToken(TokenType::kString, start, GetPosition()); + } + default: + break; + } + if (c == 'r' || c == 'R' || c == 'b' || c == 'B') { + bool is_bytes = (c == 'b' || c == 'B'); + size_t lookahead = 1; + if (text_ + 1 < text_end_) { + char c2 = text_[1]; + if ((is_bytes && (c2 == 'r' || c2 == 'R')) || + (!is_bytes && (c2 == 'b' || c2 == 'B'))) { + is_bytes = true; + lookahead = 2; + } + } + if (text_ + lookahead < text_end_) { + char quote = text_[lookahead]; + if (quote == '"' || quote == '\'') { + Advance(lookahead + 1); + std::string tripe_quote(3, quote); + if (ConsumeString(std::string_view(tripe_quote.data(), 2))) { + if (!ConsumeUntilAfterString(tripe_quote)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, + start, GetPosition()); + } + if (!ConsumeUntilAfterUnescaped(quote)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, + start, GetPosition()); + } + } + } + if (c == '.' || absl::ascii_isdigit(c)) { + bool floating_point = false; + if (c == '.') { + floating_point = true; + Advance(1); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after decimal separator"); + } + } else { + Advance(1); + if (c == '0') { + if (ConsumeIgnoreCase('x')) { + if (!ConsumeHexDigits()) { + return SetError( + start, GetPosition(), + "integral literal missing digits after hexadecimal separator"); + } + auto token_type = ConsumeIntegralSuffix(); + if (ConsumeIf(IsIdentTrailing)) { + return SetError( + start, GetPosition(), + absl::StrCat(TokenTypeToString(token_type), + " literal has unexpected trailing characters")); + } + return MakeToken(token_type, start, GetPosition()); + } + } + static_cast(ConsumeDigits()); + if (text_ < text_end_ && *text_ == '.' && text_ + 1 < text_end_ && + absl::ascii_isdigit(text_[1])) { + floating_point = true; + Advance(1); + static_cast(ConsumeDigits()); + } + } + if (ConsumeIgnoreCase('e')) { + floating_point = true; + static_cast(ConsumeIf(IsPlusOrMinus)); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after exponent separator"); + } + } + auto token_type = + floating_point ? TokenType::kFloat : ConsumeIntegralSuffix(); + if (ConsumeIf(IsIdentTrailing)) { + return SetError( + start, GetPosition(), + absl::StrCat(TokenTypeToString(token_type), + " literal has unexpected trailing characters")); + } + return MakeToken(token_type, start, GetPosition()); + } + if (c == '_' || absl::ascii_isalpha(c)) { + const char* text = text_; + ConsumeIdentTrailing(); + int32_t end = GetPosition(); + std::string_view word(text, static_cast(end - start)); + const auto& keywords = Keywords(); + if (auto it = keywords.find(word); it != keywords.end()) { + return MakeToken(it->second, start, end); + } + return MakeToken(TokenType::kIdent, start, end); + } + Advance(1); + return SetError(start, GetPosition(), "unexpected character"); +} + +bool Lexer::ConsumeUntilAfter(char c) { + ABSL_DCHECK_NE(c, '\n'); + auto pos = GetRemainingText().find(c); + if (pos == std::string_view::npos) { + AdvanceProcessingNewLines(text_end_); + return false; + } + AdvanceProcessingNewLines(pos + 1); + return true; +} + +bool Lexer::ConsumeUntilAfterString(std::string_view s) { + ABSL_DCHECK(!absl::StrContains(s, '\n')); + auto pos = GetRemainingText().find(s); + if (pos == std::string_view::npos) { + AdvanceProcessingNewLines(text_end_); + return false; + } + AdvanceProcessingNewLines(pos + s.size()); + return true; +} + +bool Lexer::ConsumeUntilAfterUnescaped(char c) { + ABSL_DCHECK_NE(c, '\n'); + ABSL_DCHECK_NE(c, '\\'); + const char* text = text_; + bool escaped = false; + while (text != text_end_) { + std::string_view chunk = + std::string_view(text, static_cast(text_end_ - text)); + for (size_t i = 0; i < chunk.size(); ++i) { + char cc = chunk[i]; + if (cc == '\\') { + escaped = !escaped; + } else { + if (cc == c && !escaped) { + AdvanceProcessingNewLines(static_cast(text - text_) + i + 1); + return true; + } + escaped = false; + } + } + text += chunk.size(); + } + AdvanceProcessingNewLines(text_end_); + return false; +} + +void Lexer::ConsumeIdentTrailing() { + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + for (size_t i = 0; i < chunk.size(); ++i) { + char c = chunk[i]; + if (!IsIdentTrailing(c)) { + Advance(i); + return; + } + } + Advance(chunk.size()); + } +} + +bool Lexer::MatchString(std::string_view s) const { + return absl::StartsWith(GetRemainingText(), s); +} + +bool Lexer::MatchStringIgnoreCase(std::string_view s) const { + return absl::StartsWithIgnoreCase(GetRemainingText(), s); +} + +std::optional Lexer::MatchIf( + absl::FunctionRef predicate) const { + if (text_ != text_end_) { + char c = *text_; + if (predicate(c)) { + return c; + } + } + return std::nullopt; +} + +void Lexer::ConsumeLine() { + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + auto pos = chunk.find('\n'); + if (pos != std::string_view::npos) { + Advance(pos + 1); + if (line_offsets_ != nullptr) { + line_offsets_->push_back(GetPosition()); + } + break; + } + Advance(chunk.size()); + } +} + +void Lexer::ConsumeWhitespace() { + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + size_t i = 0; + next_char: + while (i < chunk.size()) { + char c = chunk[i]; + switch (c) { + case '\n': + if (line_offsets_ != nullptr) { + line_offsets_->push_back(GetPosition() + static_cast(i) + + 1); + } + ABSL_FALLTHROUGH_INTENDED; + case ' ': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\v': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + ++i; + goto next_char; + default: + Advance(i); + return; + } + } + Advance(chunk.size()); + } +} + +bool Lexer::Consume(char c) { + ABSL_DCHECK_NE(c, '\n'); + if (Match(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeIgnoreCase(char c) { + ABSL_DCHECK_NE(c, '\n'); + if (MatchIgnoreCase(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeString(std::string_view s) { + ABSL_DCHECK(!absl::StrContains(s, '\n')); + if (MatchString(s)) { + Advance(s.size()); + return true; + } + return false; +} + +bool Lexer::ConsumeStringIgnoreCase(std::string_view s) { + ABSL_DCHECK(!absl::StrContains(s, '\n')); + if (MatchStringIgnoreCase(s)) { + Advance(s.size()); + return true; + } + return false; +} + +std::optional Lexer::ConsumeIf( + absl::FunctionRef predicate) { + std::optional match = MatchIf(predicate); + if (match.has_value()) { + ABSL_DCHECK_NE(*match, '\n'); + Advance(1); + } + return match; +} + +bool Lexer::ConsumeDigits() { + bool advanced = false; + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + for (size_t i = 0; i < chunk.size(); ++i) { + if (!absl::ascii_isdigit(chunk[i])) { + if (i != 0) { + Advance(i); + return true; + } + return advanced; + } + } + Advance(chunk.size()); + advanced = true; + } + return advanced; +} + +bool Lexer::ConsumeHexDigits() { + bool advanced = false; + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + for (size_t i = 0; i < chunk.size(); ++i) { + if (!absl::ascii_isxdigit(chunk[i])) { + if (i != 0) { + Advance(i); + return true; + } + return advanced; + } + } + Advance(chunk.size()); + advanced = true; + } + return advanced; +} + +TokenType Lexer::ConsumeIntegralSuffix() { + if (ConsumeIgnoreCase('u')) { + return TokenType::kUint; + } + return TokenType::kInt; +} + +void Lexer::AdvanceProcessingNewLines(size_t n) { + while (n > 0) { + std::string_view chunk = GetRemainingText(); + chunk = chunk.substr(0, std::min(chunk.size(), n)); + std::string_view::size_type pos = 0; + while (pos < chunk.size()) { + std::string_view::size_type npos = chunk.find('\n', pos); + if (npos == std::string_view::npos) { + break; + } + ++npos; + if (line_offsets_ != nullptr) { + line_offsets_->push_back(GetPosition() + static_cast(npos)); + } + pos = npos; + } + n -= chunk.size(); + Advance(chunk.size()); + } +} + +void Lexer::AdvanceProcessingNewLines(const char* end) { + ABSL_DCHECK_LE(end, text_end_); + ABSL_DCHECK_GE(end, text_); + AdvanceProcessingNewLines(static_cast(end - text_)); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h new file mode 100644 index 000000000..5bf6ef24d --- /dev/null +++ b/parser/internal/lexer.h @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" + +namespace cel::parser_internal { + +enum class TokenType { + kError = 0, + kEnd, + kWhitespace, + kComment, + + // Keywords + kNull, + kFalse, + kTrue, + kIn, + kReservedWord, + + // Literals + kInt, + kUint, + kFloat, + kString, + kBytes, + + // Identifiers + kIdent, + + // Delimiters + kLeftBracket, // [ + kRightBracket, // ] + kLeftBrace, // { + kRightBrace, // } + kLeftParen, // ( + kRightParen, // ) + + // Operators + kDot, // . + kComma, // , + kMinus, // - + kPlus, // + + kAsterisk, // * + kSlash, // / + kPercent, // % + kQuestion, // ? + kColon, // : + kExclamation, // ! + kEqual, // = + kEqualEqual, // == + kExclamationEqual, // != + kLess, // < + kLessEqual, // <= + kGreater, // > + kGreaterEqual, // >= + kLogicalAnd, // && + kLogicalOr, // || +}; + +ABSL_ATTRIBUTE_PURE_FUNCTION std::string_view TokenTypeToString(TokenType type); + +struct Token final { + TokenType type = TokenType::kError; + int32_t start = 0; + int32_t end = 0; +}; + +struct LexerError final { + int32_t start = 0; + int32_t end = 0; + std::string message; +}; + +// Lexer performs fast tokenization of CEL expression source code. +// +// Responsibilities & Parser Expectations: +// This lexer is designed for speed and does not perform comprehensive semantic +// or syntax validation. It does not verify escape sequences or other special +// characters in string/bytes literals, and performs only general bounds and +// format matching for integers and floating-point numeric literals. The lexer +// expects the parser to perform final validation and conversion when building +// the AST. +class Lexer final { + public: + explicit Lexer(std::string_view source, + std::vector* absl_nullable line_offsets = nullptr) + : line_offsets_(line_offsets), + text_begin_(source.data() != nullptr ? source.data() : ""), + text_end_(text_begin_ + source.size()), + text_(text_begin_) { + ABSL_DCHECK_LT(source.size(), + static_cast(std::numeric_limits::max())); + } + + Lexer(const Lexer&) = delete; + Lexer(Lexer&&) = delete; + Lexer& operator=(const Lexer&) = delete; + Lexer& operator=(Lexer&&) = delete; + + // Scans and returns the next token from the source. + [[nodiscard]] ABSL_ATTRIBUTE_NOINLINE Token Lex(); + + [[nodiscard]] const LexerError& GetError() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + ABSL_DCHECK(at_error_); + return error_; + } + + [[nodiscard]] int32_t GetPosition() const { + return static_cast(text_ - text_begin_); + } + + private: + [[nodiscard]] int32_t Find(char c) const; + + [[nodiscard]] bool Match(char c) const { + return text_ != text_end_ && *text_ == c; + } + + [[nodiscard]] bool MatchIgnoreCase(char c) const { + return text_ != text_end_ && + absl::ascii_tolower(*text_) == absl::ascii_tolower(c); + } + + void Advance(size_t n) { + ABSL_DCHECK_LE(n, static_cast(text_end_ - text_)); + text_ += n; + } + + void AdvanceProcessingNewLines(size_t n); + void AdvanceProcessingNewLines(const char* end); + + [[nodiscard]] std::string_view GetRemainingText() const { + return std::string_view(text_, static_cast(text_end_ - text_)); + } + + [[nodiscard]] Token MakeToken(TokenType type, int32_t start, int32_t end) { + if (ABSL_PREDICT_FALSE(at_end_)) { + AtEndTokenCreated(); + } + return Token{.type = type, .start = start, .end = end}; + } + + [[nodiscard]] Token SetError(int32_t start, int32_t end, + std::string message) { + ABSL_DCHECK(!at_error_); + at_error_ = true; + error_ = + LexerError{.start = start, .end = end, .message = std::move(message)}; + return Token{.type = TokenType::kError, .start = start, .end = end}; + } + + void AtEndTokenCreated() { done_ = true; } + + [[nodiscard]] bool ConsumeUntilAfter(char c); + + [[nodiscard]] bool ConsumeUntilAfterString(std::string_view s); + + [[nodiscard]] bool ConsumeUntilAfterUnescaped(char c); + + void ConsumeIdentTrailing(); + + [[nodiscard]] bool MatchString(std::string_view s) const; + + [[nodiscard]] bool MatchStringIgnoreCase(std::string_view s) const; + + [[nodiscard]] std::optional MatchIf( + absl::FunctionRef predicate) const; + + void ConsumeLine(); + + void ConsumeWhitespace(); + + [[nodiscard]] bool Consume(char c); + + [[nodiscard]] bool ConsumeIgnoreCase(char c); + + [[nodiscard]] bool ConsumeString(std::string_view s); + + [[nodiscard]] bool ConsumeStringIgnoreCase(std::string_view s); + + [[nodiscard]] std::optional ConsumeIf( + absl::FunctionRef predicate); + + [[nodiscard]] bool ConsumeDigits(); + + [[nodiscard]] bool ConsumeHexDigits(); + + [[nodiscard]] TokenType ConsumeIntegralSuffix(); + + std::vector* absl_nullable line_offsets_; + const char* absl_nonnull text_begin_; + const char* absl_nonnull text_end_; + const char* absl_nonnull text_; + bool at_end_ = false; + bool at_error_ = false; + bool done_ = false; + LexerError error_; +}; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc new file mode 100644 index 000000000..0305c9475 --- /dev/null +++ b/parser/internal/lexer_test.cc @@ -0,0 +1,392 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include +#include + +#include "internal/testing.h" + +namespace cel::parser_internal { +namespace { + +MATCHER_P3(IsToken, source, expected_type, expected_text, "") { + if (arg.type != expected_type) { + *result_listener << "type is " << TokenTypeToString(arg.type) + << " (expected " << TokenTypeToString(expected_type) + << ")"; + return false; + } + std::string_view actual_text = source.substr(arg.start, arg.end - arg.start); + if (actual_text != expected_text) { + *result_listener << "text is '" << actual_text << "' (expected '" + << expected_text << "')"; + return false; + } + return true; +} + +struct LexerTestCase { + std::string_view name; + std::string_view source; + std::vector> expected_tokens; +}; + +using LexerTest = testing::TestWithParam; + +TEST_P(LexerTest, LexesSuccessTokens) { + const LexerTestCase& test_case = GetParam(); + Lexer lexer(test_case.source); + + for (const auto& [type, text] : test_case.expected_tokens) { + EXPECT_THAT(lexer.Lex(), IsToken(test_case.source, type, text)); + } + EXPECT_THAT(lexer.Lex(), IsToken(test_case.source, TokenType::kEnd, "")); +} + +INSTANTIATE_TEST_SUITE_P( + LexerTest, LexerTest, + testing::ValuesIn({ + {"NullSource", std::string_view(nullptr, 0), {}}, + {"Empty", "", {}}, + {"Whitespace", " \n ", {{TokenType::kWhitespace, " \n "}}}, + {"KeywordsAndIdents", + "null false true in as return foo_bar _foo_bar _ `quoted.ident`", + {{TokenType::kNull, "null"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFalse, "false"}, + {TokenType::kWhitespace, " "}, + {TokenType::kTrue, "true"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIn, "in"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "as"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "return"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`quoted.ident`"}}}, + {"Numbers", + "123 45u 0x1A 3.14 .5 1e6 2.5e-3 45U 0x1Au 0x1AU", + {{TokenType::kInt, "123"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x1A"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "3.14"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, ".5"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "1e6"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "2.5e-3"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45U"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1Au"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1AU"}}}, + {"IntEOF", "123456", {{TokenType::kInt, "123456"}}}, + {"HexIntEOF", "0x1A2B", {{TokenType::kInt, "0x1A2B"}}}, + {"FloatPositiveExponentEOF", "1e+6", {{TokenType::kFloat, "1e+6"}}}, + {"FloatEOF", ".12345", {{TokenType::kFloat, ".12345"}}}, + {"IntDotIdent", + "1.foo", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kIdent, "foo"}}}, + {"IntDotWhitespace", + "1. ", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}}}, + {"IntDotEOF", "1.", {{TokenType::kInt, "1"}, {TokenType::kDot, "."}}}, + {"DotAtEOFBeforeDigit", + std::string_view(".6", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"DotAtEOFBeforeIdent", + std::string_view(".a", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"ZeroNumbers", + "0 0u 0x0", + {{TokenType::kInt, "0"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x0"}}}, + {"StringsAndBytes", + R"("hello" 'world' """multi +line""" r"raw" b"bytes" rb'\x00' '''multi +single''' R"raw_upper" B"bytes_upper" b'''multi +bytes''' br"raw_bytes" `a.b-c/d e` +"\a\b\f\n\r\t\v\"\'\\\?\` \x1A \u00A0 \U0001F600 \012")", + {{TokenType::kString, "\"hello\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'world'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "\"\"\"multi\nline\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"raw\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "rb'\\x00'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'''multi\nsingle'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "R\"raw_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "B\"bytes_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b'''multi\nbytes'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "br\"raw_bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`a.b-c/d e`"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kString, + "\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\\?\\` \\x1A \\u00A0 \\U0001F600 " + "\\012\""}}}, + {"EmptyStrings", + "\"\" '' \"\"\"\"\"\" '''''' r\"\" r'' r\"\"\"\"\"\" r'''''' b\"\" " + "b'' b\"\"\"\"\"\" b''''''", + {{TokenType::kString, "\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''''''"}}}, + {"OperatorsAndDelimiters", + ". , + - * / % == != < <= > >= && || ! ? : [] { } ( )", + {{TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kPlus, "+"}, + {TokenType::kWhitespace, " "}, + {TokenType::kMinus, "-"}, + {TokenType::kWhitespace, " "}, + {TokenType::kAsterisk, "*"}, + {TokenType::kWhitespace, " "}, + {TokenType::kSlash, "/"}, + {TokenType::kWhitespace, " "}, + {TokenType::kPercent, "%"}, + {TokenType::kWhitespace, " "}, + {TokenType::kEqualEqual, "=="}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamationEqual, "!="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLess, "<"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLessEqual, "<="}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreater, ">"}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreaterEqual, ">="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalAnd, "&&"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalOr, "||"}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamation, "!"}, + {TokenType::kWhitespace, " "}, + {TokenType::kQuestion, "?"}, + {TokenType::kWhitespace, " "}, + {TokenType::kColon, ":"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, + {TokenType::kRightBracket, "]"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightBrace, "}"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftParen, "("}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightParen, ")"}}}, + {"Comments", + "a\n// comment\nb", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kComment, "// comment\n"}, + {TokenType::kIdent, "b"}}}, + {"CommentWithoutTrailingNewlineEOF", + "// comment without trailing newline", + {{TokenType::kComment, "// comment without trailing newline"}}}, + {"CommentAfterTokenWithoutTrailingNewlineEOF", + "a // comment without trailing newline", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, " "}, + {TokenType::kComment, "// comment without trailing newline"}}}, + }), + [](const testing::TestParamInfo& info) { + return std::string(info.param.name); + }); + +TEST(LexerTest, LineOffsets) { + std::string_view source = "a\n// comment\nb"; + std::vector line_offsets; + Lexer lexer(source, &line_offsets); + + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kIdent, "a")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source, TokenType::kComment, "// comment\n")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kIdent, "b")); + + ASSERT_EQ(line_offsets.size(), 2); + EXPECT_EQ(line_offsets[0], 2); + EXPECT_EQ(line_offsets[1], 13); +} + +TEST(LexerTest, LineOffsetsInStringsAndIdentifiers) { + std::string_view source = + "'''multi\nline'''\n\"another\nline\"\n`ident\nhere`"; + std::vector line_offsets; + Lexer lexer(source, &line_offsets); + + EXPECT_THAT(lexer.Lex(), + IsToken(source, TokenType::kString, "'''multi\nline'''")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source, TokenType::kString, "\"another\nline\"")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kIdent, "`ident\nhere`")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kEnd, "")); + + ASSERT_EQ(line_offsets.size(), 5); + EXPECT_EQ(line_offsets[0], 9); + EXPECT_EQ(line_offsets[1], 17); + EXPECT_EQ(line_offsets[2], 26); + EXPECT_EQ(line_offsets[3], 32); + EXPECT_EQ(line_offsets[4], 39); +} + +struct LexerErrorTestCase { + std::string_view source; + std::string_view expected_error_message; + int32_t expected_position; +}; + +using LexerErrorTest = testing::TestWithParam; + +TEST_P(LexerErrorTest, LexesErrorTokenAndStoresError) { + const LexerErrorTestCase& test_case = GetParam(); + Lexer lexer(test_case.source); + Token token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kError); + EXPECT_EQ(lexer.GetError().message, test_case.expected_error_message); + EXPECT_EQ(lexer.GetPosition(), test_case.expected_position); +} + +INSTANTIATE_TEST_SUITE_P( + ErrorCases, LexerErrorTest, + testing::Values( + LexerErrorTestCase{ + .source = "\"unterminated", + .expected_error_message = "unterminated string literal", + .expected_position = 13, + }, + LexerErrorTestCase{ + .source = "0x", + .expected_error_message = + "integral literal missing digits after hexadecimal separator", + .expected_position = 2, + }, + LexerErrorTestCase{ + .source = "@", + .expected_error_message = "unexpected character", + .expected_position = 1, + }, + LexerErrorTestCase{ + .source = "0x1A_invalid", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_position = 5, + }, + LexerErrorTestCase{ + .source = "123_invalid", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_position = 4, + }, + LexerErrorTestCase{ + .source = "1x0", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_position = 2, + }, + LexerErrorTestCase{ + .source = "2x", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_position = 2, + }, + LexerErrorTestCase{ + .source = "`unterminated quoted", + .expected_error_message = "unterminated quoted identifier", + .expected_position = 20, + }, + LexerErrorTestCase{ + .source = "'''unterminated multi", + .expected_error_message = "unterminated string literal", + .expected_position = 21, + }, + LexerErrorTestCase{ + .source = "r'unterminated raw", + .expected_error_message = "unterminated string literal", + .expected_position = 18, + }, + LexerErrorTestCase{ + .source = "b'unterminated bytes", + .expected_error_message = "unterminated bytes literal", + .expected_position = 20, + }, + LexerErrorTestCase{ + .source = "1e", + .expected_error_message = + "floating point literal missing digits after exponent " + "separator", + .expected_position = 2, + })); + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/options.h b/parser/internal/options.h index ec2552204..4066d2619 100644 --- a/parser/internal/options.h +++ b/parser/internal/options.h @@ -15,7 +15,7 @@ #ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ -namespace cel_parser_internal { +namespace cel::parser_internal { inline constexpr int kDefaultErrorRecoveryLimit = 12; inline constexpr int kDefaultMaxRecursionDepth = 32; @@ -23,6 +23,6 @@ inline constexpr int kExpressionSizeCodepointLimit = 100'000; inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = 512; inline constexpr bool kDefaultAddMacroCalls = false; -} // namespace cel_parser_internal +} // namespace cel::parser_internal #endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ diff --git a/parser/internal/pratt_parser.cc b/parser/internal/pratt_parser.cc new file mode 100644 index 000000000..d346dfd6a --- /dev/null +++ b/parser/internal/pratt_parser.cc @@ -0,0 +1,214 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/expr.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "parser/internal/ast_factory.h" // IWYU pragma: keep +#include "parser/internal/pratt_parser_worker.h" +#include "parser/macro.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +namespace { + +std::string DisplayParserError(const cel::Source& source, + SourceLocation location, + std::string_view message) { + return absl::StrCat( + absl::StrFormat("ERROR: %s:%zu:%zu: %s", source.description(), + location.line, location.column + 1, message), + source.DisplayErrorLocation(location)); +} + +std::string FormatIssues(const cel::Source& source, + absl::Span issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend( + out, DisplayParserError(source, issue.location(), issue.message())); + }); +} + +class PrattParserBuilderImpl final : public cel::ParserBuilder { + public: + explicit PrattParserBuilderImpl(const cel::ParserOptions& options) + : options_(options) {} + + cel::ParserOptions& GetOptions() override { return options_; } + + absl::Status AddMacro(const cel::Macro& macro) override { + for (const auto& existing_macro : macros_) { + if (existing_macro.key() == macro.key()) { + return absl::AlreadyExistsError( + absl::StrCat("macro already exists: ", macro.key())); + } + } + macros_.push_back(macro); + return absl::OkStatus(); + } + + absl::Status AddLibrary(cel::ParserLibrary library) override { + if (!library.id.empty()) { + auto [it, inserted] = library_ids_.insert(library.id); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library already exists: ", library.id)); + } + } + libraries_.push_back(std::move(library)); + return absl::OkStatus(); + } + + absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override { + if (subset.library_id.empty()) { + return absl::InvalidArgumentError("subset must have a library id"); + } + std::string library_id = subset.library_id; + auto [it, inserted] = + library_subsets_.insert({library_id, std::move(subset)}); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library subset already exists: ", library_id)); + } + return absl::OkStatus(); + } + + absl::StatusOr> Build() override { + using std::swap; + std::vector individual_macros; + swap(individual_macros, macros_); + absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); + + cel::MacroRegistry macro_registry; + + for (const auto& library : libraries_) { + CEL_RETURN_IF_ERROR(library.configure(*this)); + if (!library.id.empty()) { + auto it = library_subsets_.find(library.id); + if (it != library_subsets_.end()) { + const cel::ParserLibrarySubset& subset = it->second; + for (const auto& macro : macros_) { + if (subset.should_include_macro(macro)) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); + } + } + macros_.clear(); + continue; + } + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); + macros_.clear(); + } + + absl::flat_hash_set library_ids(library_ids_); + + if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); + library_ids.insert("stdlib"); + } + + if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); + library_ids.insert("optional"); + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); + return std::make_unique( + options_, std::move(macro_registry), std::move(library_ids)); + } + + cel::ParserOptions options_; + std::vector macros_; + absl::flat_hash_set library_ids_; + std::vector libraries_; + absl::flat_hash_map library_subsets_; +}; + +} // namespace + +template class PrattParserWorker; + +using CelPrattParserWorker = PrattParserWorker; + +absl::StatusOr> PrattParserImpl::ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const { + if (source.content().size() > options_.expression_size_codepoint_limit) { + return absl::InvalidArgumentError(absl::StrFormat( + "expression size exceeds codepoint limit: %zu > %d", + source.content().size(), options_.expression_size_codepoint_limit)); + } + std::vector local_issues; + std::vector* actual_issues = + parse_issues != nullptr ? parse_issues : &local_issues; + CelPrattParserWorker worker(source, options_, actual_issues); + Expr expr = worker.Parse(); + if (worker.has_errors()) { + std::string err_msg = FormatIssues(source, *actual_issues); + return absl::InvalidArgumentError(err_msg); + } + + cel::SourceInfo source_info; + source_info.set_location(std::string(source.description())); + for (const auto& [id, pos] : worker.GetNodePositions()) { + source_info.mutable_positions().insert({id, pos}); + } + source_info.mutable_line_offsets().reserve(worker.GetLineOffsets().size()); + for (int32_t offset : worker.GetLineOffsets()) { + source_info.mutable_line_offsets().push_back(offset); + } + + return std::make_unique(std::move(expr), std::move(source_info)); +} + +std::unique_ptr PrattParserImpl::ToBuilder() const { + auto ins = std::make_unique(options_); + ins->library_ids_ = library_ids_; + ins->macros_ = macro_registry_.ListMacros(); + return ins; +} + +std::unique_ptr NewPrattParserBuilder( + const cel::ParserOptions& options) { + return std::make_unique(options); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser.h b/parser/internal/pratt_parser.h new file mode 100644 index 000000000..2be081184 --- /dev/null +++ b/parser/internal/pratt_parser.h @@ -0,0 +1,62 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/statusor.h" +#include "common/ast.h" +#include "common/source.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +class PrattParserImpl final : public cel::Parser { + public: + explicit PrattParserImpl(const cel::ParserOptions& options, + cel::MacroRegistry macro_registry, + absl::flat_hash_set library_ids) + : options_(options), + macro_registry_(std::move(macro_registry)), + library_ids_(std::move(library_ids)) {} + + ~PrattParserImpl() override = default; + + absl::StatusOr> ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const override; + + std::unique_ptr ToBuilder() const override; + + private: + cel::ParserOptions options_; + cel::MacroRegistry macro_registry_; + absl::flat_hash_set library_ids_; +}; + +std::unique_ptr NewPrattParserBuilder( + const cel::ParserOptions& options = cel::ParserOptions()); + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc new file mode 100644 index 000000000..7ab148b12 --- /dev/null +++ b/parser/internal/pratt_parser_test.cc @@ -0,0 +1,542 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "parser/options.h" +#include "parser/parser_interface.h" +#include "testutil/expr_printer.h" + +namespace cel::parser_internal { +namespace { + +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::testing::HasSubstr; +using ::testing::NotNull; + +absl::StatusOr> Parse( + std::string_view expression, + const cel::ParserOptions& options = cel::ParserOptions()) { + auto builder = NewPrattParserBuilder(options); + CEL_ASSIGN_OR_RETURN(auto parser, builder->Build()); + CEL_ASSIGN_OR_RETURN(auto source, cel::NewSource(expression)); + return parser->Parse(*source); +} + +struct TestCase { + std::string_view source; + std::string_view expected_ast; + bool enable_optional_syntax = false; +}; + +class PrattParserTest : public testing::TestWithParam {}; + +std::string Unindent(std::string_view multiline) { + std::vector unindented_lines; + int indent = -1; + for (std::string_view line : absl::StrSplit(multiline, '\n')) { + std::size_t pos = line.find_first_not_of(" \t"); + if (pos == std::string_view::npos) continue; + if (indent == -1) indent = pos; + unindented_lines.push_back(std::string(line.substr(indent))); + } + return absl::StrJoin(unindented_lines, "\n"); +} + +std::string_view ConstantKind(const cel::Constant& c) { + switch (c.kind_case()) { + case ConstantKindCase::kBool: + return "bool"; + case ConstantKindCase::kInt: + return "int64"; + case ConstantKindCase::kUint: + return "uint64"; + case ConstantKindCase::kDouble: + return "double"; + case ConstantKindCase::kString: + return "string"; + case ConstantKindCase::kBytes: + return "bytes"; + case ConstantKindCase::kNull: + return "NullValue"; + default: + return "unspecified_constant"; + } +} + +std::string_view ExprKind(const cel::Expr& e) { + switch (e.kind_case()) { + case ExprKindCase::kConstant: + // special cased, this doesn't appear. + return "Expr.Constant"; + case ExprKindCase::kIdentExpr: + return "Expr.Ident"; + case ExprKindCase::kSelectExpr: + return "Expr.Select"; + case ExprKindCase::kCallExpr: + return "Expr.Call"; + case ExprKindCase::kListExpr: + return "Expr.CreateList"; + case ExprKindCase::kMapExpr: + case ExprKindCase::kStructExpr: + return "Expr.CreateStruct"; + case ExprKindCase::kComprehensionExpr: + return "Expr.Comprehension"; + default: + return "unspecified_expr"; + } +} + +class KindAndIdAdorner : public cel::test::ExpressionAdorner { + public: + // // Use default source_info constructor to make source_info "optional". This + // // will prevent macro_calls lookups from interfering with adorning + // expressions + // // that don't need to use macro_calls, such as the parsed AST. + // explicit KindAndIdAdorner( + // const cel::expr::SourceInfo& source_info = + // cel::expr::SourceInfo::default_instance()) + // : source_info_(source_info) {} + + std::string Adorn(const cel::Expr& e) const override { + // source_info_ might be empty on non-macro_calls tests + // if (source_info_.macro_calls_size() != 0 && + // source_info_.macro_calls().contains(e.id())) { + // return absl::StrFormat( + // "^#%d:%s#", e.id(), + // source_info_.macro_calls().at(e.id()).call_expr().function()); + // } + + if (e.has_const_expr()) { + auto& const_expr = e.const_expr(); + return absl::StrCat("^#", e.id(), ":", ConstantKind(const_expr), "#"); + } else { + return absl::StrCat("^#", e.id(), ":", ExprKind(e), "#"); + } + } + + std::string AdornStructField(const cel::StructExprField& e) const override { + return absl::StrFormat("^#%d:Expr.CreateStruct.Entry#", e.id()); + } + + std::string AdornMapEntry(const cel::MapExprEntry& e) const override { + return absl::StrFormat("^#%d:Expr.CreateStruct.Entry#", e.id()); + } + + // private: + // const cel::expr::SourceInfo& source_info_; +}; + +TEST_P(PrattParserTest, Parse) { + const TestCase& test_case = GetParam(); + cel::ParserOptions options; + options.enable_optional_syntax = test_case.enable_optional_syntax; + ASSERT_OK_AND_ASSIGN(auto ast, Parse(test_case.source, options)); + const Expr& root = ast->root_expr(); + KindAndIdAdorner kind_and_id_adorner; + test::ExprPrinter printer(kind_and_id_adorner); + EXPECT_EQ(Unindent(printer.Print(root)), Unindent(test_case.expected_ast)); +} + +std::vector GetParserTestCases() { + return { + TestCase{ + .source = "null", + .expected_ast = R"( + null^#1:NullValue# + )", + }, + TestCase{ + .source = "42u", + .expected_ast = R"( + 42u^#1:uint64# + )", + }, + TestCase{ + .source = "b'hello'", + .expected_ast = R"( + b"hello"^#1:bytes# + )", + }, + TestCase{ + .source = "a.b(1)", + .expected_ast = R"( + a^#1:Expr.Ident#.b( + 1^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "123", + .expected_ast = R"( + 123^#1:int64# + )", + }, + TestCase{ + .source = "3.14159", + .expected_ast = R"( + 3.14159^#1:double# + )", + }, + TestCase{ + .source = "'hello world'", + .expected_ast = R"( + "hello world"^#1:string# + )", + }, + TestCase{ + .source = "1 + 2 * 3 == 7 && true || false", + .expected_ast = R"( + _||_( + _&&_( + _==_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + 7^#7:int64# + )^#6:Expr.Call#, + true^#9:bool# + )^#8:Expr.Call#, + false^#11:bool# + )^#10:Expr.Call# + )", + }, + TestCase{ + .source = "x > 0 ? 'pos' : 'neg'", + .expected_ast = R"( + _?_:_( + _>_( + x^#1:Expr.Ident#, + 0^#3:int64# + )^#2:Expr.Call#, + "pos"^#5:string#, + "neg"^#6:string# + )^#4:Expr.Call# + )", + }, + TestCase{ + .source = "[1, 2, 3]", + .expected_ast = R"( + [ + 1^#2:int64#, + 2^#3:int64#, + 3^#4:int64# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "{'key': 'value', 'num': 42}", + .expected_ast = R"( + { + "key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, + "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "google.protobuf.Empty{}", + .expected_ast = R"( + google.protobuf.Empty{}^#3:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "Msg{field: 10, other: 'val'}", + .expected_ast = R"( + Msg{ + field:10^#2:int64#^#1:Expr.CreateStruct.Entry#, + other:"val"^#4:string#^#3:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "has(message.field)", + .expected_ast = R"( + message^#2:Expr.Ident#.field~test-only~^#1:Expr.Select# + )", + }, + TestCase{ + .source = "[?1, 2]", + .expected_ast = R"( + [ + ?1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "{?'key': 'value', 'num': 42}", + .expected_ast = R"( + { + ?"key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, + "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "Msg{?field: 10, other: 'val'}", + .expected_ast = R"( + Msg{ + ?field:10^#2:int64#^#1:Expr.CreateStruct.Entry#, + other:"val"^#4:string#^#3:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + .enable_optional_syntax = true, + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserTest, PrattParserTest, + testing::ValuesIn(GetParserTestCases())); + +struct ErrorTestCase { + std::string_view source; + std::string_view expected_error; + bool enable_optional_syntax = false; + bool enable_quoted_identifiers = false; +}; + +class PrattParserErrorTest : public testing::TestWithParam {}; + +TEST_P(PrattParserErrorTest, ParseSyntaxError) { + const ErrorTestCase& test_case = GetParam(); + cel::ParserOptions options; + options.enable_optional_syntax = test_case.enable_optional_syntax; + options.enable_quoted_identifiers = test_case.enable_quoted_identifiers; + auto result = Parse(test_case.source, options); + EXPECT_THAT(result, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr(test_case.expected_error))); +} + +std::vector GetErrorTestCases() { + return { + ErrorTestCase{ + .source = "1 + 2 * 3 4", + .expected_error = + "ERROR: :1:11: unexpected token after expression\n" + " | 1 + 2 * 3 4\n" + " | ..........^", + }, + ErrorTestCase{ + .source = "1{}", + .expected_error = + "ERROR: :1:2: unexpected token after expression\n" + " | 1{}\n" + " | .^", + }, + ErrorTestCase{ + .source = "true ? 1", + .expected_error = + "ERROR: :1:9: expected ':' in conditional expression\n" + " | true ? 1\n" + " | ........^", + }, + ErrorTestCase{ + .source = "a.?b", + .expected_error = "ERROR: :1:2: unsupported syntax '.?'\n" + " | a.?b\n" + " | .^", + }, + ErrorTestCase{ + .source = "a.", + .expected_error = + "ERROR: :1:3: expected identifier after '.'\n" + " | a.\n" + " | ..^", + }, + ErrorTestCase{ + .source = "a[?0]", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | a[?0]\n" + " | .^", + }, + ErrorTestCase{ + .source = ". *", + .expected_error = "ERROR: :1:3: expected identifier\n" + " | . *\n" + " | ..^", + }, + ErrorTestCase{ + .source = ".as", + .expected_error = "ERROR: :1:2: reserved identifier: as\n" + " | .as\n" + " | .^", + }, + ErrorTestCase{ + .source = "* 2", + .expected_error = "ERROR: :1:1: unexpected token\n" + " | * 2\n" + " | ^", + }, + ErrorTestCase{ + .source = "(1 + 2", + .expected_error = "ERROR: :1:4: expected ')'\n" + " | (1 + 2\n" + " | ...^", + }, + ErrorTestCase{ + .source = "[?1]", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | [?1]\n" + " | .^", + }, + ErrorTestCase{ + .source = "[1, 2", + .expected_error = "ERROR: :1:6: expected ']'\n" + " | [1, 2\n" + " | .....^", + }, + ErrorTestCase{ + .source = "{?'k': 'v'}", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | {?'k': 'v'}\n" + " | .^", + }, + ErrorTestCase{ + .source = "{'k' 'v'}", + .expected_error = "ERROR: :1:6: expected ':' in map entry\n" + " | {'k' 'v'}\n" + " | .....^", + }, + ErrorTestCase{ + .source = "{'k': 'v'", + .expected_error = "ERROR: :1:10: expected '}'\n" + " | {'k': 'v'\n" + " | .........^", + }, + ErrorTestCase{ + .source = "Msg{?f: 1}", + .expected_error = "ERROR: :1:5: unsupported syntax '?'\n" + " | Msg{?f: 1}\n" + " | ....^", + }, + ErrorTestCase{ + .source = "Msg{1: 2}", + .expected_error = "ERROR: :1:5: expected struct field name\n" + " | Msg{1: 2}\n" + " | ....^", + }, + ErrorTestCase{ + .source = "Msg{f 10}", + .expected_error = "ERROR: :1:7: expected ':' in struct field\n" + " | Msg{f 10}\n" + " | ......^", + }, + ErrorTestCase{ + .source = "Msg{f: 10", + .expected_error = "ERROR: :1:10: expected '}'\n" + " | Msg{f: 10\n" + " | .........^", + }, + ErrorTestCase{ + .source = "f(1, 2", + .expected_error = "ERROR: :1:7: expected )\n" + " | f(1, 2\n" + " | ......^", + }, + ErrorTestCase{ + .source = "999999999999999999999999999999999999999", + .expected_error = "ERROR: :1:1: invalid int literal\n" + " | 999999999999999999999999999999999999999\n" + " | ^", + }, + ErrorTestCase{ + .source = "999999999999999999999999999999999999999u", + .expected_error = "ERROR: :1:1: invalid uint literal\n" + " | 999999999999999999999999999999999999999u\n" + " | ^", + }, + ErrorTestCase{ + .source = "1e", + .expected_error = + "ERROR: :1:1: floating point literal missing digits after " + "exponent separator\n" + " | 1e\n" + " | ^", + }, + ErrorTestCase{ + .source = "\"unterminated", + .expected_error = "ERROR: :1:1: unterminated string literal\n" + " | \"unterminated\n" + " | ^", + }, + ErrorTestCase{ + .source = "b\"unterminated", + .expected_error = "ERROR: :1:1: unterminated bytes literal\n" + " | b\"unterminated\n" + " | ^", + }, + ErrorTestCase{ + .source = "a.?`foo`", + .expected_error = "ERROR: :1:4: unexpected quoted identifier\n" + " | a.?`foo`\n" + " | ...^", + .enable_optional_syntax = true, + }, + ErrorTestCase{ + .source = "a.`foo`", + .expected_error = "ERROR: :1:3: unsupported syntax '`'\n" + " | a.`foo`\n" + " | ..^", + .enable_quoted_identifiers = false, + }, + ErrorTestCase{ + .source = "`foo", + .expected_error = + "ERROR: :1:1: unterminated quoted identifier\n" + " | `foo\n" + " | ^", + .enable_quoted_identifiers = true, + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserErrorTest, PrattParserErrorTest, + testing::ValuesIn(GetErrorTestCases())); + +TEST(PrattParserRecursionDepthTest, ParseRecursionDepth) { + cel::ParserOptions options; + options.max_recursion_depth = 5; + EXPECT_THAT(Parse("((((((1))))))", options), IsOkAndHolds(NotNull())); + EXPECT_THAT(Parse("[[[[[[1]]]]]]", options), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc new file mode 100644 index 000000000..9fd2752b3 --- /dev/null +++ b/parser/internal/pratt_parser_worker.cc @@ -0,0 +1,187 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser_worker.h" + +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "parser/internal/lexer.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +ParserWorker::ParserWorker( + const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues) + : source_(source), + options_(options), + source_content_test_owner_(source_.content().ToString()), + lexer_(source_content_test_owner_, &line_offsets_), + parse_issues_(parse_issues) {} + +void ParserWorker::InitTokenStream() { + current_token_ = Token{.type = TokenType::kError, .start = 0, .end = 0}; + peek_token_ = LexNextSignificantToken(); +} + +std::string_view ParserWorker::GetTokenText(const Token& tok) const { + if (tok.start >= 0 && tok.end >= tok.start && + tok.end <= static_cast(source_content_test_owner_.size())) { + return std::string_view(source_content_test_owner_) + .substr(tok.start, tok.end - tok.start); + } + return ""; +} + +Token ParserWorker::LexNextSignificantToken() { + while (true) { + Token tok = lexer_.Lex(); + if (tok.type == TokenType::kWhitespace || tok.type == TokenType::kComment) { + continue; + } + if (tok.type == TokenType::kError) { + ReportError(tok, lexer_.GetError().message); + } + return tok; + } +} + +Token ParserWorker::NextToken() { + current_token_ = peek_token_; + peek_token_ = LexNextSignificantToken(); + return current_token_; +} + +bool ParserWorker::Match(TokenType type) { + if (peek_token_.type == type) { + NextToken(); + return true; + } + return false; +} + +bool ParserWorker::Expect(TokenType type, std::string_view msg) { + if (peek_token_.type == type) { + NextToken(); + return true; + } + std::string err_msg = msg.empty() + ? absl::StrCat("expected ", TokenTypeToString(type)) + : std::string(msg); + ReportError(peek_token_, err_msg); + SynchronizeOnDelimiter(); + return false; +} + +void ParserWorker::SynchronizeOnDelimiter() { + if (is_recovery_limit_exceeded()) { + while (peek_token_.type != TokenType::kEnd) { + NextToken(); + } + return; + } + while (peek_token_.type != TokenType::kEnd) { + if (peek_token_.type == TokenType::kComma || + peek_token_.type == TokenType::kRightParen || + peek_token_.type == TokenType::kRightBracket || + peek_token_.type == TokenType::kRightBrace) { + break; + } + NextToken(); + } +} + +bool ParserWorker::CheckRecursionDepth(int32_t start_pos) { + if (recursion_depth_ >= options_.max_recursion_depth) { + ReportError( + Token{.type = TokenType::kError, .start = start_pos, .end = start_pos}, + absl::StrFormat("Exceeded max recursion depth of %d when parsing.", + options_.max_recursion_depth)); + return false; + } + return true; +} + +bool ParserWorker::PushScope(ScopeKind kind, int32_t start_pos) { + if (!CheckRecursionDepth(start_pos)) { + return false; + } + recursion_depth_++; + scope_stack_.push_back(Scope{.kind = kind, .start_position = start_pos}); + return true; +} + +void ParserWorker::PopScope() { + ABSL_DCHECK(!scope_stack_.empty()); + if (!scope_stack_.empty()) { + scope_stack_.pop_back(); + recursion_depth_--; + } +} + +int64_t ParserWorker::NextId(int32_t position) { + int64_t id = next_id_++; + if (position >= 0) { + positions_.insert({id, position}); + } + return id; +} + +int64_t ParserWorker::CopyId(int64_t id) { + if (id == 0) { + return 0; + } + int32_t pos = 0; + if (auto it = positions_.find(id); it != positions_.end()) { + pos = it->second; + } + return NextId(pos); +} + +void ParserWorker::EraseId(int64_t id) { + positions_.erase(id); + if (next_id_ == id + 1) { + --next_id_; + } +} + +void ParserWorker::ReportError(int32_t position, std::string_view msg) { + error_count_++; + if (parse_issues_ != nullptr) { + cel::SourceLocation loc; + if (auto found = source_.GetLocation(position); found.has_value()) { + loc = *found; + } + parse_issues_->push_back(cel::ParseIssue(loc, std::string(msg))); + } +} + +void ParserWorker::ReportError(const SourceLocation& loc, + std::string_view msg) { + error_count_++; + if (parse_issues_ != nullptr) { + parse_issues_->push_back(cel::ParseIssue(loc, std::string(msg))); + } +} + +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h new file mode 100644 index 000000000..e106b28d5 --- /dev/null +++ b/parser/internal/pratt_parser_worker.h @@ -0,0 +1,817 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "common/operators.h" +#include "common/source.h" +#include "internal/lexis.h" +#include "internal/strings.h" +#include "parser/internal/ast_factory_interface.h" +#include "parser/internal/lexer.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +enum class ScopeKind { + kParen, // (...) + kList, // [...] + kMap, // {...} + kStruct, // Message{...} + kCallArgs, // f(...) +}; + +struct Scope final { + ScopeKind kind; + int32_t start_position = 0; +}; + +class ParserWorker { + public: + ParserWorker(const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues); + + const cel::Source& source() const { return source_; } + const cel::ParserOptions& options() const { return options_; } + const std::string& source_content() const { + return source_content_test_owner_; + } + + // Token stream access + Token NextToken(); + const Token& PeekToken() const { return peek_token_; } + const Token& CurrentToken() const { return current_token_; } + bool Match(TokenType type); + bool Expect(TokenType type, std::string_view msg = ""); + std::string_view GetTokenText(const Token& tok) const; + void SynchronizeOnDelimiter(); + + // Scope & Recursion Depth management + bool PushScope(ScopeKind kind, int32_t start_pos); + void PopScope(); + bool CheckRecursionDepth(int32_t start_pos); + + // ID and Position tracking + int64_t NextId(int32_t position); + int64_t NextId(const Token& token) { return NextId(token.start); } + int64_t NextId() { return next_id_++; } + int64_t CopyId(int64_t id); + void EraseId(int64_t id); + + const absl::flat_hash_map& GetNodePositions() const { + return positions_; + } + const std::vector& GetLineOffsets() const { return line_offsets_; } + + // Error reporting and recovery + bool has_errors() const { return error_count_ > 0; } + bool is_recovery_limit_exceeded() const { + return error_count_ >= options_.error_recovery_limit; + } + void ReportError(int32_t position, std::string_view msg); + void ReportError(const SourceLocation& loc, std::string_view msg); + void ReportError(const Token& token, std::string_view msg) { + ReportError(token.start, msg); + } + + protected: + void InitTokenStream(); + Token LexNextSignificantToken(); + + const cel::Source& source_; + cel::ParserOptions options_; + std::string source_content_test_owner_; + std::vector line_offsets_; + Lexer lexer_; + Token current_token_; + Token peek_token_; + std::vector scope_stack_; + int recursion_depth_ = 0; + int64_t next_id_ = 1; + absl::flat_hash_map positions_; + std::vector* absl_nullable parse_issues_; + int error_count_ = 0; +}; + +// RAII guard for managing parser scope and recursion depth during nested AST +// construction (e.g., when parsing lists, maps, structs, or function calls). +class ScopedRecursionGuard final { + public: + explicit ScopedRecursionGuard(ParserWorker& worker, bool success) + : worker_(worker), active_(success) {} + ~ScopedRecursionGuard() { + if (active_) { + worker_.PopScope(); + } + } + explicit operator bool() const { return active_; } + ScopedRecursionGuard(const ScopedRecursionGuard&) = delete; + ScopedRecursionGuard& operator=(const ScopedRecursionGuard&) = delete; + + private: + ParserWorker& worker_; + bool active_; +}; + +// Generic Pratt parser implementation parameterized by the AST node type +// (`ExprNode`). +// +// This class implements the core recursive-descent and operator-precedence +// parsing logic for CEL without depending on a concrete expression node data +// structure. All inspection and construction of AST nodes is performed through +// `AstFactoryInterface`. +// +// See `AstFactoryInterface` documentation in `ast_factory_interface.h` for +// details on how to use this generic parser with alternative AST structures. +template +class PrattParserWorker : public ParserWorker { + public: + explicit PrattParserWorker( + const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues) + : ParserWorker(source, options, parse_issues) { + this->InitTokenStream(); + } + + ExprNode Parse(); + + private: + using CelOperator = ::google::api::expr::common::CelOperator; + + ExprNode ParseExpr(); + ExprNode ParseConditionalOr(); + ExprNode ParseConditionalAnd(); + ExprNode ParseRelation(); + ExprNode ParseCalc(int min_prec); + ExprNode ParseUnary(); + ExprNode ParseMember(); + ExprNode ParsePrimary(); + ExprNode ParseList(); + ExprNode ParseMap(); + ExprNode ParseStruct(int64_t obj_id, std::string struct_name); + std::vector ParseArguments(TokenType close_token); + ExprNode ParseIntLiteral(); + ExprNode ParseUintLiteral(); + ExprNode ParseDoubleLiteral(); + ExprNode ParseStringLiteral(); + ExprNode ParseBytesLiteral(); + std::string NormalizeIdent(const Token& tok, bool allow_quoted); + std::optional ExtractStructName(const ExprNode& expr); + ExprNode BalancedTree(std::string_view op, std::vector& terms, + const std::vector& ops, int lo, int hi); + ExprNode BalanceLogical(std::string_view op, std::vector terms, + std::vector ops, bool enable_variadic); + + AstFactoryInterface ast_factory_; +}; + +template +ExprNode PrattParserWorker::Parse() { + ExprNode expr = ParseExpr(); + if (PeekToken().type != TokenType::kEnd) { + ReportError(PeekToken(), "unexpected token after expression"); + } + return expr; +} + +template +ExprNode PrattParserWorker::ParseExpr() { + ExprNode lhs = ParseConditionalOr(); + if (Match(TokenType::kQuestion)) { + int64_t op_id = NextId(); + ExprNode true_expr = ParseConditionalOr(); + if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { + return lhs; + } + ExprNode false_expr = ParseExpr(); + return ast_factory_.NewCall( + op_id, CelOperator::CONDITIONAL, + std::vector{std::move(lhs), std::move(true_expr), + std::move(false_expr)}); + } + return lhs; +} + +// Parses logical OR expressions (e.g., `a || b || c`). +template +ExprNode PrattParserWorker::ParseConditionalOr() { + ExprNode lhs = ParseConditionalAnd(); + if (PeekToken().type != TokenType::kLogicalOr) { + return lhs; + } + std::vector terms; + std::vector ops; + terms.push_back(std::move(lhs)); + while (PeekToken().type == TokenType::kLogicalOr) { + Token op_tok = NextToken(); + ops.push_back(NextId(op_tok)); + terms.push_back(ParseConditionalAnd()); + } + return BalanceLogical(CelOperator::LOGICAL_OR, std::move(terms), + std::move(ops), + options_.enable_variadic_logical_operators); +} + +// Parses logical AND expressions (e.g., `a && b && c`). +template +ExprNode PrattParserWorker::ParseConditionalAnd() { + ExprNode lhs = ParseRelation(); + if (PeekToken().type != TokenType::kLogicalAnd) { + return lhs; + } + std::vector terms; + std::vector ops; + terms.push_back(std::move(lhs)); + while (PeekToken().type == TokenType::kLogicalAnd) { + Token op_tok = NextToken(); + ops.push_back(NextId(op_tok)); + terms.push_back(ParseRelation()); + } + return BalanceLogical(CelOperator::LOGICAL_AND, std::move(terms), + std::move(ops), + options_.enable_variadic_logical_operators); +} + +// Parses relational & equality ops (e.g., `a < b`, `x == y`, `a in b`). +template +ExprNode PrattParserWorker::ParseRelation() { + ExprNode lhs = ParseCalc(0); + while (true) { + TokenType tok = PeekToken().type; + std::string_view op_name; + switch (tok) { + case TokenType::kLess: + op_name = CelOperator::LESS; + break; + case TokenType::kLessEqual: + op_name = CelOperator::LESS_EQUALS; + break; + case TokenType::kGreater: + op_name = CelOperator::GREATER; + break; + case TokenType::kGreaterEqual: + op_name = CelOperator::GREATER_EQUALS; + break; + case TokenType::kEqualEqual: + op_name = CelOperator::EQUALS; + break; + case TokenType::kExclamationEqual: + op_name = CelOperator::NOT_EQUALS; + break; + case TokenType::kIn: + op_name = CelOperator::IN; + break; + default: + return lhs; + } + Token op = NextToken(); + int64_t op_id = NextId(op); + ExprNode rhs = ParseCalc(0); + lhs = ast_factory_.NewCall( + op_id, std::string(op_name), + std::vector{std::move(lhs), std::move(rhs)}); + } + return lhs; +} + +// Parses arithmetic calculation expressions (e.g., `a + b * c`, `x - y % z`). +template +ExprNode PrattParserWorker::ParseCalc(int min_prec) { + ExprNode lhs = ParseUnary(); + while (true) { + TokenType tok = PeekToken().type; + int prec = 0; + std::string_view op_name; + if (tok == TokenType::kAsterisk) { + prec = 2; + op_name = CelOperator::MULTIPLY; + } else if (tok == TokenType::kSlash) { + prec = 2; + op_name = CelOperator::DIVIDE; + } else if (tok == TokenType::kPercent) { + prec = 2; + op_name = CelOperator::MODULO; + } else if (tok == TokenType::kPlus) { + prec = 1; + op_name = CelOperator::ADD; + } else if (tok == TokenType::kMinus) { + prec = 1; + op_name = CelOperator::SUBTRACT; + } else { + break; + } + + if (prec < min_prec) break; + Token op = NextToken(); + int64_t op_id = NextId(op); + ExprNode rhs = ParseCalc(prec + 1); + lhs = ast_factory_.NewCall( + op_id, std::string(op_name), + std::vector{std::move(lhs), std::move(rhs)}); + } + return lhs; +} + +// Parses unary logical NOT and negation expressions (e.g., `!a`, `-b`). +template +ExprNode PrattParserWorker::ParseUnary() { + TokenType tok = PeekToken().type; + if (tok == TokenType::kExclamation || tok == TokenType::kMinus) { + Token op = NextToken(); + int64_t op_id = NextId(op); + std::string_view op_name = (tok == TokenType::kExclamation) + ? CelOperator::LOGICAL_NOT + : CelOperator::NEGATE; + ExprNode operand = ParseUnary(); + return ast_factory_.NewCall(op_id, std::string(op_name), + std::vector{std::move(operand)}); + } + return ParseMember(); +} + +// Parses member calls & indexing (e.g., `a.b`, `a.b(c)`, `a[b]`, `a.?b`). +template +ExprNode PrattParserWorker::ParseMember() { + ExprNode lhs = ParsePrimary(); + while (true) { + TokenType tok = PeekToken().type; + if (tok == TokenType::kDot) { + Token dot_tok = NextToken(); + bool optional = false; + if (PeekToken().type == TokenType::kQuestion) { + NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(dot_tok, "unsupported syntax '.?'"); + } + } + Token id_tok = NextToken(); + if (id_tok.type != TokenType::kIdent && + id_tok.type != TokenType::kReservedWord) { + ReportError(id_tok, "expected identifier after '.'"); + SynchronizeOnDelimiter(); + return lhs; + } + std::string id_text = NormalizeIdent(id_tok, /*allow_quoted=*/!optional); + if (optional) { + int64_t op_id = NextId(dot_tok); + lhs = ast_factory_.NewCall( + op_id, "_?._", + std::vector{ + std::move(lhs), + ast_factory_.NewStringConst(NextId(id_tok), id_text)}); + } else if (PeekToken().type == TokenType::kLeftParen) { + Token lparen = NextToken(); + int64_t call_id = NextId(lparen); + ScopedRecursionGuard guard( + *this, PushScope(ScopeKind::kCallArgs, lparen.start)); + std::vector args = ParseArguments(TokenType::kRightParen); + lhs = ast_factory_.NewMemberCall(call_id, id_text, std::move(lhs), + std::move(args)); + } else { + lhs = ast_factory_.NewSelect(NextId(dot_tok), std::move(lhs), id_text); + } + } else if (tok == TokenType::kLeftBracket) { + Token bracket_tok = NextToken(); + int64_t op_id = NextId(bracket_tok); + bool optional = false; + if (PeekToken().type == TokenType::kQuestion) { + NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(bracket_tok, "unsupported syntax '?'"); + } + } + ScopedRecursionGuard guard( + *this, PushScope(ScopeKind::kList, bracket_tok.start)); + ExprNode index = ParseExpr(); + Expect(TokenType::kRightBracket, "expected ']'"); + lhs = ast_factory_.NewCall( + op_id, optional ? "_[?_]" : CelOperator::INDEX, + std::vector{std::move(lhs), std::move(index)}); + } else if (tok == TokenType::kLeftBrace) { + if (auto struct_name = ExtractStructName(lhs); struct_name.has_value()) { + lhs = ParseStruct(ast_factory_.GetId(lhs), *struct_name); + } else { + break; + } + } else { + break; + } + } + return lhs; +} + +// Parses primary expressions & literals (e.g., `null`, `true`, `x`, +// `has(x.y)`). +template +ExprNode PrattParserWorker::ParsePrimary() { + int paren_count = 0; + while (Match(TokenType::kLeftParen)) { + paren_count++; + } + ExprNode expr; + TokenType tok_type = PeekToken().type; + if (tok_type == TokenType::kNull) { + Token tok = NextToken(); + expr = ast_factory_.NewNullConst(NextId(tok)); + } else if (tok_type == TokenType::kTrue || tok_type == TokenType::kFalse) { + Token tok = NextToken(); + expr = ast_factory_.NewBoolConst(NextId(tok), tok_type == TokenType::kTrue); + } else if (tok_type == TokenType::kInt) { + expr = ParseIntLiteral(); + } else if (tok_type == TokenType::kUint) { + expr = ParseUintLiteral(); + } else if (tok_type == TokenType::kFloat) { + expr = ParseDoubleLiteral(); + } else if (tok_type == TokenType::kString) { + expr = ParseStringLiteral(); + } else if (tok_type == TokenType::kBytes) { + expr = ParseBytesLiteral(); + } else if (tok_type == TokenType::kLeftBracket) { + expr = ParseList(); + } else if (tok_type == TokenType::kLeftBrace) { + expr = ParseMap(); + } else if (tok_type == TokenType::kDot || tok_type == TokenType::kIdent || + tok_type == TokenType::kReservedWord) { + bool leading_dot = false; + Token first_tok = PeekToken(); + if (tok_type == TokenType::kDot) { + NextToken(); + leading_dot = true; + } + Token id_tok = NextToken(); + if (id_tok.type != TokenType::kIdent && + id_tok.type != TokenType::kReservedWord) { + ReportError(id_tok, "expected identifier"); + expr = ast_factory_.NewUnspecified(NextId(id_tok)); + } else { + std::string_view id_text_view = GetTokenText(id_tok); + if (cel::internal::LexisIsReserved(id_text_view)) { + ReportError(id_tok, + absl::StrFormat("reserved identifier: %s", id_text_view)); + } + std::string name = leading_dot ? absl::StrCat(".", id_text_view) + : std::string(id_text_view); + int64_t id = NextId(leading_dot ? first_tok : id_tok); + if (PeekToken().type == TokenType::kLeftParen) { + Token lparen = NextToken(); + ScopedRecursionGuard guard( + *this, PushScope(ScopeKind::kCallArgs, lparen.start)); + std::vector args = ParseArguments(TokenType::kRightParen); + if (!leading_dot && name == CelOperator::HAS && args.size() == 1 && + ast_factory_.IsSelect(args[0]) && + !ast_factory_.IsPresenceTest(args[0])) { + const auto* operand = ast_factory_.GetSelectOperand(args[0]); + std::string field(ast_factory_.GetSelectField(args[0])); + expr = ast_factory_.NewPresenceTest(id, *operand, std::move(field)); + } else { + expr = ast_factory_.NewCall(id, name, std::move(args)); + } + } else { + expr = ast_factory_.NewIdent(id, std::move(name)); + } + } + } else { + Token bad_tok = NextToken(); + ReportError(bad_tok, "unexpected token"); + expr = ast_factory_.NewUnspecified(NextId(bad_tok)); + } + + while (paren_count > 0) { + Expect(TokenType::kRightParen, "expected ')'"); + paren_count--; + } + return expr; +} + +// Parses list creation literals (e.g., `[1, 2, ?3]`). +template +ExprNode PrattParserWorker::ParseList() { + Token open_tok = NextToken(); + int64_t list_id = NextId(open_tok); + ScopedRecursionGuard guard(*this, + PushScope(ScopeKind::kList, open_tok.start)); + ExprNode list_expr = ast_factory_.NewList(list_id); + while (PeekToken().type != TokenType::kRightBracket && !has_errors() && + PeekToken().type != TokenType::kEnd) { + bool optional = false; + if (PeekToken().type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + } + ExprNode elem = ParseExpr(); + ast_factory_.AddListElement(list_expr, std::move(elem), optional); + if (PeekToken().type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + Expect(TokenType::kRightBracket, "expected ']'"); + return list_expr; +} + +// Parses map creation literals (e.g., `{"key": "value", ?"opt_key": 42}`). +template +ExprNode PrattParserWorker::ParseMap() { + Token open_tok = NextToken(); + int64_t map_id = NextId(open_tok); + ScopedRecursionGuard guard(*this, PushScope(ScopeKind::kMap, open_tok.start)); + ExprNode map_expr = ast_factory_.NewMap(map_id); + while (PeekToken().type != TokenType::kRightBrace && !has_errors() && + PeekToken().type != TokenType::kEnd) { + bool optional = false; + Token key_start = PeekToken(); + if (key_start.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + key_start = PeekToken(); + } + ExprNode key = ParseExpr(); + Token colon = PeekToken(); + Expect(TokenType::kColon, "expected ':' in map entry"); + int64_t entry_id = NextId(colon); + ExprNode val = ParseExpr(); + ast_factory_.AddMapEntry(map_expr, entry_id, std::move(key), std::move(val), + optional); + if (PeekToken().type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + Expect(TokenType::kRightBrace, "expected '}'"); + return map_expr; +} + +// Parses struct literas for a type name (e.g., `Msg{field: 10, ?opt: "val"}`). +template +ExprNode PrattParserWorker::ParseStruct(int64_t obj_id, + std::string struct_name) { + Token open_tok = NextToken(); + ScopedRecursionGuard guard(*this, + PushScope(ScopeKind::kStruct, open_tok.start)); + ExprNode struct_expr = ast_factory_.NewStruct(obj_id, std::move(struct_name)); + while (PeekToken().type != TokenType::kRightBrace && !has_errors() && + PeekToken().type != TokenType::kEnd) { + bool optional = false; + if (PeekToken().type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + } + Token field_tok = NextToken(); + if (field_tok.type != TokenType::kIdent && + field_tok.type != TokenType::kReservedWord) { + ReportError(field_tok, "expected struct field name"); + SynchronizeOnDelimiter(); + break; + } + std::string field_name = NormalizeIdent(field_tok, /*allow_quoted=*/true); + Token colon = PeekToken(); + Expect(TokenType::kColon, "expected ':' in struct field"); + int64_t field_id = NextId(colon); + ExprNode val = ParseExpr(); + ast_factory_.AddStructField(struct_expr, field_id, std::move(field_name), + std::move(val), optional); + if (PeekToken().type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + Expect(TokenType::kRightBrace, "expected '}'"); + return struct_expr; +} + +// Parses comma-separated arguments (e.g., `(arg1, arg2)` in call). +template +std::vector PrattParserWorker::ParseArguments( + TokenType close_token) { + std::vector args; + if (PeekToken().type == close_token || PeekToken().type == TokenType::kEnd) { + return args; + } + while (true) { + args.push_back(ParseExpr()); + if (PeekToken().type == TokenType::kComma) { + NextToken(); + if (PeekToken().type == close_token) { + break; + } + continue; + } + break; + } + Expect(close_token, + absl::StrCat("expected ", TokenTypeToString(close_token))); + return args; +} + +// Parses decimal & hexadecimal ints (e.g., `42`, `0x1A`). +template +ExprNode PrattParserWorker::ParseIntLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + int64_t int_val = 0; + if (absl::StartsWith(value, "0x") || absl::StartsWith(value, "0X")) { + if (absl::SimpleHexAtoi(value, &int_val)) { + return ast_factory_.NewIntConst(NextId(tok), int_val); + } + } else if (absl::SimpleAtoi(value, &int_val)) { + return ast_factory_.NewIntConst(NextId(tok), int_val); + } + ReportError(tok, "invalid int literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses unsigned ints (e.g., `42u`, `0x1Au`). +template +ExprNode PrattParserWorker::ParseUintLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + if (!value.empty() && (value.back() == 'u' || value.back() == 'U')) { + value.pop_back(); + } + uint64_t uint_val = 0; + if (absl::StartsWith(value, "0x") || absl::StartsWith(value, "0X")) { + if (absl::SimpleHexAtoi(value, &uint_val)) { + return ast_factory_.NewUintConst(NextId(tok), uint_val); + } + } else if (absl::SimpleAtoi(value, &uint_val)) { + return ast_factory_.NewUintConst(NextId(tok), uint_val); + } + ReportError(tok, "invalid uint literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses floating-point numbers (e.g., `3.14159`, `1e-10`). +template +ExprNode PrattParserWorker::ParseDoubleLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + double double_val = 0.0; + if (absl::SimpleAtod(value, &double_val)) { + return ast_factory_.NewDoubleConst(NextId(tok), double_val); + } + ReportError(tok, "invalid double literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses string literals (e.g., `"hello"`, `'world'`, `"""multi"""`). +template +ExprNode PrattParserWorker::ParseStringLiteral() { + Token tok = NextToken(); + auto status_or_val = cel::internal::ParseStringLiteral(GetTokenText(tok)); + if (!status_or_val.ok()) { + ReportError(tok, status_or_val.status().message()); + return ast_factory_.NewUnspecified(NextId(tok)); + } + return ast_factory_.NewStringConst(NextId(tok), std::move(*status_or_val)); +} + +// Parses byte sequence literals (e.g., `b"hello"`, `b'world'`). +template +ExprNode PrattParserWorker::ParseBytesLiteral() { + Token tok = NextToken(); + auto status_or_val = cel::internal::ParseBytesLiteral(GetTokenText(tok)); + if (!status_or_val.ok()) { + ReportError(tok, status_or_val.status().message()); + return ast_factory_.NewUnspecified(NextId(tok)); + } + return ast_factory_.NewBytesConst(NextId(tok), std::move(*status_or_val)); +} + +// Normalizes regular & quoted identifiers (e.g., `foo`, `` `quoted.ident` ``). +template +std::string PrattParserWorker::NormalizeIdent(const Token& tok, + bool allow_quoted) { + std::string_view text = GetTokenText(tok); + if (text.empty()) return ""; + if (text.front() == '`') { + if (!allow_quoted) { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + if (!options_.enable_quoted_identifiers) { + ReportError(tok, "unsupported syntax '`'"); + } + if (text.size() < 2 || text.back() != '`') { + ReportError(tok, "unterminated quoted identifier"); + return ""; + } + return std::string(text.substr(1, text.size() - 2)); + } + return std::string(text); +} + +// Extracts qualified type names (e.g., `a.b.Msg` from `a.b.Msg{}`). +template +std::optional PrattParserWorker::ExtractStructName( + const ExprNode& expr) { + if (ast_factory_.IsConst(expr)) { + return std::nullopt; + } + if (ast_factory_.IsIdent(expr)) { + std::string name(ast_factory_.GetIdentName(expr)); + EraseId(ast_factory_.GetId(expr)); + return name; + } + if (ast_factory_.IsSelect(expr)) { + if (ast_factory_.IsPresenceTest(expr)) return std::nullopt; + const auto* operand = ast_factory_.GetSelectOperand(expr); + if (operand == nullptr) return std::nullopt; + auto prefix = ExtractStructName(*operand); + if (!prefix) return std::nullopt; + std::string name = + absl::StrCat(*prefix, ".", ast_factory_.GetSelectField(expr)); + EraseId(ast_factory_.GetId(expr)); + return name; + } + return std::nullopt; +} + +// Recursively constructs a balanced binary AST tree for chained associative +// operators (e.g., chains of `+` or `*`). To prevent deep recursion and +// stack overflow during evaluation of expressions like `a + b + c + d`, this +// method splits the operand terms in half at midpoint `(lo + hi + 1) / 2` and +// combines the left and right subtrees with binary call nodes. +template +ExprNode PrattParserWorker::BalancedTree( + std::string_view op, std::vector& terms, + const std::vector& ops, int lo, int hi) { + int mid = (lo + hi + 1) / 2; + std::vector arguments; + arguments.reserve(2); + if (mid == lo) { + arguments.push_back(std::move(terms[mid])); + } else { + arguments.push_back(BalancedTree(op, terms, ops, lo, mid - 1)); + } + if (mid == hi) { + arguments.push_back(std::move(terms[mid + 1])); + } else { + arguments.push_back(BalancedTree(op, terms, ops, mid + 1, hi)); + } + return ast_factory_.NewCall(ops[mid], std::string(op), std::move(arguments)); +} + +// Constructs the AST representation for chained logical operators (e.g., +// `a && b && c` or `a || b || c`). When variadic logical ops are enabled +// (`enable_variadic = true`), it creates a single N-ary function call node +// containing all terms as arguments (e.g., `_&&_(a, b, c)`). Otherwise, it +// delegates to `BalancedTree` to produce a balanced binary tree of logical +// operations `(a && b) && c`. +template +ExprNode PrattParserWorker::BalanceLogical( + std::string_view op, std::vector terms, std::vector ops, + bool enable_variadic) { + if (terms.size() == 1) { + return std::move(terms[0]); + } + if (enable_variadic) { + return ast_factory_.NewCall(ops[0], std::string(op), std::move(terms)); + } + return BalancedTree(op, terms, ops, 0, ops.size() - 1); +} + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ diff --git a/parser/options.h b/parser/options.h index 719bed454..22154e1f9 100644 --- a/parser/options.h +++ b/parser/options.h @@ -25,25 +25,25 @@ struct ParserOptions final { // Limit of the number of error recovery attempts made by the ANTLR parser // when processing an input. This limit, when reached, will halt further // parsing of the expression. - int error_recovery_limit = ::cel_parser_internal::kDefaultErrorRecoveryLimit; + int error_recovery_limit = ::cel::parser_internal::kDefaultErrorRecoveryLimit; // Limit on the amount of recursive parse instructions permitted when building // the abstract syntax tree for the expression. This prevents pathological // inputs from causing stack overflows. - int max_recursion_depth = ::cel_parser_internal::kDefaultMaxRecursionDepth; + int max_recursion_depth = ::cel::parser_internal::kDefaultMaxRecursionDepth; // Limit on the number of codepoints in the input string which the parser will // attempt to parse. int expression_size_codepoint_limit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; // Limit on the number of lookahead tokens to consume when attempting to // recover from an error. int error_recovery_token_lookahead_limit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; // Add macro calls to macro_calls list in source_info. - bool add_macro_calls = ::cel_parser_internal::kDefaultAddMacroCalls; + bool add_macro_calls = ::cel::parser_internal::kDefaultAddMacroCalls; // Enable support for optional syntax. bool enable_optional_syntax = false; @@ -76,20 +76,20 @@ using ParserOptions = ::cel::ParserOptions; ABSL_DEPRECATED("Use ParserOptions().error_recovery_limit instead.") inline constexpr int kDefaultErrorRecoveryLimit = - ::cel_parser_internal::kDefaultErrorRecoveryLimit; + ::cel::parser_internal::kDefaultErrorRecoveryLimit; ABSL_DEPRECATED("Use ParserOptions().max_recursion_depth instead.") inline constexpr int kDefaultMaxRecursionDepth = - ::cel_parser_internal::kDefaultMaxRecursionDepth; + ::cel::parser_internal::kDefaultMaxRecursionDepth; ABSL_DEPRECATED("Use ParserOptions().expression_size_codepoint_limit instead.") inline constexpr int kExpressionSizeCodepointLimit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; ABSL_DEPRECATED( "Use ParserOptions().error_recovery_token_lookahead_limit instead.") inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; ABSL_DEPRECATED("Use ParserOptions().add_macro_calls instead.") inline constexpr bool kDefaultAddMacroCalls = - ::cel_parser_internal::kDefaultAddMacroCalls; + ::cel::parser_internal::kDefaultAddMacroCalls; } // namespace google::api::expr::parser