From e36caf03c93f45b0bafeab6330ae5dff719026dc Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 6 Jul 2026 16:45:41 -0700 Subject: [PATCH] [Pratt Parser] Add AstFactory templatized to support different AST implementations PiperOrigin-RevId: 943561016 --- parser/internal/BUILD | 64 +++ parser/internal/ast_factory.h | 236 ++++++++ parser/internal/ast_factory_interface.h | 86 +++ parser/internal/ast_factory_test.cc | 197 +++++++ parser/internal/lexer.cc | 694 ++++++++++++++++++++++++ parser/internal/lexer.h | 232 ++++++++ parser/internal/lexer_test.cc | 392 +++++++++++++ parser/internal/options.h | 4 +- parser/options.h | 20 +- 9 files changed, 1913 insertions(+), 12 deletions(-) create mode 100644 parser/internal/ast_factory.h create mode 100644 parser/internal/ast_factory_interface.h create mode 100644 parser/internal/ast_factory_test.cc create mode 100644 parser/internal/lexer.cc create mode 100644 parser/internal/lexer.h create mode 100644 parser/internal/lexer_test.cc diff --git a/parser/internal/BUILD b/parser/internal/BUILD index af815588e..d89f17d85 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -13,12 +13,29 @@ # limitations under the License. load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") load("//bazel:antlr.bzl", "antlr_cc_library") package(default_visibility = ["//visibility:public"]) licenses(["notice"]) +cc_library( + name = "ast_factory_interface", + hdrs = ["ast_factory_interface.h"], +) + +cc_library( + name = "ast_factory", + hdrs = ["ast_factory.h"], + deps = [ + ":ast_factory_interface", + "//common:constant", + "//common:expr", + "@com_google_absl//absl/strings:string_view", + ], +) + cc_library( name = "options", hdrs = ["options.h"], @@ -29,3 +46,50 @@ antlr_cc_library( src = "Cel.g4", package = "cel_parser_internal", ) + +cc_library( + name = "lexer", + srcs = ["lexer.cc"], + hdrs = ["lexer.h"], + deps = [ + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_test( + name = "ast_factory_test", + srcs = ["ast_factory_test.cc"], + deps = [ + ":ast_factory", + ":ast_factory_interface", + "//common:expr", + "//internal:testing", + ], +) + +cc_test( + name = "lexer_test", + srcs = ["lexer_test.cc"], + deps = [ + ":lexer", + "//internal:testing", + ], +) + +cc_test( + name = "lexer_fuzzer", + size = "large", + srcs = ["lexer_fuzzer.cc"], + deps = [ + ":lexer", + "//internal:testing", + "//testing/fuzzing:fuzztest", + ], +) diff --git a/parser/internal/ast_factory.h b/parser/internal/ast_factory.h new file mode 100644 index 000000000..96a02bcf8 --- /dev/null +++ b/parser/internal/ast_factory.h @@ -0,0 +1,236 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ + +#include +#include +#include +#include +#include + +#include "common/constant.h" +#include "common/expr.h" +#include "parser/internal/ast_factory_interface.h" + +namespace cel::parser_internal { + +// Explicit specialization of `AstFactoryInterface` for `cel::Expr` AST nodes. +template <> +class AstFactoryInterface { + public: + AstFactoryInterface() = default; + + AstFactoryInterface(const AstFactoryInterface&) = delete; + AstFactoryInterface(AstFactoryInterface&&) = delete; + AstFactoryInterface& operator=(const AstFactoryInterface&) = delete; + AstFactoryInterface& operator=(AstFactoryInterface&&) = delete; + + ~AstFactoryInterface() = default; + + // Node inspection and encapsulation API + int64_t GetId(const cel::Expr& expr) const { return expr.id(); } + + bool IsEmpty(const cel::Expr& expr) const { return expr.id() == 0; } + + bool IsConst(const cel::Expr& expr) const { return expr.has_const_expr(); } + + bool IsIdent(const cel::Expr& expr) const { return expr.has_ident_expr(); } + + std::string_view GetIdentName(const cel::Expr& expr) const { + return expr.has_ident_expr() ? std::string_view(expr.ident_expr().name()) + : std::string_view(); + } + + bool IsSelect(const cel::Expr& expr) const { return expr.has_select_expr(); } + + bool IsPresenceTest(const cel::Expr& expr) const { + return expr.has_select_expr() && expr.select_expr().test_only(); + } + + const cel::Expr* GetSelectOperand(const cel::Expr& expr) const { + return expr.has_select_expr() ? &expr.select_expr().operand() : nullptr; + } + + std::string_view GetSelectField(const cel::Expr& expr) const { + return expr.has_select_expr() ? std::string_view(expr.select_expr().field()) + : std::string_view(); + } + + // Node creation API + cel::Expr NewUnspecified(int64_t id) { + cel::Expr expr; + expr.set_id(id); + return expr; + } + + cel::Expr NewNullConst(int64_t id) { + cel::Constant constant; + constant.set_null_value(); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewBoolConst(int64_t id, bool value) { + cel::Constant constant; + constant.set_bool_value(value); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewIntConst(int64_t id, int64_t value) { + cel::Constant constant; + constant.set_int_value(value); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewUintConst(int64_t id, uint64_t value) { + cel::Constant constant; + constant.set_uint_value(value); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewDoubleConst(int64_t id, double value) { + cel::Constant constant; + constant.set_double_value(value); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewBytesConst(int64_t id, std::string value) { + cel::Constant constant; + constant.set_bytes_value(std::move(value)); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewStringConst(int64_t id, std::string value) { + cel::Constant constant; + constant.set_string_value(std::move(value)); + return NewConst(id, std::move(constant)); + } + + cel::Expr NewIdent(int64_t id, std::string name) { + cel::Expr expr; + expr.set_id(id); + cel::IdentExpr& ident_expr = expr.mutable_ident_expr(); + ident_expr.set_name(std::move(name)); + return expr; + } + + cel::Expr NewSelect(int64_t id, cel::Expr operand, std::string field) { + cel::Expr expr; + expr.set_id(id); + cel::SelectExpr& select_expr = expr.mutable_select_expr(); + select_expr.set_operand(std::move(operand)); + select_expr.set_field(std::move(field)); + select_expr.set_test_only(false); + return expr; + } + + cel::Expr NewPresenceTest(int64_t id, cel::Expr operand, std::string field) { + cel::Expr expr; + expr.set_id(id); + cel::SelectExpr& select_expr = expr.mutable_select_expr(); + select_expr.set_operand(std::move(operand)); + select_expr.set_field(std::move(field)); + select_expr.set_test_only(true); + return expr; + } + + cel::Expr NewCall(int64_t id, std::string function, + std::vector args) { + cel::Expr expr; + expr.set_id(id); + cel::CallExpr& call_expr = expr.mutable_call_expr(); + call_expr.set_function(std::move(function)); + call_expr.set_args(std::move(args)); + return expr; + } + + cel::Expr NewMemberCall(int64_t id, std::string function, cel::Expr target, + std::vector args) { + cel::Expr expr; + expr.set_id(id); + cel::CallExpr& call_expr = expr.mutable_call_expr(); + call_expr.set_function(std::move(function)); + call_expr.set_target(std::move(target)); + call_expr.set_args(std::move(args)); + return expr; + } + + cel::Expr NewList(int64_t id) { + cel::Expr expr; + expr.set_id(id); + expr.mutable_list_expr(); + return expr; + } + + void AddListElement(cel::Expr& list_expr, cel::Expr element, + bool optional = false) { + cel::ListExpr& list_val = list_expr.mutable_list_expr(); + cel::ListExprElement expr_element; + expr_element.set_expr(std::move(element)); + expr_element.set_optional(optional); + list_val.mutable_elements().push_back(std::move(expr_element)); + } + + cel::Expr NewStruct(int64_t id, std::string name) { + cel::Expr expr; + expr.set_id(id); + cel::StructExpr& struct_expr = expr.mutable_struct_expr(); + struct_expr.set_name(std::move(name)); + return expr; + } + + void AddStructField(cel::Expr& struct_expr, int64_t id, std::string name, + cel::Expr value, bool optional = false) { + cel::StructExpr& struct_val = struct_expr.mutable_struct_expr(); + cel::StructExprField field; + field.set_id(id); + field.set_name(std::move(name)); + field.set_value(std::move(value)); + field.set_optional(optional); + struct_val.mutable_fields().push_back(std::move(field)); + } + + cel::Expr NewMap(int64_t id) { + cel::Expr expr; + expr.set_id(id); + expr.mutable_map_expr(); + return expr; + } + + void AddMapEntry(cel::Expr& map_expr, int64_t id, cel::Expr key, + cel::Expr value, bool optional = false) { + cel::MapExpr& map_val = map_expr.mutable_map_expr(); + cel::MapExprEntry entry; + entry.set_id(id); + entry.set_key(std::move(key)); + entry.set_value(std::move(value)); + entry.set_optional(optional); + map_val.mutable_entries().push_back(std::move(entry)); + } + + private: + cel::Expr NewConst(int64_t id, cel::Constant value) { + cel::Expr expr; + expr.set_id(id); + expr.mutable_const_expr() = std::move(value); + return expr; + } +}; + +using AstFactory = AstFactoryInterface; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ diff --git a/parser/internal/ast_factory_interface.h b/parser/internal/ast_factory_interface.h new file mode 100644 index 000000000..7052c0f87 --- /dev/null +++ b/parser/internal/ast_factory_interface.h @@ -0,0 +1,86 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ + +#include +#include +#include +#include + +namespace cel::parser_internal { + +// Interface for decoupling parser logic from the underlying AST node +// data structures. +// +// By parameterizing the parser and factory on `ExprNode`, alternative AST node +// representations (such as `cel::Expr`) can be constructed without modifying +// parser rules. +// +// To implement AST construction using an alternative AST structure: +// 1. Define or specify your custom node type `MyNode`. +// 2. Implement a concrete factory specialization `AstFactoryInterface` +// that provides inspection (`GetId`, `IsSelect`, etc.) and creation +// (`NewCall`, `AddListElement`, etc.) operations for `MyNode`. +// 3. Instantiate the parser worker with your node type: +// `PrattParserWorker`. +template +class AstFactoryInterface { + public: + AstFactoryInterface() = default; + AstFactoryInterface(const AstFactoryInterface&) = delete; + AstFactoryInterface(AstFactoryInterface&&) = delete; + AstFactoryInterface& operator=(const AstFactoryInterface&) = delete; + AstFactoryInterface& operator=(AstFactoryInterface&&) = delete; + + int64_t GetId(const ExprNode& expr) const; + bool IsEmpty(const ExprNode& expr) const; + bool IsConst(const ExprNode& expr) const; + bool IsIdent(const ExprNode& expr) const; + std::string_view GetIdentName(const ExprNode& expr) const; + bool IsSelect(const ExprNode& expr) const; + bool IsPresenceTest(const ExprNode& expr) const; + const ExprNode* GetSelectOperand(const ExprNode& expr) const; + std::string_view GetSelectField(const ExprNode& expr) const; + + ExprNode NewUnspecified(int64_t id); + ExprNode NewNullConst(int64_t id); + ExprNode NewBoolConst(int64_t id, bool value); + ExprNode NewIntConst(int64_t id, int64_t value); + ExprNode NewUintConst(int64_t id, uint64_t value); + ExprNode NewDoubleConst(int64_t id, double value); + ExprNode NewBytesConst(int64_t id, std::string value); + ExprNode NewStringConst(int64_t id, std::string value); + ExprNode NewIdent(int64_t id, std::string name); + ExprNode NewSelect(int64_t id, ExprNode operand, std::string field); + ExprNode NewPresenceTest(int64_t id, ExprNode operand, std::string field); + ExprNode NewCall(int64_t id, std::string function, + std::vector args); + ExprNode NewMemberCall(int64_t id, std::string function, ExprNode target, + std::vector args); + ExprNode NewList(int64_t id); + void AddListElement(ExprNode& list_expr, ExprNode element, + bool optional = false); + ExprNode NewStruct(int64_t id, std::string name); + void AddStructField(ExprNode& struct_expr, int64_t id, std::string name, + ExprNode value, bool optional = false); + ExprNode NewMap(int64_t id); + void AddMapEntry(ExprNode& map_expr, int64_t id, ExprNode key, ExprNode value, + bool optional = false); +}; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ diff --git a/parser/internal/ast_factory_test.cc b/parser/internal/ast_factory_test.cc new file mode 100644 index 000000000..706f13e88 --- /dev/null +++ b/parser/internal/ast_factory_test.cc @@ -0,0 +1,197 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/ast_factory.h" + +#include +#include +#include + +#include "common/expr.h" +#include "internal/testing.h" + +namespace cel::parser_internal { +namespace { + +TEST(AstFactoryInterfaceTest, AstFactoryUnspecified) { + AstFactory factory; + cel::Expr expr = factory.NewUnspecified(1); + EXPECT_EQ(factory.GetId(expr), 1); + EXPECT_FALSE(factory.IsEmpty(expr)); + EXPECT_FALSE(factory.IsConst(expr)); + EXPECT_FALSE(factory.IsIdent(expr)); + EXPECT_FALSE(factory.IsSelect(expr)); + + cel::Expr empty_expr = factory.NewUnspecified(0); + EXPECT_TRUE(factory.IsEmpty(empty_expr)); +} + +TEST(AstFactoryInterfaceTest, AstFactoryConstNodes) { + AstFactory factory; + + cel::Expr null_expr = factory.NewNullConst(10); + EXPECT_EQ(factory.GetId(null_expr), 10); + EXPECT_TRUE(factory.IsConst(null_expr)); + ASSERT_TRUE(null_expr.has_const_expr()); + EXPECT_TRUE(null_expr.const_expr().has_null_value()); + + cel::Expr bool_expr = factory.NewBoolConst(11, true); + EXPECT_EQ(factory.GetId(bool_expr), 11); + EXPECT_TRUE(factory.IsConst(bool_expr)); + ASSERT_TRUE(bool_expr.has_const_expr()); + EXPECT_TRUE(bool_expr.const_expr().bool_value()); + + cel::Expr int_expr = factory.NewIntConst(12, -42); + EXPECT_EQ(factory.GetId(int_expr), 12); + EXPECT_TRUE(factory.IsConst(int_expr)); + ASSERT_TRUE(int_expr.has_const_expr()); + EXPECT_EQ(int_expr.const_expr().int_value(), -42); + + cel::Expr uint_expr = factory.NewUintConst(13, 100u); + EXPECT_EQ(factory.GetId(uint_expr), 13); + EXPECT_TRUE(factory.IsConst(uint_expr)); + ASSERT_TRUE(uint_expr.has_const_expr()); + EXPECT_EQ(uint_expr.const_expr().uint_value(), 100u); + + cel::Expr double_expr = factory.NewDoubleConst(14, 3.14159); + EXPECT_EQ(factory.GetId(double_expr), 14); + EXPECT_TRUE(factory.IsConst(double_expr)); + ASSERT_TRUE(double_expr.has_const_expr()); + EXPECT_DOUBLE_EQ(double_expr.const_expr().double_value(), 3.14159); + + cel::Expr bytes_expr = factory.NewBytesConst(15, "bytes_val"); + EXPECT_EQ(factory.GetId(bytes_expr), 15); + EXPECT_TRUE(factory.IsConst(bytes_expr)); + ASSERT_TRUE(bytes_expr.has_const_expr()); + EXPECT_EQ(bytes_expr.const_expr().bytes_value(), "bytes_val"); + + cel::Expr string_expr = factory.NewStringConst(16, "string_val"); + EXPECT_EQ(factory.GetId(string_expr), 16); + EXPECT_TRUE(factory.IsConst(string_expr)); + ASSERT_TRUE(string_expr.has_const_expr()); + EXPECT_EQ(string_expr.const_expr().string_value(), "string_val"); +} + +TEST(AstFactoryInterfaceTest, AstFactoryIdentAndSelect) { + AstFactory factory; + + cel::Expr ident_expr = factory.NewIdent(20, "foo"); + EXPECT_EQ(factory.GetId(ident_expr), 20); + EXPECT_TRUE(factory.IsIdent(ident_expr)); + EXPECT_EQ(factory.GetIdentName(ident_expr), "foo"); + + cel::Expr select_expr = + factory.NewSelect(21, factory.NewIdent(20, "foo"), "bar"); + EXPECT_EQ(factory.GetId(select_expr), 21); + EXPECT_TRUE(factory.IsSelect(select_expr)); + EXPECT_FALSE(factory.IsPresenceTest(select_expr)); + EXPECT_EQ(factory.GetSelectField(select_expr), "bar"); + ASSERT_NE(factory.GetSelectOperand(select_expr), nullptr); + EXPECT_EQ(factory.GetIdentName(*factory.GetSelectOperand(select_expr)), + "foo"); + + cel::Expr presence_expr = + factory.NewPresenceTest(22, factory.NewIdent(20, "foo"), "bar"); + EXPECT_EQ(factory.GetId(presence_expr), 22); + EXPECT_TRUE(factory.IsSelect(presence_expr)); + EXPECT_TRUE(factory.IsPresenceTest(presence_expr)); + EXPECT_EQ(factory.GetSelectField(presence_expr), "bar"); +} + +TEST(AstFactoryInterfaceTest, AstFactoryCalls) { + AstFactory factory; + + std::vector call_args; + call_args.push_back(factory.NewIntConst(30, 1)); + call_args.push_back(factory.NewIntConst(31, 2)); + cel::Expr call_expr = factory.NewCall(32, "_+_", std::move(call_args)); + EXPECT_EQ(factory.GetId(call_expr), 32); + ASSERT_TRUE(call_expr.has_call_expr()); + EXPECT_EQ(call_expr.call_expr().function(), "_+_"); + EXPECT_FALSE(call_expr.call_expr().has_target()); + EXPECT_EQ(call_expr.call_expr().args().size(), 2); + + std::vector member_args; + member_args.push_back(factory.NewStringConst(33, "suffix")); + cel::Expr member_call_expr = factory.NewMemberCall( + 34, "endsWith", factory.NewIdent(35, "str_var"), std::move(member_args)); + EXPECT_EQ(factory.GetId(member_call_expr), 34); + ASSERT_TRUE(member_call_expr.has_call_expr()); + EXPECT_EQ(member_call_expr.call_expr().function(), "endsWith"); + EXPECT_TRUE(member_call_expr.call_expr().has_target()); + EXPECT_EQ(member_call_expr.call_expr().target().ident_expr().name(), + "str_var"); + EXPECT_EQ(member_call_expr.call_expr().args().size(), 1); +} + +TEST(AstFactoryInterfaceTest, AstFactoryList) { + AstFactory factory; + + cel::Expr list_expr = factory.NewList(42); + factory.AddListElement(list_expr, factory.NewIntConst(40, 1), false); + factory.AddListElement(list_expr, factory.NewIntConst(41, 2), true); + EXPECT_EQ(factory.GetId(list_expr), 42); + ASSERT_TRUE(list_expr.has_list_expr()); + ASSERT_EQ(list_expr.list_expr().elements().size(), 2); + EXPECT_FALSE(list_expr.list_expr().elements()[0].optional()); + EXPECT_EQ(list_expr.list_expr().elements()[0].expr().const_expr().int_value(), + 1); + EXPECT_TRUE(list_expr.list_expr().elements()[1].optional()); + EXPECT_EQ(list_expr.list_expr().elements()[1].expr().const_expr().int_value(), + 2); +} + +TEST(AstFactoryInterfaceTest, AstFactoryStruct) { + AstFactory factory; + + cel::Expr struct_expr = factory.NewStruct(54, "MyMessage"); + factory.AddStructField(struct_expr, 50, "field1", + factory.NewIntConst(51, 100), false); + factory.AddStructField(struct_expr, 52, "field2", + factory.NewIntConst(53, 200), true); + EXPECT_EQ(factory.GetId(struct_expr), 54); + ASSERT_TRUE(struct_expr.has_struct_expr()); + EXPECT_EQ(struct_expr.struct_expr().name(), "MyMessage"); + ASSERT_EQ(struct_expr.struct_expr().fields().size(), 2); + EXPECT_EQ(struct_expr.struct_expr().fields()[0].id(), 50); + EXPECT_EQ(struct_expr.struct_expr().fields()[0].name(), "field1"); + EXPECT_FALSE(struct_expr.struct_expr().fields()[0].optional()); + EXPECT_EQ(struct_expr.struct_expr().fields()[1].id(), 52); + EXPECT_EQ(struct_expr.struct_expr().fields()[1].name(), "field2"); + EXPECT_TRUE(struct_expr.struct_expr().fields()[1].optional()); +} + +TEST(AstFactoryInterfaceTest, AstFactoryMap) { + AstFactory factory; + + cel::Expr map_expr = factory.NewMap(66); + factory.AddMapEntry(map_expr, 60, factory.NewStringConst(61, "key1"), + factory.NewIntConst(62, 10), false); + factory.AddMapEntry(map_expr, 63, factory.NewStringConst(64, "key2"), + factory.NewIntConst(65, 20), true); + EXPECT_EQ(factory.GetId(map_expr), 66); + ASSERT_TRUE(map_expr.has_map_expr()); + ASSERT_EQ(map_expr.map_expr().entries().size(), 2); + EXPECT_EQ(map_expr.map_expr().entries()[0].id(), 60); + EXPECT_EQ(map_expr.map_expr().entries()[0].key().const_expr().string_value(), + "key1"); + EXPECT_EQ(map_expr.map_expr().entries()[0].value().const_expr().int_value(), + 10); + EXPECT_FALSE(map_expr.map_expr().entries()[0].optional()); + EXPECT_EQ(map_expr.map_expr().entries()[1].id(), 63); + EXPECT_TRUE(map_expr.map_expr().entries()[1].optional()); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc new file mode 100644 index 000000000..7b8da71d1 --- /dev/null +++ b/parser/internal/lexer.cc @@ -0,0 +1,694 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" +#include "absl/container/flat_hash_map.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace cel::parser_internal { + +namespace { + +[[nodiscard]] bool IsIdentTrailing(unsigned char c) { + return absl::ascii_isdigit(c) || absl::ascii_isalpha(c) || c == '_'; +} + +[[nodiscard]] bool IsPlusOrMinus(unsigned char c) { + return c == '+' || c == '-'; +} + +[[nodiscard]] const absl::flat_hash_map& +Keywords() { + static const absl::NoDestructor< + absl::flat_hash_map> + kKeywords({ + {"false", TokenType::kFalse}, + {"true", TokenType::kTrue}, + {"null", TokenType::kNull}, + {"in", TokenType::kIn}, + {"as", TokenType::kReservedWord}, + {"break", TokenType::kReservedWord}, + {"const", TokenType::kReservedWord}, + {"continue", TokenType::kReservedWord}, + {"else", TokenType::kReservedWord}, + {"for", TokenType::kReservedWord}, + {"function", TokenType::kReservedWord}, + {"if", TokenType::kReservedWord}, + {"import", TokenType::kReservedWord}, + {"let", TokenType::kReservedWord}, + {"loop", TokenType::kReservedWord}, + {"package", TokenType::kReservedWord}, + {"namespace", TokenType::kReservedWord}, + {"return", TokenType::kReservedWord}, + {"var", TokenType::kReservedWord}, + {"void", TokenType::kReservedWord}, + {"while", TokenType::kReservedWord}, + }); + return *kKeywords; +} + +} // namespace + +std::string_view TokenTypeToString(TokenType type) { + switch (type) { + case TokenType::kError: + return "error"; + case TokenType::kEnd: + return "end"; + case TokenType::kWhitespace: + return "whitespace"; + case TokenType::kComment: + return "comment"; + case TokenType::kNull: + return "null"; + case TokenType::kFalse: + return "false"; + case TokenType::kTrue: + return "true"; + case TokenType::kIn: + return "in"; + case TokenType::kReservedWord: + return "reserved_word"; + case TokenType::kInt: + return "int"; + case TokenType::kUint: + return "uint"; + case TokenType::kFloat: + return "float"; + case TokenType::kString: + return "string"; + case TokenType::kBytes: + return "bytes"; + case TokenType::kIdent: + return "ident"; + case TokenType::kLeftBracket: + return "["; + case TokenType::kRightBracket: + return "]"; + case TokenType::kLeftBrace: + return "{"; + case TokenType::kRightBrace: + return "}"; + case TokenType::kLeftParen: + return "("; + case TokenType::kRightParen: + return ")"; + case TokenType::kDot: + return "."; + case TokenType::kComma: + return ","; + case TokenType::kMinus: + return "-"; + case TokenType::kPlus: + return "+"; + case TokenType::kAsterisk: + return "*"; + case TokenType::kSlash: + return "/"; + case TokenType::kPercent: + return "%"; + case TokenType::kQuestion: + return "?"; + case TokenType::kColon: + return ":"; + case TokenType::kExclamation: + return "!"; + case TokenType::kEqual: + return "="; + case TokenType::kEqualEqual: + return "=="; + case TokenType::kExclamationEqual: + return "!="; + case TokenType::kLess: + return "<"; + case TokenType::kLessEqual: + return "<="; + case TokenType::kGreater: + return ">"; + case TokenType::kGreaterEqual: + return ">="; + case TokenType::kLogicalAnd: + return "&&"; + case TokenType::kLogicalOr: + return "||"; + default: + return ""; + } +} + +Token Lexer::Lex() { + if (ABSL_PREDICT_FALSE(at_error_)) { + return MakeToken(TokenType::kError, error_.start, error_.end); + } + int32_t start = GetPosition(); + if (ABSL_PREDICT_FALSE(text_ == text_end_)) { + at_end_ = true; + done_ = true; + return MakeToken(TokenType::kEnd, start, start); + } + char c = *text_; + switch (c) { + case '\v': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\n': + ABSL_FALLTHROUGH_INTENDED; + case ' ': { + ConsumeWhitespace(); + return MakeToken(TokenType::kWhitespace, start, GetPosition()); + } + case '.': { + // Check if this looks like a double literal and break if it is, we handle + // that after this switch. + if (text_ + 1 < text_end_ && absl::ascii_isdigit(text_[1])) { + break; + } + Advance(1); + return MakeToken(TokenType::kDot, start, GetPosition()); + } + case ',': { + Advance(1); + return MakeToken(TokenType::kComma, start, GetPosition()); + } + case '!': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kExclamationEqual, start, GetPosition()); + } + return MakeToken(TokenType::kExclamation, start, GetPosition()); + } + case '?': { + Advance(1); + return MakeToken(TokenType::kQuestion, start, GetPosition()); + } + case '(': { + Advance(1); + return MakeToken(TokenType::kLeftParen, start, GetPosition()); + } + case ')': { + Advance(1); + return MakeToken(TokenType::kRightParen, start, GetPosition()); + } + case '{': { + Advance(1); + return MakeToken(TokenType::kLeftBrace, start, GetPosition()); + } + case '}': { + Advance(1); + return MakeToken(TokenType::kRightBrace, start, GetPosition()); + } + case '[': { + Advance(1); + return MakeToken(TokenType::kLeftBracket, start, GetPosition()); + } + case ']': { + Advance(1); + return MakeToken(TokenType::kRightBracket, start, GetPosition()); + } + case '=': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kEqualEqual, start, GetPosition()); + } + return MakeToken(TokenType::kEqual, start, GetPosition()); + } + case '<': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kLessEqual, start, GetPosition()); + } + return MakeToken(TokenType::kLess, start, GetPosition()); + } + case '>': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kGreaterEqual, start, GetPosition()); + } + return MakeToken(TokenType::kGreater, start, GetPosition()); + } + case ':': { + Advance(1); + return MakeToken(TokenType::kColon, start, GetPosition()); + } + case '%': { + Advance(1); + return MakeToken(TokenType::kPercent, start, GetPosition()); + } + case '+': { + Advance(1); + return MakeToken(TokenType::kPlus, start, GetPosition()); + } + case '-': { + Advance(1); + return MakeToken(TokenType::kMinus, start, GetPosition()); + } + case '*': { + Advance(1); + return MakeToken(TokenType::kAsterisk, start, GetPosition()); + } + case '/': { + Advance(1); + if (Consume('/')) { + ConsumeLine(); + return MakeToken(TokenType::kComment, start, GetPosition()); + } + return MakeToken(TokenType::kSlash, start, GetPosition()); + } + case '&': { + Advance(1); + if (Consume('&')) { + return MakeToken(TokenType::kLogicalAnd, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '&', expected '&&'"); + } + case '|': { + Advance(1); + if (Consume('|')) { + return MakeToken(TokenType::kLogicalOr, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '|', expected '||'"); + } + case '`': { + // Report the entire match including the backticks because the parser + // forbids quoted identifiers in certain places and needs to be able to + // detect them. + Advance(1); + if (!ConsumeUntilAfter('`')) { + return SetError(start, GetPosition(), "unterminated quoted identifier"); + } + return MakeToken(TokenType::kIdent, start, GetPosition()); + } + case '\'': { + Advance(1); + if (ConsumeString("''")) { + if (!ConsumeUntilAfterString("'''")) { + return SetError(start, GetPosition(), "unterminated string literal"); + } + return MakeToken(TokenType::kString, start, GetPosition()); + } + if (!ConsumeUntilAfterUnescaped('\'')) { + return SetError(start, GetPosition(), "unterminated string literal"); + } + return MakeToken(TokenType::kString, start, GetPosition()); + } + case '"': { + Advance(1); + if (ConsumeString("\"\"")) { + if (!ConsumeUntilAfterString("\"\"\"")) { + return SetError(start, GetPosition(), "unterminated string literal"); + } + return MakeToken(TokenType::kString, start, GetPosition()); + } + if (!ConsumeUntilAfterUnescaped('"')) { + return SetError(start, GetPosition(), "unterminated string literal"); + } + return MakeToken(TokenType::kString, start, GetPosition()); + } + default: + break; + } + if (c == 'r' || c == 'R' || c == 'b' || c == 'B') { + bool is_bytes = (c == 'b' || c == 'B'); + size_t lookahead = 1; + if (text_ + 1 < text_end_) { + char c2 = text_[1]; + if ((is_bytes && (c2 == 'r' || c2 == 'R')) || + (!is_bytes && (c2 == 'b' || c2 == 'B'))) { + is_bytes = true; + lookahead = 2; + } + } + if (text_ + lookahead < text_end_) { + char quote = text_[lookahead]; + if (quote == '"' || quote == '\'') { + Advance(lookahead + 1); + std::string tripe_quote(3, quote); + if (ConsumeString(std::string_view(tripe_quote.data(), 2))) { + if (!ConsumeUntilAfterString(tripe_quote)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, + start, GetPosition()); + } + if (!ConsumeUntilAfterUnescaped(quote)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, + start, GetPosition()); + } + } + } + if (c == '.' || absl::ascii_isdigit(c)) { + bool floating_point = false; + if (c == '.') { + floating_point = true; + Advance(1); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after decimal separator"); + } + } else { + Advance(1); + if (c == '0') { + if (ConsumeIgnoreCase('x')) { + if (!ConsumeHexDigits()) { + return SetError( + start, GetPosition(), + "integral literal missing digits after hexadecimal separator"); + } + auto token_type = ConsumeIntegralSuffix(); + if (ConsumeIf(IsIdentTrailing)) { + return SetError( + start, GetPosition(), + absl::StrCat(TokenTypeToString(token_type), + " literal has unexpected trailing characters")); + } + return MakeToken(token_type, start, GetPosition()); + } + } + static_cast(ConsumeDigits()); + if (text_ < text_end_ && *text_ == '.' && text_ + 1 < text_end_ && + absl::ascii_isdigit(text_[1])) { + floating_point = true; + Advance(1); + static_cast(ConsumeDigits()); + } + } + if (ConsumeIgnoreCase('e')) { + floating_point = true; + static_cast(ConsumeIf(IsPlusOrMinus)); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after exponent separator"); + } + } + auto token_type = + floating_point ? TokenType::kFloat : ConsumeIntegralSuffix(); + if (ConsumeIf(IsIdentTrailing)) { + return SetError( + start, GetPosition(), + absl::StrCat(TokenTypeToString(token_type), + " literal has unexpected trailing characters")); + } + return MakeToken(token_type, start, GetPosition()); + } + if (c == '_' || absl::ascii_isalpha(c)) { + const char* text = text_; + ConsumeIdentTrailing(); + int32_t end = GetPosition(); + std::string_view word(text, static_cast(end - start)); + const auto& keywords = Keywords(); + if (auto it = keywords.find(word); it != keywords.end()) { + return MakeToken(it->second, start, end); + } + return MakeToken(TokenType::kIdent, start, end); + } + Advance(1); + return SetError(start, GetPosition(), "unexpected character"); +} + +bool Lexer::ConsumeUntilAfter(char c) { + ABSL_DCHECK_NE(c, '\n'); + auto pos = GetRemainingText().find(c); + if (pos == std::string_view::npos) { + AdvanceProcessingNewLines(text_end_); + return false; + } + AdvanceProcessingNewLines(pos + 1); + return true; +} + +bool Lexer::ConsumeUntilAfterString(std::string_view s) { + ABSL_DCHECK(!absl::StrContains(s, '\n')); + auto pos = GetRemainingText().find(s); + if (pos == std::string_view::npos) { + AdvanceProcessingNewLines(text_end_); + return false; + } + AdvanceProcessingNewLines(pos + s.size()); + return true; +} + +bool Lexer::ConsumeUntilAfterUnescaped(char c) { + ABSL_DCHECK_NE(c, '\n'); + ABSL_DCHECK_NE(c, '\\'); + const char* text = text_; + bool escaped = false; + while (text != text_end_) { + std::string_view chunk = + std::string_view(text, static_cast(text_end_ - text)); + for (size_t i = 0; i < chunk.size(); ++i) { + char cc = chunk[i]; + if (cc == '\\') { + escaped = !escaped; + } else { + if (cc == c && !escaped) { + AdvanceProcessingNewLines(static_cast(text - text_) + i + 1); + return true; + } + escaped = false; + } + } + text += chunk.size(); + } + AdvanceProcessingNewLines(text_end_); + return false; +} + +void Lexer::ConsumeIdentTrailing() { + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + for (size_t i = 0; i < chunk.size(); ++i) { + char c = chunk[i]; + if (!IsIdentTrailing(c)) { + Advance(i); + return; + } + } + Advance(chunk.size()); + } +} + +bool Lexer::MatchString(std::string_view s) const { + return absl::StartsWith(GetRemainingText(), s); +} + +bool Lexer::MatchStringIgnoreCase(std::string_view s) const { + return absl::StartsWithIgnoreCase(GetRemainingText(), s); +} + +std::optional Lexer::MatchIf( + absl::FunctionRef predicate) const { + if (text_ != text_end_) { + char c = *text_; + if (predicate(c)) { + return c; + } + } + return std::nullopt; +} + +void Lexer::ConsumeLine() { + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + auto pos = chunk.find('\n'); + if (pos != std::string_view::npos) { + Advance(pos + 1); + if (line_offsets_ != nullptr) { + line_offsets_->push_back(GetPosition()); + } + break; + } + Advance(chunk.size()); + } +} + +void Lexer::ConsumeWhitespace() { + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + size_t i = 0; + next_char: + while (i < chunk.size()) { + char c = chunk[i]; + switch (c) { + case '\n': + if (line_offsets_ != nullptr) { + line_offsets_->push_back(GetPosition() + static_cast(i) + + 1); + } + ABSL_FALLTHROUGH_INTENDED; + case ' ': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\v': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + ++i; + goto next_char; + default: + Advance(i); + return; + } + } + Advance(chunk.size()); + } +} + +bool Lexer::Consume(char c) { + ABSL_DCHECK_NE(c, '\n'); + if (Match(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeIgnoreCase(char c) { + ABSL_DCHECK_NE(c, '\n'); + if (MatchIgnoreCase(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeString(std::string_view s) { + ABSL_DCHECK(!absl::StrContains(s, '\n')); + if (MatchString(s)) { + Advance(s.size()); + return true; + } + return false; +} + +bool Lexer::ConsumeStringIgnoreCase(std::string_view s) { + ABSL_DCHECK(!absl::StrContains(s, '\n')); + if (MatchStringIgnoreCase(s)) { + Advance(s.size()); + return true; + } + return false; +} + +std::optional Lexer::ConsumeIf( + absl::FunctionRef predicate) { + std::optional match = MatchIf(predicate); + if (match.has_value()) { + ABSL_DCHECK_NE(*match, '\n'); + Advance(1); + } + return match; +} + +bool Lexer::ConsumeDigits() { + bool advanced = false; + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + for (size_t i = 0; i < chunk.size(); ++i) { + if (!absl::ascii_isdigit(chunk[i])) { + if (i != 0) { + Advance(i); + return true; + } + return advanced; + } + } + Advance(chunk.size()); + advanced = true; + } + return advanced; +} + +bool Lexer::ConsumeHexDigits() { + bool advanced = false; + while (text_ != text_end_) { + std::string_view chunk = GetRemainingText(); + for (size_t i = 0; i < chunk.size(); ++i) { + if (!absl::ascii_isxdigit(chunk[i])) { + if (i != 0) { + Advance(i); + return true; + } + return advanced; + } + } + Advance(chunk.size()); + advanced = true; + } + return advanced; +} + +TokenType Lexer::ConsumeIntegralSuffix() { + if (ConsumeIgnoreCase('u')) { + return TokenType::kUint; + } + return TokenType::kInt; +} + +void Lexer::AdvanceProcessingNewLines(size_t n) { + while (n > 0) { + std::string_view chunk = GetRemainingText(); + chunk = chunk.substr(0, std::min(chunk.size(), n)); + std::string_view::size_type pos = 0; + while (pos < chunk.size()) { + std::string_view::size_type npos = chunk.find('\n', pos); + if (npos == std::string_view::npos) { + break; + } + ++npos; + if (line_offsets_ != nullptr) { + line_offsets_->push_back(GetPosition() + static_cast(npos)); + } + pos = npos; + } + n -= chunk.size(); + Advance(chunk.size()); + } +} + +void Lexer::AdvanceProcessingNewLines(const char* end) { + ABSL_DCHECK_LE(end, text_end_); + ABSL_DCHECK_GE(end, text_); + AdvanceProcessingNewLines(static_cast(end - text_)); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h new file mode 100644 index 000000000..5bf6ef24d --- /dev/null +++ b/parser/internal/lexer.h @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" + +namespace cel::parser_internal { + +enum class TokenType { + kError = 0, + kEnd, + kWhitespace, + kComment, + + // Keywords + kNull, + kFalse, + kTrue, + kIn, + kReservedWord, + + // Literals + kInt, + kUint, + kFloat, + kString, + kBytes, + + // Identifiers + kIdent, + + // Delimiters + kLeftBracket, // [ + kRightBracket, // ] + kLeftBrace, // { + kRightBrace, // } + kLeftParen, // ( + kRightParen, // ) + + // Operators + kDot, // . + kComma, // , + kMinus, // - + kPlus, // + + kAsterisk, // * + kSlash, // / + kPercent, // % + kQuestion, // ? + kColon, // : + kExclamation, // ! + kEqual, // = + kEqualEqual, // == + kExclamationEqual, // != + kLess, // < + kLessEqual, // <= + kGreater, // > + kGreaterEqual, // >= + kLogicalAnd, // && + kLogicalOr, // || +}; + +ABSL_ATTRIBUTE_PURE_FUNCTION std::string_view TokenTypeToString(TokenType type); + +struct Token final { + TokenType type = TokenType::kError; + int32_t start = 0; + int32_t end = 0; +}; + +struct LexerError final { + int32_t start = 0; + int32_t end = 0; + std::string message; +}; + +// Lexer performs fast tokenization of CEL expression source code. +// +// Responsibilities & Parser Expectations: +// This lexer is designed for speed and does not perform comprehensive semantic +// or syntax validation. It does not verify escape sequences or other special +// characters in string/bytes literals, and performs only general bounds and +// format matching for integers and floating-point numeric literals. The lexer +// expects the parser to perform final validation and conversion when building +// the AST. +class Lexer final { + public: + explicit Lexer(std::string_view source, + std::vector* absl_nullable line_offsets = nullptr) + : line_offsets_(line_offsets), + text_begin_(source.data() != nullptr ? source.data() : ""), + text_end_(text_begin_ + source.size()), + text_(text_begin_) { + ABSL_DCHECK_LT(source.size(), + static_cast(std::numeric_limits::max())); + } + + Lexer(const Lexer&) = delete; + Lexer(Lexer&&) = delete; + Lexer& operator=(const Lexer&) = delete; + Lexer& operator=(Lexer&&) = delete; + + // Scans and returns the next token from the source. + [[nodiscard]] ABSL_ATTRIBUTE_NOINLINE Token Lex(); + + [[nodiscard]] const LexerError& GetError() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + ABSL_DCHECK(at_error_); + return error_; + } + + [[nodiscard]] int32_t GetPosition() const { + return static_cast(text_ - text_begin_); + } + + private: + [[nodiscard]] int32_t Find(char c) const; + + [[nodiscard]] bool Match(char c) const { + return text_ != text_end_ && *text_ == c; + } + + [[nodiscard]] bool MatchIgnoreCase(char c) const { + return text_ != text_end_ && + absl::ascii_tolower(*text_) == absl::ascii_tolower(c); + } + + void Advance(size_t n) { + ABSL_DCHECK_LE(n, static_cast(text_end_ - text_)); + text_ += n; + } + + void AdvanceProcessingNewLines(size_t n); + void AdvanceProcessingNewLines(const char* end); + + [[nodiscard]] std::string_view GetRemainingText() const { + return std::string_view(text_, static_cast(text_end_ - text_)); + } + + [[nodiscard]] Token MakeToken(TokenType type, int32_t start, int32_t end) { + if (ABSL_PREDICT_FALSE(at_end_)) { + AtEndTokenCreated(); + } + return Token{.type = type, .start = start, .end = end}; + } + + [[nodiscard]] Token SetError(int32_t start, int32_t end, + std::string message) { + ABSL_DCHECK(!at_error_); + at_error_ = true; + error_ = + LexerError{.start = start, .end = end, .message = std::move(message)}; + return Token{.type = TokenType::kError, .start = start, .end = end}; + } + + void AtEndTokenCreated() { done_ = true; } + + [[nodiscard]] bool ConsumeUntilAfter(char c); + + [[nodiscard]] bool ConsumeUntilAfterString(std::string_view s); + + [[nodiscard]] bool ConsumeUntilAfterUnescaped(char c); + + void ConsumeIdentTrailing(); + + [[nodiscard]] bool MatchString(std::string_view s) const; + + [[nodiscard]] bool MatchStringIgnoreCase(std::string_view s) const; + + [[nodiscard]] std::optional MatchIf( + absl::FunctionRef predicate) const; + + void ConsumeLine(); + + void ConsumeWhitespace(); + + [[nodiscard]] bool Consume(char c); + + [[nodiscard]] bool ConsumeIgnoreCase(char c); + + [[nodiscard]] bool ConsumeString(std::string_view s); + + [[nodiscard]] bool ConsumeStringIgnoreCase(std::string_view s); + + [[nodiscard]] std::optional ConsumeIf( + absl::FunctionRef predicate); + + [[nodiscard]] bool ConsumeDigits(); + + [[nodiscard]] bool ConsumeHexDigits(); + + [[nodiscard]] TokenType ConsumeIntegralSuffix(); + + std::vector* absl_nullable line_offsets_; + const char* absl_nonnull text_begin_; + const char* absl_nonnull text_end_; + const char* absl_nonnull text_; + bool at_end_ = false; + bool at_error_ = false; + bool done_ = false; + LexerError error_; +}; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc new file mode 100644 index 000000000..0305c9475 --- /dev/null +++ b/parser/internal/lexer_test.cc @@ -0,0 +1,392 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include +#include + +#include "internal/testing.h" + +namespace cel::parser_internal { +namespace { + +MATCHER_P3(IsToken, source, expected_type, expected_text, "") { + if (arg.type != expected_type) { + *result_listener << "type is " << TokenTypeToString(arg.type) + << " (expected " << TokenTypeToString(expected_type) + << ")"; + return false; + } + std::string_view actual_text = source.substr(arg.start, arg.end - arg.start); + if (actual_text != expected_text) { + *result_listener << "text is '" << actual_text << "' (expected '" + << expected_text << "')"; + return false; + } + return true; +} + +struct LexerTestCase { + std::string_view name; + std::string_view source; + std::vector> expected_tokens; +}; + +using LexerTest = testing::TestWithParam; + +TEST_P(LexerTest, LexesSuccessTokens) { + const LexerTestCase& test_case = GetParam(); + Lexer lexer(test_case.source); + + for (const auto& [type, text] : test_case.expected_tokens) { + EXPECT_THAT(lexer.Lex(), IsToken(test_case.source, type, text)); + } + EXPECT_THAT(lexer.Lex(), IsToken(test_case.source, TokenType::kEnd, "")); +} + +INSTANTIATE_TEST_SUITE_P( + LexerTest, LexerTest, + testing::ValuesIn({ + {"NullSource", std::string_view(nullptr, 0), {}}, + {"Empty", "", {}}, + {"Whitespace", " \n ", {{TokenType::kWhitespace, " \n "}}}, + {"KeywordsAndIdents", + "null false true in as return foo_bar _foo_bar _ `quoted.ident`", + {{TokenType::kNull, "null"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFalse, "false"}, + {TokenType::kWhitespace, " "}, + {TokenType::kTrue, "true"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIn, "in"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "as"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "return"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`quoted.ident`"}}}, + {"Numbers", + "123 45u 0x1A 3.14 .5 1e6 2.5e-3 45U 0x1Au 0x1AU", + {{TokenType::kInt, "123"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x1A"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "3.14"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, ".5"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "1e6"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "2.5e-3"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45U"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1Au"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1AU"}}}, + {"IntEOF", "123456", {{TokenType::kInt, "123456"}}}, + {"HexIntEOF", "0x1A2B", {{TokenType::kInt, "0x1A2B"}}}, + {"FloatPositiveExponentEOF", "1e+6", {{TokenType::kFloat, "1e+6"}}}, + {"FloatEOF", ".12345", {{TokenType::kFloat, ".12345"}}}, + {"IntDotIdent", + "1.foo", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kIdent, "foo"}}}, + {"IntDotWhitespace", + "1. ", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}}}, + {"IntDotEOF", "1.", {{TokenType::kInt, "1"}, {TokenType::kDot, "."}}}, + {"DotAtEOFBeforeDigit", + std::string_view(".6", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"DotAtEOFBeforeIdent", + std::string_view(".a", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"ZeroNumbers", + "0 0u 0x0", + {{TokenType::kInt, "0"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x0"}}}, + {"StringsAndBytes", + R"("hello" 'world' """multi +line""" r"raw" b"bytes" rb'\x00' '''multi +single''' R"raw_upper" B"bytes_upper" b'''multi +bytes''' br"raw_bytes" `a.b-c/d e` +"\a\b\f\n\r\t\v\"\'\\\?\` \x1A \u00A0 \U0001F600 \012")", + {{TokenType::kString, "\"hello\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'world'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "\"\"\"multi\nline\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"raw\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "rb'\\x00'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'''multi\nsingle'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "R\"raw_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "B\"bytes_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b'''multi\nbytes'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "br\"raw_bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`a.b-c/d e`"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kString, + "\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\\?\\` \\x1A \\u00A0 \\U0001F600 " + "\\012\""}}}, + {"EmptyStrings", + "\"\" '' \"\"\"\"\"\" '''''' r\"\" r'' r\"\"\"\"\"\" r'''''' b\"\" " + "b'' b\"\"\"\"\"\" b''''''", + {{TokenType::kString, "\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''''''"}}}, + {"OperatorsAndDelimiters", + ". , + - * / % == != < <= > >= && || ! ? : [] { } ( )", + {{TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kPlus, "+"}, + {TokenType::kWhitespace, " "}, + {TokenType::kMinus, "-"}, + {TokenType::kWhitespace, " "}, + {TokenType::kAsterisk, "*"}, + {TokenType::kWhitespace, " "}, + {TokenType::kSlash, "/"}, + {TokenType::kWhitespace, " "}, + {TokenType::kPercent, "%"}, + {TokenType::kWhitespace, " "}, + {TokenType::kEqualEqual, "=="}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamationEqual, "!="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLess, "<"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLessEqual, "<="}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreater, ">"}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreaterEqual, ">="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalAnd, "&&"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalOr, "||"}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamation, "!"}, + {TokenType::kWhitespace, " "}, + {TokenType::kQuestion, "?"}, + {TokenType::kWhitespace, " "}, + {TokenType::kColon, ":"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, + {TokenType::kRightBracket, "]"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightBrace, "}"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftParen, "("}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightParen, ")"}}}, + {"Comments", + "a\n// comment\nb", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kComment, "// comment\n"}, + {TokenType::kIdent, "b"}}}, + {"CommentWithoutTrailingNewlineEOF", + "// comment without trailing newline", + {{TokenType::kComment, "// comment without trailing newline"}}}, + {"CommentAfterTokenWithoutTrailingNewlineEOF", + "a // comment without trailing newline", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, " "}, + {TokenType::kComment, "// comment without trailing newline"}}}, + }), + [](const testing::TestParamInfo& info) { + return std::string(info.param.name); + }); + +TEST(LexerTest, LineOffsets) { + std::string_view source = "a\n// comment\nb"; + std::vector line_offsets; + Lexer lexer(source, &line_offsets); + + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kIdent, "a")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source, TokenType::kComment, "// comment\n")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kIdent, "b")); + + ASSERT_EQ(line_offsets.size(), 2); + EXPECT_EQ(line_offsets[0], 2); + EXPECT_EQ(line_offsets[1], 13); +} + +TEST(LexerTest, LineOffsetsInStringsAndIdentifiers) { + std::string_view source = + "'''multi\nline'''\n\"another\nline\"\n`ident\nhere`"; + std::vector line_offsets; + Lexer lexer(source, &line_offsets); + + EXPECT_THAT(lexer.Lex(), + IsToken(source, TokenType::kString, "'''multi\nline'''")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source, TokenType::kString, "\"another\nline\"")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kIdent, "`ident\nhere`")); + EXPECT_THAT(lexer.Lex(), IsToken(source, TokenType::kEnd, "")); + + ASSERT_EQ(line_offsets.size(), 5); + EXPECT_EQ(line_offsets[0], 9); + EXPECT_EQ(line_offsets[1], 17); + EXPECT_EQ(line_offsets[2], 26); + EXPECT_EQ(line_offsets[3], 32); + EXPECT_EQ(line_offsets[4], 39); +} + +struct LexerErrorTestCase { + std::string_view source; + std::string_view expected_error_message; + int32_t expected_position; +}; + +using LexerErrorTest = testing::TestWithParam; + +TEST_P(LexerErrorTest, LexesErrorTokenAndStoresError) { + const LexerErrorTestCase& test_case = GetParam(); + Lexer lexer(test_case.source); + Token token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kError); + EXPECT_EQ(lexer.GetError().message, test_case.expected_error_message); + EXPECT_EQ(lexer.GetPosition(), test_case.expected_position); +} + +INSTANTIATE_TEST_SUITE_P( + ErrorCases, LexerErrorTest, + testing::Values( + LexerErrorTestCase{ + .source = "\"unterminated", + .expected_error_message = "unterminated string literal", + .expected_position = 13, + }, + LexerErrorTestCase{ + .source = "0x", + .expected_error_message = + "integral literal missing digits after hexadecimal separator", + .expected_position = 2, + }, + LexerErrorTestCase{ + .source = "@", + .expected_error_message = "unexpected character", + .expected_position = 1, + }, + LexerErrorTestCase{ + .source = "0x1A_invalid", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_position = 5, + }, + LexerErrorTestCase{ + .source = "123_invalid", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_position = 4, + }, + LexerErrorTestCase{ + .source = "1x0", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_position = 2, + }, + LexerErrorTestCase{ + .source = "2x", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_position = 2, + }, + LexerErrorTestCase{ + .source = "`unterminated quoted", + .expected_error_message = "unterminated quoted identifier", + .expected_position = 20, + }, + LexerErrorTestCase{ + .source = "'''unterminated multi", + .expected_error_message = "unterminated string literal", + .expected_position = 21, + }, + LexerErrorTestCase{ + .source = "r'unterminated raw", + .expected_error_message = "unterminated string literal", + .expected_position = 18, + }, + LexerErrorTestCase{ + .source = "b'unterminated bytes", + .expected_error_message = "unterminated bytes literal", + .expected_position = 20, + }, + LexerErrorTestCase{ + .source = "1e", + .expected_error_message = + "floating point literal missing digits after exponent " + "separator", + .expected_position = 2, + })); + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/options.h b/parser/internal/options.h index ec2552204..4066d2619 100644 --- a/parser/internal/options.h +++ b/parser/internal/options.h @@ -15,7 +15,7 @@ #ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ -namespace cel_parser_internal { +namespace cel::parser_internal { inline constexpr int kDefaultErrorRecoveryLimit = 12; inline constexpr int kDefaultMaxRecursionDepth = 32; @@ -23,6 +23,6 @@ inline constexpr int kExpressionSizeCodepointLimit = 100'000; inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = 512; inline constexpr bool kDefaultAddMacroCalls = false; -} // namespace cel_parser_internal +} // namespace cel::parser_internal #endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ diff --git a/parser/options.h b/parser/options.h index 719bed454..22154e1f9 100644 --- a/parser/options.h +++ b/parser/options.h @@ -25,25 +25,25 @@ struct ParserOptions final { // Limit of the number of error recovery attempts made by the ANTLR parser // when processing an input. This limit, when reached, will halt further // parsing of the expression. - int error_recovery_limit = ::cel_parser_internal::kDefaultErrorRecoveryLimit; + int error_recovery_limit = ::cel::parser_internal::kDefaultErrorRecoveryLimit; // Limit on the amount of recursive parse instructions permitted when building // the abstract syntax tree for the expression. This prevents pathological // inputs from causing stack overflows. - int max_recursion_depth = ::cel_parser_internal::kDefaultMaxRecursionDepth; + int max_recursion_depth = ::cel::parser_internal::kDefaultMaxRecursionDepth; // Limit on the number of codepoints in the input string which the parser will // attempt to parse. int expression_size_codepoint_limit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; // Limit on the number of lookahead tokens to consume when attempting to // recover from an error. int error_recovery_token_lookahead_limit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; // Add macro calls to macro_calls list in source_info. - bool add_macro_calls = ::cel_parser_internal::kDefaultAddMacroCalls; + bool add_macro_calls = ::cel::parser_internal::kDefaultAddMacroCalls; // Enable support for optional syntax. bool enable_optional_syntax = false; @@ -76,20 +76,20 @@ using ParserOptions = ::cel::ParserOptions; ABSL_DEPRECATED("Use ParserOptions().error_recovery_limit instead.") inline constexpr int kDefaultErrorRecoveryLimit = - ::cel_parser_internal::kDefaultErrorRecoveryLimit; + ::cel::parser_internal::kDefaultErrorRecoveryLimit; ABSL_DEPRECATED("Use ParserOptions().max_recursion_depth instead.") inline constexpr int kDefaultMaxRecursionDepth = - ::cel_parser_internal::kDefaultMaxRecursionDepth; + ::cel::parser_internal::kDefaultMaxRecursionDepth; ABSL_DEPRECATED("Use ParserOptions().expression_size_codepoint_limit instead.") inline constexpr int kExpressionSizeCodepointLimit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; ABSL_DEPRECATED( "Use ParserOptions().error_recovery_token_lookahead_limit instead.") inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; ABSL_DEPRECATED("Use ParserOptions().add_macro_calls instead.") inline constexpr bool kDefaultAddMacroCalls = - ::cel_parser_internal::kDefaultAddMacroCalls; + ::cel::parser_internal::kDefaultAddMacroCalls; } // namespace google::api::expr::parser