From 9c09634bff32f0a8f9ce4455a6a7b33a47666241 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 28 May 2026 13:11:11 -0700 Subject: [PATCH 01/23] Internal change PiperOrigin-RevId: 922934620 --- MODULE.bazel | 4 +--- README.md | 8 -------- bazel/BUILD | 10 ++++++++++ bazel/antlr.patch | 30 ++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 bazel/BUILD create mode 100644 bazel/antlr.patch diff --git a/MODULE.bazel b/MODULE.bazel index 43c90b6..1ed9a28 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -50,9 +50,7 @@ bazel_dep(name = "rules_python", version = "1.9.0") # antlr4-cpp-runtime/VERSION and the system `#include ` single_version_override( module_name = "antlr4-cpp-runtime", - patch_cmds = [ - "python3 -c \"import os; [os.rename(f, f + '.txt') for f in ['VERSION', 'version'] if os.path.exists(f)]\" || python -c \"import os; [os.rename(f, f + '.txt') for f in ['VERSION', 'version'] if os.path.exists(f)]\"", - ], + patches = ["//bazel:antlr.patch"], ) # Configure rules_python's hermetic toolchains to resolve external diff --git a/README.md b/README.md index f25ab76..177f0b8 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,6 @@ This is a Python wrapper for the CEL C++ implementation. -## Installation - -``` -pip install cel-expr-python -``` - -Available on PyPI: https://pypi.org/project/cel-expr-python - ## Usage ### Importing CEL module diff --git a/bazel/BUILD b/bazel/BUILD new file mode 100644 index 0000000..18c905f --- /dev/null +++ b/bazel/BUILD @@ -0,0 +1,10 @@ +load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") +load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_python//python:py_test.bzl", "py_test") + +exports_files( + srcs = [ + "antlr.patch", + ], + visibility = ["//visibility:public"], +) diff --git a/bazel/antlr.patch b/bazel/antlr.patch new file mode 100644 index 0000000..afe9cb4 --- /dev/null +++ b/bazel/antlr.patch @@ -0,0 +1,30 @@ +--- BUILD.bazel ++++ BUILD.bazel +@@ -17,21 +17,21 @@ + cc_library( + name = "antlr4-cpp-runtime", + srcs = glob(["runtime/src/**/*.cpp"]), + hdrs = ["runtime/src/antlr4-runtime.h"], + copts = ["-fexceptions"], +- defines = ["ANTLR4CPP_USING_ABSEIL"], ++ defines = ["ANTLR4CPP_USING_ABSEIL", "ANTLR4CPP_STATIC"], + features = ["-use_header_modules"], + includes = ["runtime/src"], + textual_hdrs = glob( + ["runtime/src/**/*.h"], + exclude = ["runtime/src/antlr4-runtime.h"], + ), + visibility = ["//visibility:public"], + deps = [ + "@com_google_absl//absl/base", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/synchronization", + ], + ) + +--- VERSION ++++ /dev/null +@@ -1,1 +1,0 @@ +-4.13.2 From 92628a4737d17c7813540020fe011e73e5de043e Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 29 May 2026 13:46:57 -0700 Subject: [PATCH 02/23] Internal change PiperOrigin-RevId: 923583737 --- release/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/setup.py b/release/setup.py index be1068a..1fec997 100644 --- a/release/setup.py +++ b/release/setup.py @@ -116,6 +116,8 @@ def build_extension(self, ext): def platform_config_windows(self, cmd, python_version): """Applies Windows-specific Bazel workarounds for Hermetic Python.""" + cmd.insert(1, '--output_user_root=C:/tmp') + # 1. Get output base output_base = subprocess.check_output( ['bazel', '--output_user_root=C:/tmp', 'info', 'output_base'], text=True From 76b61aec7d44687a6fa170309039613da74b4db3 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 1 Jun 2026 13:56:18 -0700 Subject: [PATCH 03/23] Internal change PiperOrigin-RevId: 924897098 --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 177f0b8..f25ab76 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,14 @@ This is a Python wrapper for the CEL C++ implementation. +## Installation + +``` +pip install cel-expr-python +``` + +Available on PyPI: https://pypi.org/project/cel-expr-python + ## Usage ### Importing CEL module From 9d5a61d9334e3a6c130d5850397980cd617272e6 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 11 Jun 2026 11:24:29 -0700 Subject: [PATCH 04/23] Internal change PiperOrigin-RevId: 930640971 --- .bazelrc | 4 + release/kokoro/release_linux.cfg | 5 + release/kokoro/release_linux.sh | 171 +++++++++++++++++++++++++++++-- release/pyproject.toml | 11 +- 4 files changed, 180 insertions(+), 11 deletions(-) diff --git a/.bazelrc b/.bazelrc index 3da0a4c..a727ae9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -32,3 +32,7 @@ build:windows --google_default_credentials=true build:macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache build:macos --google_default_credentials=true +# Silence deprecation warnings from external dependencies (Linux and macOS) +build:linux --cxxopt=-Wno-deprecated-declarations +build:macos --cxxopt=-Wno-deprecated-declarations + diff --git a/release/kokoro/release_linux.cfg b/release/kokoro/release_linux.cfg index 6a91c7b..a42e2e3 100644 --- a/release/kokoro/release_linux.cfg +++ b/release/kokoro/release_linux.cfg @@ -3,3 +3,8 @@ build_file: "cel-python/release/kokoro/release_linux.sh" timeout_mins: 120 + +container_properties { + docker_image: "us-central1-docker.pkg.dev/kokoro-container-bakery/kokoro/ubuntu/ubuntu2204/ktcb:current" + docker_sibling_containers: true +} diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 3d92d51..5292ef6 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -1,6 +1,29 @@ #!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -e +# Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host +export VIRTUALENV_NO_DOWNLOAD=1 +export PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Pass these environment variables to the cibuildwheel Docker container +export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1" +export CIBW_DEPENDENCY_VERSIONS="latest" +export CIBW_CONTAINER_ENGINE_EXTRA_ARGS="--network=host" + # If running locally (not on Kokoro), authenticate with gcloud. if [ -z "${KOKORO_BUILD_ID}" ]; then if ! gcloud auth application-default print-access-token --quiet > /dev/null; then @@ -8,14 +31,137 @@ if [ -z "${KOKORO_BUILD_ID}" ]; then fi fi -pip install -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel +# We use --no-cache-dir to force pip to download packages fresh and bypass the local +# cache. In a sandboxed build environment, writing to the default cache directory +# (~/.cache/pip) can encounter permission/sandbox restrictions or lead to stale +# dependency resolution. Disabling the cache ensures a reliable, reproducible install. +pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel + +# ============================================================================== +# FUTURE-PROOF RUNTIME PATCHING OF CIBUILDWHEEL +# ============================================================================== +# To run cibuildwheel on Google's sandboxed RBE/Kokoro infrastructure, we must: +# 1. Bypass RBE's stdout proxy buffering deadlock (requires 32KB padding). +# 2. Bypass RBE's stdin EOF deadlock during copy-in (requires 'docker cp' +# since we use disable_host_mount: True in pyproject.toml). +# +# Since cibuildwheel is installed fresh from PyPI on every build (ensuring we get +# the latest security and feature updates), we apply these patches at runtime. +# +# Why this patching strategy is future-proof and safe: +# - Strict Validation: The Python patcher strictly validates that all target +# code blocks exist before applying replacements. If cibuildwheel's internal +# code changes in a future release, the patcher will FAIL LOUDLY and exit the +# build immediately (sys.exit(1)) rather than silently running a broken, +# hanging build. +# - Stable Boundaries: The copy_into patch uses a robust regular expression +# anchored to class method boundaries (def copy_into -> def copy_out). These +# are stable, long-standing internal APIs of cibuildwheel's OCIContainer. +# - Core Protocol Stability: The buffering patches target the core protocol +# used to communicate with the container's persistent bash shell. This +# protocol is fundamental to cibuildwheel and highly unlikely to change. +# ============================================================================== +OCI_PATH=$(python3 -c "import cibuildwheel.oci_container; print(cibuildwheel.oci_container.__file__)") +echo "Patching cibuildwheel at $OCI_PATH..." + +cat << 'EOF' > patch_oci.py +import sys +import re + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +# 1. Force a 32KB flush at the end of every command execution +target_write = 'printf "%04d%s\\n" $? {end_of_message}' +replacement_write = 'printf "%04d%s\\n%32768s\\n" $? {end_of_message} " "' +if target_write in content: + content = content.replace(target_write, replacement_write) + print("Patched write loop.") +else: + print("ERROR: Could not find write loop target in oci_container.py! The cibuildwheel version might have changed.") + sys.exit(1) + +# 2. Read and discard the 32KB padding to keep the stream clean +target_read = """ # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + output_io.flush() + break""" + +replacement_read = """ # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + output_io.flush() + # Read and discard the 32KB padding line to clear the stream! + self.bash_stdout.readline() + break""" + +if target_read in content: + content = content.replace(target_read, replacement_read) + print("Patched read loop.") +else: + print("ERROR: Could not find read loop target in oci_container.py! The cibuildwheel version might have changed.") + sys.exit(1) + +# 3. Patch the entire copy_into method using a unique regex to use native 'docker cp'. +# This bypasses the RBE stdin EOF deadlock when copying the project into the container. +pattern = re.compile(r' def copy_into\(self,.*?\).*?:.*? def copy_out', re.DOTALL) + +replacement_copy = """ def copy_into(self, from_path: Path, to_path: PurePath) -> None: + if from_path.is_dir(): + self.call(["mkdir", "-p", to_path]) + subprocess.run( + f"tar -c {self.host_tar_format} -f - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -", + shell=True, + check=True, + cwd=from_path, + ) + else: + self.call(["mkdir", "-p", to_path.parent]) + # Use native docker cp to copy the file, avoiding stdin EOF deadlocks in RBE + subprocess.run( + [ + self.engine.name, + "cp", + str(from_path), + f"{self.name}:{to_path}", + ], + check=True, + ) + + def copy_out""" + +if pattern.search(content): + content = pattern.sub(replacement_copy, content) + print("Patched copy_into method using unique regex.") +else: + print("ERROR: Could not find copy_into method boundary in oci_container.py! The cibuildwheel version might have changed.") + sys.exit(1) + +with open(path, 'w') as f: + f.write(content) + +print("Successfully patched oci_container.py!") +EOF + +python3 patch_oci.py "$OCI_PATH" +rm patch_oci.py + +# Verify that the patched file is syntactically valid Python +echo "Verifying patched oci_container.py syntax..." +python3 -m py_compile "$OCI_PATH" || { echo "ERROR: Patched oci_container.py is corrupted!"; exit 1; } + +REPO_DIR="" +TMP_DIR="" +cleanup() { + echo "Cleaning up temporary directories..." + [ -n "${REPO_DIR}" ] && rm -rf "${REPO_DIR}" + [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" +} +trap cleanup EXIT REPO_DIR=$(mktemp -d) echo "Created temporary directory: ${REPO_DIR}" -# Ensure the temporary directory is removed on script exit -trap 'echo "Cleaning up temporary directory: ${REPO_DIR}"; rm -rf "${REPO_DIR}"' EXIT - if [ "${DRY_RUN}" = "true" ]; then echo "[DRY RUN] Using local Kokoro clone instead of cloning main." SRC_DIR="$(cd "$(dirname "$0")/../.." && pwd)" @@ -40,11 +186,14 @@ fi VERSION=${VERSION#v} echo "Building release for version: ${VERSION}" -TMP_DIR=$(mktemp -d) +# Create the build directory inside the workspace volume (SRC_DIR) +# instead of the ephemeral /tmp, so that the sibling container can +# access it natively via volume propagation +TMP_DIR="${SRC_DIR}/build_area" +mkdir -p "${TMP_DIR}" echo "Build directory: ${TMP_DIR}" - -# Add trap cleanup for TMP_DIR as well -trap 'echo "Cleaning up temporary directories: ${REPO_DIR} ${TMP_DIR}"; rm -rf "${REPO_DIR}" "${TMP_DIR}"' EXIT +export TMPDIR="${TMP_DIR}/tmp" +mkdir -p "${TMPDIR}" pushd "${TMP_DIR}" @@ -52,12 +201,16 @@ cp -r "${SRC_DIR}"/{*,.*} . 2>/dev/null || true cp -r "${SRC_DIR}"/release/* . 2>/dev/null || true rm -rf cel_expr_python/*_test.py +echo "Downloading bazelisk on host..." +curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 +chmod +x bazelisk-linux-amd64 + # Check if pyproject.toml exists before running sed if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi -echo "Running cibuildwheel: ${CIBWHEEL_BIN}" +echo "Running cibuildwheel..." # Default CIBWHEEL_BIN if not set if [ -z "${CIBWHEEL_BIN}" ]; then CIBWHEEL_BIN="python3 -m cibuildwheel" diff --git a/release/pyproject.toml b/release/pyproject.toml index 3022d61..8c645c5 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -39,12 +39,19 @@ exclude = ["codelab*", "conformance*", "custom_ext*", "release*", "testing*", "w [tool.cibuildwheel] build = "cp311-* cp312-* cp313-* cp314-*" -skip = "*musllinux* *win32*" +skip = "*musllinux* *win32* *i686*" test-command = "python {project}/cel_basic_test.py" build-verbosity = 1 [tool.cibuildwheel.linux] -before-all = "echo 'Installing bazelisk'; curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 && chmod +x bazelisk-linux-amd64 && mv bazelisk-linux-amd64 /usr/local/bin/bazel" +manylinux-x86_64-image = "manylinux_2_28" +container-engine = "docker; disable_host_mount: True" +# Google's internal Kokoro/RBE network uses a secure MITM proxy that resigns HTTPS +# traffic with an internal Google CA. Since the public manylinux container does not +# trust this CA, git fetches for external dependencies (like @cel-cpp) will fail +# with SSL certificate errors. We disable http.sslVerify inside the container to +# bypass this and allow Bazel to fetch SCM dependencies through the proxy. +before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel" [tool.cibuildwheel.macos] before-all = "echo 'Installing bazelisk'; brew install bazelisk" From adc0bfd517c432754441a42f570edbadb08965f4 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 16 Jun 2026 10:20:16 -0700 Subject: [PATCH 05/23] Add Python bindings and wrapper for CEL Policy compilation. Add cel_policy.compile Python wrapper and C++ bindings to compile CEL policies into CEL AST expressions (cel.Expression). Refactor cel C++ sources into a standalone pybind_library to enable clean C++ dependency sharing across extension modules. Add comprehensive unit tests verifying policy compilation, invalid condition handling, unnesting height limits, and AST evaluation. PiperOrigin-RevId: 933160646 --- cel_expr_python/BUILD | 54 +++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index adbee6c..b817175 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -9,46 +9,42 @@ licenses(["notice"]) exports_files(["LICENSE"]) # For Python programs using CEL. -pybind_extension( - name = "cel", +pybind_library( + name = "cel_pybind_lib", srcs = [ "py_cel_activation.cc", - "py_cel_activation.h", "py_cel_arena.cc", - "py_cel_arena.h", "py_cel_env.cc", - "py_cel_env.h", "py_cel_env_config.cc", - "py_cel_env_config.h", "py_cel_env_internal.cc", - "py_cel_env_internal.h", "py_cel_expression.cc", - "py_cel_expression.h", "py_cel_function.cc", - "py_cel_function.h", "py_cel_function_decl.cc", - "py_cel_function_decl.h", - "py_cel_module.cc", "py_cel_overload.cc", - "py_cel_overload.h", "py_cel_python_extension.cc", - "py_cel_python_extension.h", "py_cel_type.cc", - "py_cel_type.h", "py_cel_value.cc", + "py_descriptor_database.cc", + "py_message_factory.cc", + ], + hdrs = [ + "py_cel_activation.h", + "py_cel_arena.h", + "py_cel_env.h", + "py_cel_env_config.h", + "py_cel_env_internal.h", + "py_cel_expression.h", + "py_cel_function.h", + "py_cel_function_decl.h", + "py_cel_overload.h", + "py_cel_python_extension.h", + "py_cel_type.h", "py_cel_value.h", "py_cel_value_provider.h", - "py_descriptor_database.cc", "py_descriptor_database.h", - "py_error_status.cc", - "py_error_status.h", - "py_message_factory.cc", "py_message_factory.h", ], - data = ["cel.pyi"], - visibility = [ - "//visibility:public", - ], + visibility = [":__subpackages__"], deps = [ ":cel_extension", ":status_macros", @@ -103,6 +99,20 @@ pybind_extension( ], ) +pybind_extension( + name = "cel", + srcs = [ + "py_cel_module.cc", + ], + data = ["cel.pyi"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":cel_pybind_lib", + ], +) + # For pybind11-based CEL extensions. pybind_library( name = "cel_extension", From 5a5538c1f6de819764f1dc241a39a8dae830776b Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 17 Jun 2026 14:12:14 -0700 Subject: [PATCH 06/23] Update cel-cpp git override commit and remote in MODULE.bazel PiperOrigin-RevId: 933916502 --- MODULE.bazel | 11 ++------- bazel/BUILD | 10 -------- bazel/antlr.patch | 30 ------------------------ cel_expr_python/py_descriptor_database.h | 4 +++- 4 files changed, 5 insertions(+), 50 deletions(-) delete mode 100644 bazel/BUILD delete mode 100644 bazel/antlr.patch diff --git a/MODULE.bazel b/MODULE.bazel index 1ed9a28..7404b9f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,8 +15,8 @@ bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "cel-cpp", version = "0.15.0", repo_name = "com_google_cel_cpp") git_override( module_name = "cel-cpp", - commit = "2e6e9ff4493bfbe0baf883107f3fb7ce6f675d88", - remote = "https://github.com/google/cel-cpp", + commit = "8b7068abb4062074a135491ad8357139287084f9", + remote = "https://github.com/cel-expr/cel-cpp", ) # https://registry.bazel.build/modules/cel-spec @@ -46,13 +46,6 @@ bazel_dep(name = "rules_proto", version = "7.1.0") # https://registry.bazel.build/modules/rules_python bazel_dep(name = "rules_python", version = "1.9.0") -# On Windows the file system is case-insensitive, which creates a collision between -# antlr4-cpp-runtime/VERSION and the system `#include ` -single_version_override( - module_name = "antlr4-cpp-runtime", - patches = ["//bazel:antlr.patch"], -) - # Configure rules_python's hermetic toolchains to resolve external # dependencies natively. Under Windows, compiling pybind C++ extensions # against Python 3.11 headers requires that the execution environment diff --git a/bazel/BUILD b/bazel/BUILD deleted file mode 100644 index 18c905f..0000000 --- a/bazel/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") -load("@rules_python//python:py_library.bzl", "py_library") -load("@rules_python//python:py_test.bzl", "py_test") - -exports_files( - srcs = [ - "antlr.patch", - ], - visibility = ["//visibility:public"], -) diff --git a/bazel/antlr.patch b/bazel/antlr.patch deleted file mode 100644 index afe9cb4..0000000 --- a/bazel/antlr.patch +++ /dev/null @@ -1,30 +0,0 @@ ---- BUILD.bazel -+++ BUILD.bazel -@@ -17,21 +17,21 @@ - cc_library( - name = "antlr4-cpp-runtime", - srcs = glob(["runtime/src/**/*.cpp"]), - hdrs = ["runtime/src/antlr4-runtime.h"], - copts = ["-fexceptions"], -- defines = ["ANTLR4CPP_USING_ABSEIL"], -+ defines = ["ANTLR4CPP_USING_ABSEIL", "ANTLR4CPP_STATIC"], - features = ["-use_header_modules"], - includes = ["runtime/src"], - textual_hdrs = glob( - ["runtime/src/**/*.h"], - exclude = ["runtime/src/antlr4-runtime.h"], - ), - visibility = ["//visibility:public"], - deps = [ - "@com_google_absl//absl/base", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/synchronization", - ], - ) - ---- VERSION -+++ /dev/null -@@ -1,1 +1,0 @@ --4.13.2 diff --git a/cel_expr_python/py_descriptor_database.h b/cel_expr_python/py_descriptor_database.h index 610f8b2..5f621f4 100644 --- a/cel_expr_python/py_descriptor_database.h +++ b/cel_expr_python/py_descriptor_database.h @@ -19,6 +19,8 @@ #include // IWYU pragma: keep - Needed for PyObject +#include // IWYU pragma: keep - Needed for string_view in OSS + #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/descriptor_database.h" @@ -27,7 +29,7 @@ namespace cel_python { // A DescriptorDatabase that uses a Python DescriptorPool to find descriptors. class PyDescriptorDatabase : public google::protobuf::DescriptorDatabase { private: - using StringViewArg = const std::string&; + using StringViewArg = std::string_view; public: explicit PyDescriptorDatabase(PyObject* py_descriptor_pool); ~PyDescriptorDatabase() override; From 200bb5fd883c704658dbf1fa2574096ea1d94ba5 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 17 Jun 2026 14:35:17 -0700 Subject: [PATCH 07/23] Add context_variable to CEL Env Context_variable is a struct type that defines top-level variables that can be directly referenced by a CEL policy. PiperOrigin-RevId: 933928877 --- cel_expr_python/cel.pyi | 2 ++ cel_expr_python/cel_env_test.py | 46 ++++++++++++++++++++++++++++ cel_expr_python/py_cel_env_config.cc | 2 ++ cel_expr_python/py_cel_env_config.h | 1 + 4 files changed, 51 insertions(+) diff --git a/cel_expr_python/cel.pyi b/cel_expr_python/cel.pyi index 549e7bd..f5c8225 100644 --- a/cel_expr_python/cel.pyi +++ b/cel_expr_python/cel.pyi @@ -13,6 +13,8 @@ class CelExtensionBase: def __init__(self, name: str) -> None: ... class EnvConfig: + @property + def context_type(self) -> str: ... def to_yaml(self) -> str: ... class ExpressionContainer: diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index 92a64ff..ce98570 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -98,6 +98,52 @@ def test_invalid_yaml(self): str(e.exception), ) + def test_parse_context_variable_config(self): + config = cel.NewEnvConfigFromYaml(""" + context_variable: + type_name: "cel.expr.conformance.proto2.TestAllTypes" + """) + self.assertEqual( + config.context_type, "cel.expr.conformance.proto2.TestAllTypes" + ) + + def test_parse_context_variable_config_alternative_syntax(self): + config = cel.NewEnvConfigFromYaml(""" + context_variable: + type: "cel.expr.conformance.proto2.TestAllTypes" + """) + self.assertEqual( + config.context_type, "cel.expr.conformance.proto2.TestAllTypes" + ) + + def test_parse_context_variable_malformed(self): + with self.assertRaisesRegex( + Exception, "Node 'context_variable' is not a map" + ): + cel.NewEnvConfigFromYaml("context_variable: 123") + + def test_parse_context_variable_malformed2(self): + with self.assertRaisesRegex( + Exception, "Node 'context_variable' does not have a valid type" + ): + cel.NewEnvConfigFromYaml(""" + context_variable: + type: + foo: bar + """) + + def test_context_variable_basic(self): + config = cel.NewEnvConfigFromYaml(""" + context_variable: + type_name: "cel.expr.conformance.proto2.TestAllTypes" + """) + env = cel.NewEnv(config=config) + ast = env.compile("single_int32 > 10") + self.assertIsNotNone(ast) + + with self.assertRaises(Exception): + env.compile("non_existent_field > 10") + def test_config_export_container(self): env: cel.Env = cel.NewEnv(container="test.container") yaml: str = env.config().to_yaml() diff --git a/cel_expr_python/py_cel_env_config.cc b/cel_expr_python/py_cel_env_config.cc index 1f4014f..ff111d8 100644 --- a/cel_expr_python/py_cel_env_config.cc +++ b/cel_expr_python/py_cel_env_config.cc @@ -33,6 +33,8 @@ void PyCelEnvConfig::DefinePythonBindings(pybind11::module& m) { m.def("NewEnvConfigFromYaml", &PyCelEnvConfig::FromYaml, py::arg("yaml")); cel_class.def("to_yaml", &PyCelEnvConfig::ToYaml); + cel_class.def_property_readonly("context_type", + &PyCelEnvConfig::GetContextType); } PyCelEnvConfig PyCelEnvConfig::FromYaml(std::string yaml) { diff --git a/cel_expr_python/py_cel_env_config.h b/cel_expr_python/py_cel_env_config.h index 1742413..9d93bb8 100644 --- a/cel_expr_python/py_cel_env_config.h +++ b/cel_expr_python/py_cel_env_config.h @@ -34,6 +34,7 @@ class PyCelEnvConfig { std::string ToYaml() const; const cel::Config& GetConfig() const { return config_; } + std::string GetContextType() const { return config_.GetContextType(); } private: cel::Config config_; From 441420721a1f64b916c013b1787b3d8d1622567e Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 17 Jun 2026 16:55:56 -0700 Subject: [PATCH 08/23] Add support for macos/x86_64 wheels PiperOrigin-RevId: 933998105 --- cel_expr_python/BUILD | 12 ++++++++++++ release/pyproject.toml | 2 ++ release/setup.py | 27 ++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index b817175..6f9836d 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -187,3 +187,15 @@ py_test( ], }), ) + +# Platform definition for macOS x86_64. +# Used for cross-compiling from arm64 macOS hosts (Apple Silicon) to x86_64. +# This is passed to Bazel via --platforms in setup.py when building x86_64 wheels. +platform( + name = "macos_x86_64", + constraint_values = [ + "@platforms//os:osx", + "@platforms//cpu:x86_64", + ], + visibility = ["//visibility:public"], +) diff --git a/release/pyproject.toml b/release/pyproject.toml index 8c645c5..c6b322a 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -54,7 +54,9 @@ container-engine = "docker; disable_host_mount: True" before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel" [tool.cibuildwheel.macos] +archs = ["x86_64", "arm64"] before-all = "echo 'Installing bazelisk'; brew install bazelisk" +environment = { MACOSX_DEPLOYMENT_TARGET = "10.13" } [tool.cibuildwheel.windows] # Bazel is expected to be already installed and in the PATH on Windows. diff --git a/release/setup.py b/release/setup.py index 1fec997..09cfe81 100644 --- a/release/setup.py +++ b/release/setup.py @@ -16,6 +16,7 @@ import glob import os +import platform import re import shutil import subprocess @@ -161,7 +162,31 @@ def platform_config_windows(self, cmd, python_version): def platform_config_macos(self, cmd): """Applies macOS-specific Bazel configurations.""" - cmd.extend(['--macos_minimum_os=10.13', '--cxxopt=-faligned-allocation']) + deployment_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', '10.13') + cmd.extend([ + f'--macos_minimum_os={deployment_target}', + '--cxxopt=-faligned-allocation', + ]) + + archflags = os.environ.get('ARCHFLAGS', '') + if 'x86_64' in archflags: + target_arch = 'x86_64' + elif 'arm64' in archflags: + target_arch = 'arm64' + else: + machine = platform.machine() + if machine in ('AMD64', 'x86_64'): + target_arch = 'x86_64' + elif machine in ('arm64', 'aarch64'): + target_arch = 'arm64' + else: + target_arch = machine + + print(f'Target architecture for macOS: {target_arch}', flush=True) + cmd.append(f'--macos_cpus={target_arch}') + cmd.append(f'--cpu=darwin_{target_arch}') + if target_arch == 'x86_64': + cmd.append('--platforms=//cel_expr_python:macos_x86_64') setuptools.setup( From 78c80964257b5346912699d5e3ea3add70cf7413 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 18 Jun 2026 13:00:30 -0700 Subject: [PATCH 09/23] Fix Windows bazel build by unsetting ANDROID_HOME Prevent transitive rules_android dependency from attempting to configure an Android SDK repository on CI environments where ANDROID_HOME is set but contains an incompatible Android SDK build tools version. PiperOrigin-RevId: 934508617 --- .bazelrc | 4 ++++ release/kokoro/presubmit_windows.bat | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.bazelrc b/.bazelrc index a727ae9..5c70e99 100644 --- a/.bazelrc +++ b/.bazelrc @@ -2,6 +2,10 @@ # This allows Bazel to automatically pick up the `windows` config on Windows. common --enable_platform_specific_config +# Prevent rules_android (pulled in transitively) from attempting to configure an Android SDK repository +# using the preinstalled Android SDK on CI machines (e.g. Kokoro Windows), which fails build tools version checks. +common --repo_env=ANDROID_HOME= + # Disable automatic creation of `__init__.py` files, which prevent multiple # workspaces from providing files with the same package prefix (e.g., "cel"). # See https://github.com/bazelbuild/rules_python/issues/330. diff --git a/release/kokoro/presubmit_windows.bat b/release/kokoro/presubmit_windows.bat index 30136f1..5e9e8ea 100644 --- a/release/kokoro/presubmit_windows.bat +++ b/release/kokoro/presubmit_windows.bat @@ -56,7 +56,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( :fetch_loop set /a ATTEMPTS+=1 echo Fetch attempt !ATTEMPTS! of !FETCH_RETRIES!... - bazel %STARTUP_FLAGS% fetch //... > fetch.log 2>&1 + bazel !STARTUP_FLAGS! fetch //... > fetch.log 2>&1 set FETCH_STATUS=!ERRORLEVEL! type fetch.log if !FETCH_STATUS! NEQ 0 ( @@ -78,7 +78,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( if exist fetch.log del fetch.log echo --- Getting Output Base --- - for /f "tokens=*" %%i in ('bazel %STARTUP_FLAGS% info output_base') do set "OUTPUT_BASE=%%i" + for /f "tokens=*" %%i in ('bazel !STARTUP_FLAGS! info output_base') do set "OUTPUT_BASE=%%i" set "OUTPUT_BASE=!OUTPUT_BASE:/=\!" echo Output Base: !OUTPUT_BASE! @@ -99,7 +99,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( ) echo --- Bazel Build --- - bazel %STARTUP_FLAGS% build %LINK_FLAGS% //... + bazel !STARTUP_FLAGS! build !LINK_FLAGS! //... if !ERRORLEVEL! NEQ 0 ( echo Build failed! set "PRESUBMIT_STATUS=1" @@ -107,7 +107,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( ) echo --- Bazel Test Python %%V --- - bazel %STARTUP_FLAGS% test %LINK_FLAGS% --test_output=errors //... + bazel !STARTUP_FLAGS! test !LINK_FLAGS! --test_output=errors //... if !ERRORLEVEL! NEQ 0 ( echo Tests failed for Python %%V! set "PRESUBMIT_STATUS=1" From 42ea759702d001ee07d5dde9db04ee1a1471be42 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 22 Jun 2026 13:39:08 -0700 Subject: [PATCH 10/23] Update YAML configuration and constructors to use type and overload signatures PiperOrigin-RevId: 936227381 --- cel_expr_python/BUILD | 1 + cel_expr_python/cel.pyi | 4 +- cel_expr_python/cel_env_test.py | 218 ++++++++++++++---------- cel_expr_python/cel_test.py | 12 ++ cel_expr_python/py_cel_env_config.cc | 4 +- cel_expr_python/py_cel_env_config.h | 2 +- cel_expr_python/py_cel_env_internal.cc | 12 +- cel_expr_python/py_cel_function_decl.cc | 3 + cel_expr_python/py_cel_overload.cc | 46 ++++- cel_expr_python/py_cel_type.cc | 52 +++++- cel_expr_python/py_cel_type.h | 1 + codelab/index.lab.md | 18 +- codelab/solution/codelab.py | 13 +- 13 files changed, 257 insertions(+), 129 deletions(-) diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index 6f9836d..190c18a 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -70,6 +70,7 @@ pybind_library( "@com_google_cel_cpp//common:function_descriptor", "@com_google_cel_cpp//common:kind", "@com_google_cel_cpp//common:minimal_descriptor_pool", + "@com_google_cel_cpp//common:signature", "@com_google_cel_cpp//common:source", "@com_google_cel_cpp//common:type", "@com_google_cel_cpp//common:type_kind", diff --git a/cel_expr_python/cel.pyi b/cel_expr_python/cel.pyi index f5c8225..f5cc7c8 100644 --- a/cel_expr_python/cel.pyi +++ b/cel_expr_python/cel.pyi @@ -39,7 +39,7 @@ class FunctionDecl: def __init__(self, name: str, overloads: Sequence[Overload]) -> None: ... class Overload: - def __init__(self, overload_id: str, return_type: Type = ..., parameters: Sequence[Type] = ..., is_member: bool = ..., impl: Callable[..., Any] = ...) -> None: ... + def __init__(self, id: str | None = ..., return_type: Type = ..., parameters: Sequence[Type] = ..., is_member: bool = ..., impl: Callable[..., Any] = ..., signature: str | None = ...) -> None: ... class Type: BOOL: ClassVar[Type] = ... @@ -57,7 +57,7 @@ class Type: TYPE: ClassVar[Type] = ... UINT: ClassVar[Type] = ... UNKNOWN: ClassVar[Type] = ... - def __init__(self, name: str) -> None: ... + def __init__(self, signature: str) -> None: ... @staticmethod def AbstractType(name: str, params: Sequence[Type] = ...) -> Type: ... @staticmethod diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index ce98570..eacfc9f 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -46,17 +46,13 @@ def test_env_config_from_and_to_yaml(self): - name: math variables: - name: one - type_name: int + type: int value: 1 functions: - name: add overloads: - - id: "add_int_int" - args: - - type_name: int - - type_name: int - return: - type_name: int + - signature: "add(int,int)" + return: int """) yaml: str = config.to_yaml() self.assertEqual( @@ -74,17 +70,13 @@ def test_env_config_from_and_to_yaml(self): - name: "_+_" variables: - name: "one" - type_name: "int" + type: "int" value: 1 functions: - name: "add" overloads: - - id: "add_int_int" - args: - - type_name: "int" - - type_name: "int" - return: - type_name: "int" + - signature: "add(int,int)" + return: "int" """), ) @@ -185,9 +177,9 @@ def test_expression_container_abbreviations_and_aliases(self): qualified_name: "x.y.bar" variables: - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" """), ) @@ -202,9 +194,9 @@ def test_abbreviations_and_aliases_from_yaml(self): qualified_name: "x.y.bar" variables: - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" """)) res = env.compile("foo").eval(data={"x.y.foo": 42}) @@ -224,13 +216,13 @@ def test_abbreviations_and_aliases_combined(self): qualified_name: "x.y.bar" variables: - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" - name: "a.b.qux" - type_name: "string" + type: "string" - name: "a.b.baz" - type_name: "int" + type: "int" """), container=cel.ExpressionContainer( "test.container", @@ -265,13 +257,13 @@ def test_abbreviations_and_aliases_combined(self): qualified_name: "a.b.qux" variables: - name: "a.b.baz" - type_name: "int" + type: "int" - name: "a.b.qux" - type_name: "string" + type: "string" - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" """), ) @@ -321,48 +313,35 @@ def test_config_export_variables(self): normalize_yaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" - name: "var_bytes" - type_name: "bytes" + type: "bytes" - name: "var_double" - type_name: "double" + type: "double" - name: "var_duration" - type_name: "duration" + type: "duration" - name: "var_dyn" - type_name: "dyn" + type: "dyn" - name: "var_dyn_list" - type_name: "list" - params: - - type_name: "dyn" + type: "list" - name: "var_dyn_map" - type_name: "map" - params: - - type_name: "dyn" - - type_name: "dyn" + type: "map" - name: "var_int" - type_name: "int" + type: "int" - name: "var_int_map" - type_name: "map" - params: - - type_name: "int" - - type_name: "string" + type: "map" - name: "var_msg" - type_name: "cel.expr.conformance.proto2.TestAllTypes" + type: "cel.expr.conformance.proto2.TestAllTypes" - name: "var_str" - type_name: "string" + type: "string" - name: "var_string_list" - type_name: "list" - params: - - type_name: "string" + type: "list" - name: "var_string_map" - type_name: "map" - params: - - type_name: "string" - - type_name: "bool" + type: "map" - name: "var_timestamp" - type_name: "timestamp" + type: "timestamp" - name: "var_uint" - type_name: "uint" + type: "uint" """), ) @@ -370,7 +349,7 @@ def test_config_augmented_variables(self): config = cel.NewEnvConfigFromYaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" """) env: cel.Env = cel.NewEnv( config=config, @@ -384,9 +363,9 @@ def test_config_augmented_variables(self): normalize_yaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" - name: "var_msg" - type_name: "cel.expr.conformance.proto2.TestAllTypes" + type: "cel.expr.conformance.proto2.TestAllTypes" """), ) @@ -394,7 +373,7 @@ def test_config_variable_override(self): config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" """) with self.assertRaises(Exception) as e: @@ -413,9 +392,9 @@ def test_config_variable_types(self): config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" - name: "var_int" - type_name: "int" + type: "int" value: 42 """) env: cel.Env = cel.NewEnv( @@ -560,11 +539,8 @@ def test_config_functions(self): functions: - name: is_ok overloads: - - id: "is_ok_string" - target: - type_name: string - return: - type_name: bool + - signature: "string.is_ok()" + return: "bool" """) env: cel.Env = cel.NewEnv( config=config, @@ -573,12 +549,8 @@ def test_config_functions(self): "hello", [ cel.Overload( - "good_time_of_day", + signature="hello(string,string)", return_type=cel.Type.STRING, - parameters=[ - cel.Type.STRING, - cel.Type.STRING, - ], impl=lambda ampm, arg: ( "Good" f" {'morning' if ampm == 'am' else 'afternoon'}," @@ -589,7 +561,7 @@ def test_config_functions(self): ) ], function_impls={ - "is_ok_string": lambda arg: arg in ["excellent", "good", "fair"], + "string.is_ok()": lambda arg: arg in ["excellent", "good", "fair"], }, ) yaml = env.config().to_yaml() @@ -599,19 +571,12 @@ def test_config_functions(self): functions: - name: "hello" overloads: - - id: "good_time_of_day" - args: - - type_name: "string" - - type_name: "string" - return: - type_name: "string" + - signature: "hello(string,string)" + return: "string" - name: "is_ok" overloads: - - id: "is_ok_string" - target: - type_name: "string" - return: - type_name: "bool" + - signature: "string.is_ok()" + return: "bool" """), ) res: cel.Value = env.compile("hello('am', 'Sunshine')").eval() @@ -628,32 +593,109 @@ def test_config_function_override(self): functions: - name: foo overloads: - - id: "unique_id" + - signature: "foo()" """) with self.assertRaises(Exception) as e: cel.NewEnv( config=config, functions=[ cel.FunctionDecl( - "bar", + "foo", [ cel.Overload( - "unique_id", + signature="foo()", impl=lambda: "hello", ) ], ) ], function_impls={ - "unique_id": lambda: "goodbye", + "foo()": lambda: "goodbye", }, ) self.assertIn( - "An implementation for function overload id 'unique_id' already" - " exists.", + "An implementation for function overload 'foo()' already exists.", str(e.exception), ) + def test_overload_signature_errors(self): + with self.assertRaises(ValueError) as e2: + cel.Overload(signature="greet(string)", parameters=[cel.Type.STRING]) + self.assertIn( + "If 'signature' is specified, 'parameters' should not be specified", + str(e2.exception), + ) + + with self.assertRaises(ValueError) as e3: + cel.Overload() + self.assertIn( + "Either 'id' or 'signature' must be specified", str(e3.exception) + ) + + def test_config_functions_deprecated_syntax(self): + """Test that the deprecated function syntax is still supported.""" + config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" + functions: + - name: is_ok + overloads: + - id: "is_ok_string" + target: + type_name: string + return: + type_name: bool + """) + env: cel.Env = cel.NewEnv( + config=config, + functions=[ + cel.FunctionDecl( + "hello", + [ + cel.Overload( + "good_time_of_day", + return_type=cel.Type.STRING, + parameters=[ + cel.Type.STRING, + cel.Type.STRING, + ], + impl=lambda ampm, arg: ( + "Good" + f" {'morning' if ampm == 'am' else 'afternoon'}," + f" {arg}!" + ), + ) + ], + ) + ], + function_impls={ + "is_ok_string": lambda arg: arg in ["excellent", "good", "fair"], + }, + ) + yaml = env.config().to_yaml() + self.assertEqual( + normalize_yaml(yaml), + normalize_yaml(""" + functions: + - name: "hello" + overloads: + - id: "good_time_of_day" + signature: "hello(string,string)" + return: "string" + - name: "is_ok" + overloads: + - id: "is_ok_string" + signature: "string.is_ok()" + return: "bool" + """), + ) + res: cel.Value = env.compile("hello('am', 'Sunshine')").eval() + self.assertEqual(res.value(), "Good morning, Sunshine!") + res = env.compile("hello('pm', 'tea is served')").eval() + self.assertEqual(res.value(), "Good afternoon, tea is served!") + res = env.compile("'good'.is_ok()").eval() + self.assertTrue(res.value()) + res = env.compile("'bad'.is_ok()").eval() + self.assertFalse(res.value()) + class TestCelExtension(cel.CelExtension): """An example CEL extension for testing.""" diff --git a/cel_expr_python/cel_test.py b/cel_expr_python/cel_test.py index f812fe4..938db60 100644 --- a/cel_expr_python/cel_test.py +++ b/cel_expr_python/cel_test.py @@ -668,6 +668,18 @@ def testTypeType(self): res.value(), cel.Type.INT ) # This behavior is counterintuitive but works as implemented. + def testTypeInitSignature(self): + self.assertEqual(cel.Type("int"), cel.Type.INT) + self.assertEqual(cel.Type("list"), cel.Type.List(cel.Type.INT)) + self.assertEqual( + cel.Type("map"), + cel.Type.Map(cel.Type.STRING, cel.Type.DYN), + ) + self.assertEqual( + cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + ) + def testCelExpressionPersistence_checkedExpr(self): expr: cel.Expression = self.env.compile("var_msg.single_string") as_bytes: bytes = expr.serialize() diff --git a/cel_expr_python/py_cel_env_config.cc b/cel_expr_python/py_cel_env_config.cc index ff111d8..8ea330c 100644 --- a/cel_expr_python/py_cel_env_config.cc +++ b/cel_expr_python/py_cel_env_config.cc @@ -37,7 +37,7 @@ void PyCelEnvConfig::DefinePythonBindings(pybind11::module& m) { &PyCelEnvConfig::GetContextType); } -PyCelEnvConfig PyCelEnvConfig::FromYaml(std::string yaml) { +PyCelEnvConfig PyCelEnvConfig::FromYaml(const std::string& yaml) { PyCelEnvConfig config; config.config_ = ThrowIfError(cel::EnvConfigFromYaml(yaml)); return config; @@ -45,7 +45,7 @@ PyCelEnvConfig PyCelEnvConfig::FromYaml(std::string yaml) { std::string PyCelEnvConfig::ToYaml() const { std::stringstream ss; - cel::EnvConfigToYaml(config_, ss); + cel::EnvConfigToYaml(config_, ss, {.use_type_signatures = true}); return ss.str(); } diff --git a/cel_expr_python/py_cel_env_config.h b/cel_expr_python/py_cel_env_config.h index 9d93bb8..ed30ea2 100644 --- a/cel_expr_python/py_cel_env_config.h +++ b/cel_expr_python/py_cel_env_config.h @@ -30,7 +30,7 @@ class PyCelEnvConfig { PyCelEnvConfig() = default; explicit PyCelEnvConfig(const cel::Config& config) : config_(config) {} - static PyCelEnvConfig FromYaml(std::string yaml); + static PyCelEnvConfig FromYaml(const std::string& yaml); std::string ToYaml() const; const cel::Config& GetConfig() const { return config_; } diff --git a/cel_expr_python/py_cel_env_internal.cc b/cel_expr_python/py_cel_env_internal.cc index a591580..af0391b 100644 --- a/cel_expr_python/py_cel_env_internal.cc +++ b/cel_expr_python/py_cel_env_internal.cc @@ -202,11 +202,11 @@ PyCelEnvInternal::NewCelEnvInternal( if (overload.py_function().is_none()) { continue; } - if (!impls.insert({overload.overload_id(), overload.py_function()}) - .second) { + std::string overload_id = overload.overload_id(); + if (!impls.insert({overload_id, overload.py_function()}).second) { return absl::AlreadyExistsError( - absl::StrCat("An implementation for function overload id '", - overload.overload_id(), "' already exists.")); + absl::StrCat("An implementation for function overload '", + overload_id, "' already exists.")); } } } @@ -214,8 +214,8 @@ PyCelEnvInternal::NewCelEnvInternal( for (const auto& [overload_id, py_function] : function_impls) { if (!impls.insert({overload_id, py_function}).second) { return absl::AlreadyExistsError( - absl::StrCat("An implementation for function overload id '", - overload_id, "' already exists.")); + absl::StrCat("An implementation for function overload '", overload_id, + "' already exists.")); } } return std::shared_ptr( diff --git a/cel_expr_python/py_cel_function_decl.cc b/cel_expr_python/py_cel_function_decl.cc index b5f084b..382fdae 100644 --- a/cel_expr_python/py_cel_function_decl.cc +++ b/cel_expr_python/py_cel_function_decl.cc @@ -16,10 +16,13 @@ #include #include +#include #include #include "env/config.h" +#include "env/type_info.h" #include "cel_expr_python/py_cel_overload.h" +#include "cel_expr_python/py_cel_type.h" #include #include diff --git a/cel_expr_python/py_cel_overload.cc b/cel_expr_python/py_cel_overload.cc index 5311f46..321af6c 100644 --- a/cel_expr_python/py_cel_overload.cc +++ b/cel_expr_python/py_cel_overload.cc @@ -20,8 +20,11 @@ #include #include +#include "common/signature.h" #include "env/config.h" +#include "env/type_info.h" #include "cel_expr_python/py_cel_type.h" +#include "cel_expr_python/py_error_status.h" #include #include @@ -31,16 +34,47 @@ namespace py = ::pybind11; void PyCelOverload::DefinePythonBindings(py::module_& m) { py::class_>(m, "Overload") - .def(py::init([](const std::string& overload_id, + .def(py::init([](std::optional id, const PyCelType& return_type, const std::vector& parameters, bool is_member, - py::object impl) { - return PyCelOverload(overload_id, return_type, parameters, - is_member, std::move(impl)); + py::object impl, std::optional signature) { + if (signature.has_value()) { + if (!parameters.empty()) { + throw py::value_error( + "If 'signature' is specified, 'parameters' " + "should not be specified"); + } + cel::ParsedFunctionOverload parsed = + ThrowIfError(cel::ParseFunctionSignature(*signature)); + std::string overload_id = + id.has_value() ? std::move(*id) : *signature; + std::vector parsed_parameters; + if (parsed.signature_type.has_function()) { + const auto& function_type_spec = + parsed.signature_type.function(); + parsed_parameters.reserve( + function_type_spec.arg_types().size()); + for (const auto& arg : function_type_spec.arg_types()) { + parsed_parameters.push_back(PyCelType::FromTypeInfo( + ThrowIfError(cel::TypeSpecToTypeInfo(arg)))); + } + } + return PyCelOverload(std::move(overload_id), return_type, + std::move(parsed_parameters), + parsed.is_member, std::move(impl)); + } + if (id.has_value()) { + return PyCelOverload(std::move(*id), return_type, parameters, + is_member, std::move(impl)); + } + throw py::value_error( + "Either 'id' or 'signature' must be specified"); }), - py::arg("overload_id"), py::arg("return_type") = PyCelType::Dyn(), + py::arg("id") = std::nullopt, + py::arg("return_type") = PyCelType::Dyn(), py::arg("parameters") = std::vector{}, - py::arg("is_member") = false, py::arg("impl") = py::none()); + py::arg("is_member") = false, py::arg("impl") = py::none(), + py::arg("signature") = std::nullopt); } PyCelOverload::PyCelOverload(std::string overload_id, diff --git a/cel_expr_python/py_cel_type.cc b/cel_expr_python/py_cel_type.cc index 0cf7b54..be49498 100644 --- a/cel_expr_python/py_cel_type.cc +++ b/cel_expr_python/py_cel_type.cc @@ -30,6 +30,7 @@ #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "common/kind.h" +#include "common/signature.h" #include "common/type.h" #include "common/type_kind.h" #include "common/types/list_type.h" @@ -37,6 +38,7 @@ #include "common/value.h" #include "common/value_kind.h" #include "env/config.h" +#include "env/type_info.h" #include "cel_expr_python/py_error_status.h" #include "cel_expr_python/status_macros.h" #include "google/protobuf/arena.h" @@ -50,7 +52,14 @@ namespace py = ::pybind11; void PyCelType::DefinePythonBindings(py::module& m) { py::class_ type(m, "Type"); - type.def(py::init(), py::arg("name")) + type.def(py::init([](const std::string& signature) { + cel::TypeSpec type_spec = + ThrowIfError(cel::ParseTypeSpec(signature)); + cel::Config::TypeInfo type_info = + ThrowIfError(cel::TypeSpecToTypeInfo(type_spec)); + return PyCelType::FromTypeInfo(type_info); + }), + py::arg("signature")) .def("name", &PyCelType::GetName) .def("is_message", &PyCelType::IsMessage) .def("is_assignable_from", &PyCelType::IsAssignableFrom) @@ -526,6 +535,47 @@ PyCelType PyCelType::FromTypeProto(const cel::expr::Type& type) { return PyCelType::Error(); } +PyCelType PyCelType::FromTypeInfo(const cel::Config::TypeInfo& type_info) { + if (type_info.name == "null") return PyCelType::Null(); + if (type_info.name == "bool") return PyCelType::Bool(); + if (type_info.name == "int") return PyCelType::Int(); + if (type_info.name == "uint") return PyCelType::Uint(); + if (type_info.name == "double") return PyCelType::Double(); + if (type_info.name == "string") return PyCelType::String(); + if (type_info.name == "bytes") return PyCelType::Bytes(); + if (type_info.name == "timestamp") return PyCelType::Timestamp(); + if (type_info.name == "duration") return PyCelType::Duration(); + if (type_info.name == "dyn") return PyCelType::Dyn(); + if (type_info.name == "list") { + if (type_info.params.empty()) { + return PyCelType::List(); + } + return PyCelType::ListType(FromTypeInfo(type_info.params[0])); + } + if (type_info.name == "map") { + if (type_info.params.size() < 2) { + return PyCelType::Map(); + } + return PyCelType::MapType(FromTypeInfo(type_info.params[0]), + FromTypeInfo(type_info.params[1])); + } + if (type_info.name == "type") { + if (type_info.params.empty()) { + return PyCelType::Type(); + } + return PyCelType::TypeType(FromTypeInfo(type_info.params[0])); + } + if (type_info.is_type_param || !type_info.params.empty()) { + std::vector params; + params.reserve(type_info.params.size()); + for (const auto& param : type_info.params) { + params.push_back(FromTypeInfo(param)); + } + return PyCelType::AbstractType(type_info.name, params); + } + return PyCelType(type_info.name); +} + absl::StatusOr PyCelType::ToCelType( const PyCelType& type, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& descriptor_pool) { diff --git a/cel_expr_python/py_cel_type.h b/cel_expr_python/py_cel_type.h index 892c488..19cac0e 100644 --- a/cel_expr_python/py_cel_type.h +++ b/cel_expr_python/py_cel_type.h @@ -77,6 +77,7 @@ class PyCelType { static PyCelType ForCelValue(const cel::Value& cel_value); static PyCelType FromCelType(const cel::Type& cel_type); static PyCelType FromTypeProto(const cel::expr::Type& type); + static PyCelType FromTypeInfo(const cel::Config::TypeInfo& type_info); static absl::StatusOr ToCelType( const PyCelType& type, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& descriptor_pool); diff --git a/codelab/index.lab.md b/codelab/index.lab.md index eb5ef2f..5ca5fba 100644 --- a/codelab/index.lab.md +++ b/codelab/index.lab.md @@ -846,14 +846,8 @@ def exercise5(): "contains", [ cel.Overload( - "contains_key_value", + signature="map.contains(string, dyn)", return_type=cel.Type.BOOL, - parameters=[ - cel.Type.Map(cel.Type.STRING, cel.Type.DYN), - cel.Type.STRING, - cel.Type.DYN, - ], - is_member=True, # Provide the implementation as a Python function ) ], @@ -922,14 +916,8 @@ def exercise5(): "contains", [ cel.Overload( - "contains_string_any", + signature="map.contains(string, dyn)", return_type=cel.Type.BOOL, - parameters=[ - cel.Type.Map(cel.Type.STRING, cel.Type.DYN), - cel.Type.STRING, - cel.Type.DYN, - ], - is_member=True, # Reference a Python function impl=contains_key_value, ) @@ -1319,7 +1307,7 @@ def exercise7(): # Add variable definitions for 'jwt' as a map(string, Dyn) type # and for 'now' as a timestamp. variables={ - "jwt": cel.Type.Map(cel.Type.STRING, cel.Type.DYN), + "jwt": cel.Type("map"), "now": cel.Type.TIMESTAMP, }, ) diff --git a/codelab/solution/codelab.py b/codelab/solution/codelab.py index 15d12c0..861f0ef 100644 --- a/codelab/solution/codelab.py +++ b/codelab/solution/codelab.py @@ -188,14 +188,11 @@ def exercise5(): "containsKeyValue", [ cel.Overload( - "contains_key_value", + signature=( + "map.containsKeyValue(string," + " dyn)" + ), return_type=cel.Type.BOOL, - parameters=[ - cel.Type.Map(cel.Type.STRING, cel.Type.DYN), - cel.Type.STRING, - cel.Type.DYN, - ], - is_member=True, impl=contains_key_value, ) ], @@ -323,7 +320,7 @@ def exercise7(): # Add variable definitions for 'jwt' as a map(string, Dyn) type # and for 'now' as a timestamp. variables={ - "jwt": cel.Type.Map(cel.Type.STRING, cel.Type.DYN), + "jwt": cel.Type("map"), "now": cel.Type.TIMESTAMP, }, ) From 584be4f067a95387c291b34e03c55431efdcc138 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 22 Jun 2026 15:07:51 -0700 Subject: [PATCH 11/23] Fix build PiperOrigin-RevId: 936270519 --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 7404b9f..089d1b1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,7 +15,7 @@ bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "cel-cpp", version = "0.15.0", repo_name = "com_google_cel_cpp") git_override( module_name = "cel-cpp", - commit = "8b7068abb4062074a135491ad8357139287084f9", + commit = "76ae0b3c1768d93a10270f904101de338867bdb1", remote = "https://github.com/cel-expr/cel-cpp", ) From ed5fb185c730c21a974641ea5393031f3fa653ac Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 24 Jun 2026 09:51:28 -0700 Subject: [PATCH 12/23] Update release scripts and pyproject.toml for ARM and Python 3.14 support PiperOrigin-RevId: 937397017 --- .bazelrc | 16 ++++++---- release/kokoro/release_linux.cfg | 2 +- release/kokoro/release_linux.sh | 51 +++++++++++++++++++++++++++--- release/kokoro/release_macos.sh | 2 ++ release/kokoro/release_windows.bat | 6 ++++ release/pyproject.toml | 6 +++- release/setup.py | 3 ++ 7 files changed, 74 insertions(+), 12 deletions(-) diff --git a/.bazelrc b/.bazelrc index 5c70e99..333e7bb 100644 --- a/.bazelrc +++ b/.bazelrc @@ -28,13 +28,17 @@ common:windows --experimental_repository_downloader_retries=10 build --verbose_failures test --test_output=errors -# GCS remote caching config (Windows-only, active by default on Windows!) -build:windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache -build:windows --google_default_credentials=true +# GCS remote caching config (Linux-only, opt-in) +build:remote-cache-linux --remote_cache=https://storage.googleapis.com/linux-cel-python-remote-cache +build:remote-cache-linux --google_default_credentials=true -# GCS remote caching config (macOS-only, active by default on macOS!) -build:macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache -build:macos --google_default_credentials=true +# GCS remote caching config (Windows-only, opt-in) +build:remote-cache-windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache +build:remote-cache-windows --google_default_credentials=true + +# GCS remote caching config (macOS-only, opt-in) +build:remote-cache-macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache +build:remote-cache-macos --google_default_credentials=true # Silence deprecation warnings from external dependencies (Linux and macOS) build:linux --cxxopt=-Wno-deprecated-declarations diff --git a/release/kokoro/release_linux.cfg b/release/kokoro/release_linux.cfg index a42e2e3..018c022 100644 --- a/release/kokoro/release_linux.cfg +++ b/release/kokoro/release_linux.cfg @@ -5,6 +5,6 @@ build_file: "cel-python/release/kokoro/release_linux.sh" timeout_mins: 120 container_properties { - docker_image: "us-central1-docker.pkg.dev/kokoro-container-bakery/kokoro/ubuntu/ubuntu2204/ktcb:current" + docker_image: "mirror.gcr.io/library/ubuntu:24.04" docker_sibling_containers: true } diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 5292ef6..ff27fa7 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -15,12 +15,42 @@ set -e +if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null; then + echo "Installing basic dependencies..." + apt-get update && apt-get install -y python3-pip curl + + if ! command -v docker &> /dev/null; then + echo "Installing docker CLI..." + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + DOCKER_ARCH="x86_64" + elif [ "$ARCH" = "aarch64" ]; then + DOCKER_ARCH="aarch64" + else + echo "Unsupported arch: $ARCH" + exit 1 + fi + curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-24.0.7.tgz" -o docker.tgz + tar xzvf docker.tgz --strip-components=1 docker/docker + mv docker /usr/local/bin/ + rm -f docker.tgz + fi +fi + # Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host export VIRTUALENV_NO_DOWNLOAD=1 export PIP_DISABLE_PIP_VERSION_CHECK=1 +export PIP_BREAK_SYSTEM_PACKAGES=1 +export PIP_DEFAULT_TIMEOUT=60 + +if [ "$(uname -m)" = "aarch64" ]; then + export CIBW_ARCHS="aarch64" +else + export CIBW_ARCHS="x86_64" +fi # Pass these environment variables to the cibuildwheel Docker container -export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1" +export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_DEFAULT_TIMEOUT=120 CEL_BAZEL_FLAGS=--config=remote-cache-linux" export CIBW_DEPENDENCY_VERSIONS="latest" export CIBW_CONTAINER_ENGINE_EXTRA_ARGS="--network=host" @@ -35,7 +65,10 @@ fi # cache. In a sandboxed build environment, writing to the default cache directory # (~/.cache/pip) can encounter permission/sandbox restrictions or lead to stale # dependency resolution. Disabling the cache ensures a reliable, reproducible install. -pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel +pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine +curl -fsSL https://github.com/pypa/cibuildwheel/archive/refs/tags/v4.1.0.tar.gz -o cibuildwheel-4.1.0.tar.gz +pip install --no-cache-dir cibuildwheel-4.1.0.tar.gz +rm -f cibuildwheel-4.1.0.tar.gz # ============================================================================== # FUTURE-PROOF RUNTIME PATCHING OF CIBUILDWHEEL @@ -203,9 +236,19 @@ rm -rf cel_expr_python/*_test.py echo "Downloading bazelisk on host..." curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 -chmod +x bazelisk-linux-amd64 +curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-arm64 +chmod +x bazelisk-linux-amd64 bazelisk-linux-arm64 + +echo "Downloading build dependencies on host..." +mkdir -p build_deps +pip download --no-cache-dir --only-binary=:all: --dest build_deps "setuptools>=40.8.0" "wheel" +if [ "$(uname -m)" = "aarch64" ]; then + PLATFORM_SUFFIX="aarch64" +else + PLATFORM_SUFFIX="x86_64" +fi +pip download --no-cache-dir --only-binary=:all: --dest build_deps --python-version 3.9 --platform "manylinux2014_${PLATFORM_SUFFIX}" "virtualenv" "typing-extensions>=4.13.2" -# Check if pyproject.toml exists before running sed if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi diff --git a/release/kokoro/release_macos.sh b/release/kokoro/release_macos.sh index 71d4805..54b92fa 100644 --- a/release/kokoro/release_macos.sh +++ b/release/kokoro/release_macos.sh @@ -65,6 +65,8 @@ if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi +export CEL_BAZEL_FLAGS="--config=remote-cache-macos" + echo "Running cibuildwheel: ${CIBWHEEL_BIN}" # Default CIBWHEEL_BIN if not set if [ -z "${CIBWHEEL_BIN}" ]; then diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index 9f95c7e..75fc8fa 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -16,6 +16,11 @@ setlocal enabledelayedexpansion set "RELEASE_STATUS=0" set "FETCH_RETRIES=10" set "FETCH_RETRY_DELAY_S=10" +echo --- Installing Python 3.11 via Chocolatey --- +choco install python311 -y --no-progress +if !ERRORLEVEL! NEQ 0 ( + echo WARNING: Failed to install Python 3.11 via Chocolatey. +) echo === Loading Environment Configuration === call "%~dp0set_env_windows.bat" @@ -145,6 +150,7 @@ if !FETCH_STATUS! NEQ 0 ( if exist fetch.log del fetch.log echo --- Running cibuildwheel --- +set "CEL_BAZEL_FLAGS=--config=remote-cache-windows" if "%CIBWHEEL_BIN%" == "" ( set "CIBWHEEL_BIN=!PYTHON_EXE! -m cibuildwheel" ) diff --git a/release/pyproject.toml b/release/pyproject.toml index c6b322a..d7aa211 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -44,14 +44,18 @@ test-command = "python {project}/cel_basic_test.py" build-verbosity = 1 [tool.cibuildwheel.linux] +build-frontend = { name = "pip", args = ["--no-build-isolation"] } +before-build = "pip install --no-index --find-links={project}/build_deps setuptools wheel virtualenv" +archs = ["x86_64", "aarch64"] manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" container-engine = "docker; disable_host_mount: True" # Google's internal Kokoro/RBE network uses a secure MITM proxy that resigns HTTPS # traffic with an internal Google CA. Since the public manylinux container does not # trust this CA, git fetches for external dependencies (like @cel-cpp) will fail # with SSL certificate errors. We disable http.sslVerify inside the container to # bypass this and allow Bazel to fetch SCM dependencies through the proxy. -before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel" +before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && if [ $(uname -m) = 'aarch64' ]; then cp {project}/bazelisk-linux-arm64 /usr/local/bin/bazel; else cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel; fi && python3 -m pip install --no-index --find-links={project}/build_deps virtualenv" [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] diff --git a/release/setup.py b/release/setup.py index 09cfe81..e2d0962 100644 --- a/release/setup.py +++ b/release/setup.py @@ -72,6 +72,9 @@ def build_extension(self, ext): # Build with bazel # Use --compilation_mode=opt for release builds cmd = ['bazel', 'build', ext.target, '--compilation_mode=opt'] + extra_flags = os.environ.get('CEL_BAZEL_FLAGS') + if extra_flags: + cmd.extend(extra_flags.split()) if sys.platform == 'win32': self.platform_config_windows(cmd, python_version) if sys.platform == 'darwin': From b8e324c87460ee55d25ed28405119a97700d0a4d Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 24 Jun 2026 14:44:43 -0700 Subject: [PATCH 13/23] Fix release scripts for cel-python. - In release_linux.sh, install git because it is needed to clone the repository but not present in the default ubuntu image. - In release_windows.bat, restructure SCM resolution to avoid using batch labels inside parenthesized if/else blocks, which causes "The system cannot find the batch label specified" errors. PiperOrigin-RevId: 937568390 --- release/kokoro/release_linux.sh | 4 +- release/kokoro/release_windows.bat | 75 +++++++++++++++++------------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index ff27fa7..6375ed2 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -15,9 +15,9 @@ set -e -if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null; then +if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null || ! command -v git &> /dev/null; then echo "Installing basic dependencies..." - apt-get update && apt-get install -y python3-pip curl + apt-get update && apt-get install -y python3-pip curl git if ! command -v docker &> /dev/null; then echo "Installing docker CLI..." diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index 75fc8fa..8b6f003 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -51,42 +51,51 @@ echo Created temporary directories: %REPO_DIR%, %TMP_DIR% mkdir "%TMP_DIR%" echo --- Resolving Repository Source --- -if "%DRY_RUN%" == "true" ( - echo [DRY RUN] Using local Kokoro clone instead of cloning main. - set "SRC_DIR=%~dp0..\.." - pushd "!SRC_DIR!" - for /f "tokens=*" %%i in ('git tag --sort=-v:refname 2^>nul') do ( - set "VERSION=%%i" - goto :got_local_tag - ) - set "VERSION=0.1.2" - :got_local_tag +if "%DRY_RUN%" == "true" goto resolution_dry +goto resolution_real + +:resolution_dry +echo [DRY RUN] Using local Kokoro clone instead of cloning main. +set "SRC_DIR=%~dp0..\.." +pushd "!SRC_DIR!" +set "VERSION=" +for /f "tokens=*" %%i in ('git tag --sort=-v:refname 2^>nul') do ( + set "VERSION=%%i" + goto got_local_tag +) +:got_local_tag +if "%VERSION%" == "" set "VERSION=0.1.2" +popd +goto resolution_done + +:resolution_real +mkdir "%REPO_DIR%" +pushd "%REPO_DIR%" +git clone https://github.com/cel-expr/cel-python.git +if !ERRORLEVEL! NEQ 0 ( + echo Failed to clone repository! + set "RELEASE_STATUS=1" popd -) else ( - mkdir "%REPO_DIR%" - pushd "%REPO_DIR%" - git clone https://github.com/cel-expr/cel-python.git - if !ERRORLEVEL! NEQ 0 ( - echo Failed to clone repository! - set "RELEASE_STATUS=1" - popd - goto cleanup - ) - cd cel-python - for /f "tokens=*" %%i in ('git tag --sort=-v:refname') do ( - set "VERSION=%%i" - goto :got_tag - ) - :got_tag - if "%VERSION%" == "" ( - echo Failed to get version tag! - set "RELEASE_STATUS=1" - popd - goto cleanup - ) - set "SRC_DIR=%REPO_DIR%\cel-python" + goto cleanup +) +cd cel-python +set "VERSION=" +for /f "tokens=*" %%i in ('git tag --sort=-v:refname') do ( + set "VERSION=%%i" + goto got_tag +) +:got_tag +if "%VERSION%" == "" ( + echo Failed to get version tag! + set "RELEASE_STATUS=1" popd + goto cleanup ) +set "SRC_DIR=%REPO_DIR%\cel-python" +popd +goto resolution_done + +:resolution_done if "%VERSION:~0,1%" == "v" ( set "VERSION=%VERSION:~1%" From 6375d5fa41695c41dd006460296c1f6f6ce004e2 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 25 Jun 2026 10:00:12 -0700 Subject: [PATCH 14/23] Fix git SSL certificate verification in release_linux.sh. PiperOrigin-RevId: 938044930 --- release/kokoro/release_linux.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 6375ed2..868dd70 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -17,7 +17,8 @@ set -e if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null || ! command -v git &> /dev/null; then echo "Installing basic dependencies..." - apt-get update && apt-get install -y python3-pip curl git + apt-get update && apt-get install -y python3-pip curl git ca-certificates + git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt if ! command -v docker &> /dev/null; then echo "Installing docker CLI..." From 3e369fe3dfd29ef2fcb3d8a63997e58f50e592ae Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 25 Jun 2026 15:12:50 -0700 Subject: [PATCH 15/23] Internal change PiperOrigin-RevId: 938218460 --- release/kokoro/release_linux.sh | 44 ++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 868dd70..119ec14 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -15,27 +15,36 @@ set -e +echo "Updating ca-certificates..." +apt-get update && apt-get install -y ca-certificates + if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null || ! command -v git &> /dev/null; then echo "Installing basic dependencies..." - apt-get update && apt-get install -y python3-pip curl git ca-certificates - git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt + apt-get install -y python3-pip curl git +fi - if ! command -v docker &> /dev/null; then - echo "Installing docker CLI..." - ARCH=$(uname -m) - if [ "$ARCH" = "x86_64" ]; then - DOCKER_ARCH="x86_64" - elif [ "$ARCH" = "aarch64" ]; then - DOCKER_ARCH="aarch64" - else - echo "Unsupported arch: $ARCH" - exit 1 - fi - curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-24.0.7.tgz" -o docker.tgz - tar xzvf docker.tgz --strip-components=1 docker/docker - mv docker /usr/local/bin/ - rm -f docker.tgz +if ! command -v docker &> /dev/null; then + echo "Installing docker CLI..." + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + DOCKER_ARCH="x86_64" + elif [ "$ARCH" = "aarch64" ]; then + DOCKER_ARCH="aarch64" + else + echo "Unsupported arch: $ARCH" + exit 1 fi + curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-24.0.7.tgz" -o docker.tgz + tar xzvf docker.tgz --strip-components=1 docker/docker + mv docker /usr/local/bin/ + rm -f docker.tgz +fi + +if [ -f /var/cache/proxy.crt ]; then + echo "Using proxy certificate for Git..." + git config --global http.sslCAinfo /var/cache/proxy.crt +else + git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt fi # Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host @@ -206,6 +215,7 @@ if [ "${DRY_RUN}" = "true" ]; then VERSION="0.1.2" fi popd + else pushd "${REPO_DIR}" git clone https://github.com/cel-expr/cel-python.git From 5f795e53a0637da105784a981a98754ba631e0d5 Mon Sep 17 00:00:00 2001 From: Tristan Swadell Date: Fri, 24 Jul 2026 13:44:50 -0700 Subject: [PATCH 16/23] Update CEL Env tests for Python to cover more extensions PiperOrigin-RevId: 953535431 --- cel_expr_python/cel_env_test.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index eacfc9f..af1de62 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -490,6 +490,34 @@ def test_config_extensions(self): res = env.compile("hello('World')").eval() self.assertEqual(res.value(), "Hello, World!") + def test_standard_extensions_via_config(self): + config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" + extensions: + - name: lists + - name: sets + - name: two-var-comprehensions + - name: optional + - name: regex + """) + env: cel.Env = cel.NewEnv(config=config) + + # Test lists extension (e.g., reverse) + res = env.compile("[1, 2].reverse()").eval() + self.assertEqual(res.plain_value(), [2, 1]) + + # Test sets extension (e.g., intersects) + res = env.compile("sets.intersects([1, 2], [2, 3])").eval() + self.assertEqual(res.value(), True) + + # Test two-var-comprehensions (e.g., all) + res = env.compile("[1, 2].all(i, v, v > 0)").eval() + self.assertEqual(res.value(), True) + + # Test regex extension (e.g., replace) + expr = r"regex.replace('123-456', r'(\d+)-(\d+)', r'\2-\1')" + res = env.compile(expr).eval() + self.assertEqual(res.value(), "456-123") + def test_config_extension_override_same_version(self): config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" extensions: From 2ae52f45a4ea900cc34089a3f4743479dd485b8c Mon Sep 17 00:00:00 2001 From: Amer Elsheikh Date: Thu, 6 Aug 2026 07:57:25 -0700 Subject: [PATCH 17/23] Adding type suppressions for pyrefly PiperOrigin-RevId: 960302526 --- cel_expr_python/cel_env_test.py | 4 ++-- cel_expr_python/cel_test.py | 2 +- conformance/conformance_test.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index af1de62..e0cb597 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -461,9 +461,9 @@ def test_config_extension_version_out_of_range(self): for test_case in cases: with self.assertRaises(Exception) as e: cel.NewEnv( - extensions=[test_case[0]()], + extensions=[test_case[0]()], # pyrefly: ignore[not-callable] ) - self.assertRegex(str(e.exception), test_case[1]) + self.assertRegex(str(e.exception), test_case[1]) # pyrefly: ignore[bad-argument-type] def test_config_extensions(self): config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" diff --git a/cel_expr_python/cel_test.py b/cel_expr_python/cel_test.py index 938db60..e8f6e47 100644 --- a/cel_expr_python/cel_test.py +++ b/cel_expr_python/cel_test.py @@ -210,7 +210,7 @@ def testEvalDouble(self): res: cel.Value = self._eval("3.14", expected_return_type=cel.Type.DOUBLE) self.assertEqual(res.type(), cel.Type.DOUBLE) self.assertAlmostEqual(res.value(), 3.14, places=3) - self.assertAlmostEqual(res.plain_value(), 3.14, places=3) + self.assertAlmostEqual(res.plain_value(), 3.14, places=3) # pyrefly: ignore[no-matching-overload] res = self._eval("3.14 / 2.0") self.assertAlmostEqual(res.value(), 1.57, places=3) res = self._eval( diff --git a/conformance/conformance_test.py b/conformance/conformance_test.py index 35deeb4..d0b30e6 100644 --- a/conformance/conformance_test.py +++ b/conformance/conformance_test.py @@ -184,7 +184,7 @@ def _run_conformance_test(self, simple_test: simple_pb.SimpleTest): extensions: list[cel.CelExtensionBase] = [] for key, value in self.EXTENSIONS_PER_TESTFILE.items(): if self._testMethodName.find(key) != -1: - extensions = value + extensions = value # pyrefly: ignore[bad-assignment] break self.env = cel.NewEnv( From 604cc57bb20bc0a5de7a7190bd45a15cdf028e09 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 17 Aug 2026 12:08:34 -0700 Subject: [PATCH 18/23] Add support for environment options in Python CEL. This change allows passing an optional dictionary of options to EnvConfig and NewEnv. It exposes an options property on EnvConfig and uses these options to configure the underlying CEL compiler builder, specifically supporting the "enable_pratt_parser" option. PiperOrigin-RevId: 966107832 --- MODULE.bazel | 10 ++-- cel_expr_python/BUILD | 2 + cel_expr_python/cel.pyi | 16 ++++++- cel_expr_python/cel_env_test.py | 14 ++++++ cel_expr_python/cel_test.py | 61 +++++++++++++++++------- cel_expr_python/py_cel_env.cc | 18 ++++++-- cel_expr_python/py_cel_env.h | 4 +- cel_expr_python/py_cel_env_internal.cc | 13 ++++-- cel_expr_python/py_cel_env_internal.h | 10 ++-- cel_expr_python/py_cel_module.cc | 2 + cel_expr_python/py_cel_options.cc | 64 ++++++++++++++++++++++++++ cel_expr_python/py_cel_options.h | 32 +++++++++++++ 12 files changed, 213 insertions(+), 33 deletions(-) create mode 100644 cel_expr_python/py_cel_options.cc create mode 100644 cel_expr_python/py_cel_options.h diff --git a/MODULE.bazel b/MODULE.bazel index 089d1b1..a47ae41 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -3,7 +3,11 @@ module( ) # https://registry.bazel.build/modules/abseil-cpp -bazel_dep(name = "abseil-cpp", version = "20260107.0", repo_name = "com_google_absl") +bazel_dep(name = "abseil-cpp", version = "20260526.0", repo_name = "com_google_absl") +single_version_override( + module_name = "abseil-cpp", + version = "20260526.0", +) # https://registry.bazel.build/modules/abseil-py bazel_dep(name = "abseil-py", version = "2.4.0", repo_name = "com_google_absl_py") @@ -12,10 +16,10 @@ bazel_dep(name = "abseil-py", version = "2.4.0", repo_name = "com_google_absl_py bazel_dep(name = "bazel_skylib", version = "1.9.0") # https://registry.bazel.build/modules/cel-cpp -bazel_dep(name = "cel-cpp", version = "0.15.0", repo_name = "com_google_cel_cpp") +bazel_dep(name = "cel-cpp", version = "0.16.1", repo_name = "com_google_cel_cpp") git_override( module_name = "cel-cpp", - commit = "76ae0b3c1768d93a10270f904101de338867bdb1", + commit = "1dcff093b3ca3d575f982e2ca154f9c751e06055", remote = "https://github.com/cel-expr/cel-cpp", ) diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index 190c18a..acb75ab 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -20,6 +20,7 @@ pybind_library( "py_cel_expression.cc", "py_cel_function.cc", "py_cel_function_decl.cc", + "py_cel_options.cc", "py_cel_overload.cc", "py_cel_python_extension.cc", "py_cel_type.cc", @@ -36,6 +37,7 @@ pybind_library( "py_cel_expression.h", "py_cel_function.h", "py_cel_function_decl.h", + "py_cel_options.h", "py_cel_overload.h", "py_cel_python_extension.h", "py_cel_type.h", diff --git a/cel_expr_python/cel.pyi b/cel_expr_python/cel.pyi index f5cc7c8..ab151bb 100644 --- a/cel_expr_python/cel.pyi +++ b/cel_expr_python/cel.pyi @@ -12,6 +12,10 @@ class CelExtension(CelExtensionBase): class CelExtensionBase: def __init__(self, name: str) -> None: ... +class Options: + enable_pratt_parser: bool + def __init__(self, enable_pratt_parser: bool = ...) -> None: ... + class EnvConfig: @property def context_type(self) -> str: ... @@ -26,6 +30,7 @@ class Env: def compile(self, expression: str, disable_check: bool = ...) -> Expression: ... def deserialize(self, serialized: str | bytes) -> Expression: ... def config(self) -> EnvConfig: ... + def options(self) -> Options: ... class Expression: def eval(self, activation: Activation | None = ..., data: Mapping[str, Any] | None = ..., functions=..., arena: _InternalArena = ...) -> Value: ... @@ -86,6 +91,15 @@ class _InternalArena: def Arena() -> _InternalArena: ... -def NewEnv(descriptor_pool: proto_descriptor_pool.DescriptorPool | Any | None = ..., config: EnvConfig | None = ..., variables: Mapping[str, Type] | None = ..., extensions: Sequence[CelExtensionBase] | None = ..., container: str | ExpressionContainer | None = ..., functions: Sequence[FunctionDecl] | None = ..., function_impls: Mapping[str, Callable[..., Any]] | None = ...) -> Env: ... +def NewEnv( + descriptor_pool: proto_descriptor_pool.DescriptorPool | Any | None = ..., + config: EnvConfig | None = ..., + variables: Mapping[str, Type] | None = ..., + extensions: Sequence[CelExtensionBase] | None = ..., + container: str | ExpressionContainer | None = ..., + functions: Sequence[FunctionDecl] | None = ..., + function_impls: Mapping[str, Callable[..., Any]] | None = ..., + options: Options | None = ..., +) -> Env: ... def NewEnvConfigFromYaml(yaml: str) -> EnvConfig: ... diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index e0cb597..eabe446 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -724,6 +724,20 @@ def test_config_functions_deprecated_syntax(self): res = env.compile("'bad'.is_ok()").eval() self.assertFalse(res.value()) + def test_env_options(self): + options = cel.Options(enable_pratt_parser=True) + self.assertTrue(options.enable_pratt_parser) + self.assertEqual(repr(options), "Options(enable_pratt_parser=True)") + options.enable_pratt_parser = False + self.assertFalse(options.enable_pratt_parser) + self.assertEqual(repr(options), "Options(enable_pratt_parser=False)") + + env = cel.NewEnv(options=cel.Options(enable_pratt_parser=True)) + self.assertTrue(env.options().enable_pratt_parser) + + default_env = cel.NewEnv() + self.assertFalse(default_env.options().enable_pratt_parser) + class TestCelExtension(cel.CelExtension): """An example CEL extension for testing.""" diff --git a/cel_expr_python/cel_test.py b/cel_expr_python/cel_test.py index e8f6e47..2add977 100644 --- a/cel_expr_python/cel_test.py +++ b/cel_expr_python/cel_test.py @@ -28,7 +28,9 @@ from cel.expr.conformance.proto2 import test_all_types_pb2 as test_all_types_pb -class CelTest(absltest.TestCase): +@absltest.skipThisClass("Base class") +class _CelTestBase(absltest.TestCase): + options: cel.Options = cel.Options() def setUp(self): super().setUp() @@ -50,7 +52,8 @@ def setUp(self): "var_string_map": cel.Type.Map(cel.Type.STRING, cel.Type.BOOL), "var_dyn_map": cel.Type.MAP, "var_dyn": cel.Type.DYN, - } + }, + options=self.options, ) self.object_counts_before_test = self._grab_object_counts() @@ -615,10 +618,13 @@ def testDynType(self): self.assertIn("out of range for 'var_dyn'", res.value()) def testDynType_nonCelType(self): - res = self._eval("var_dyn", {"var_dyn": self}) + class NonCelValue: + pass + + res = self._eval("var_dyn", {"var_dyn": NonCelValue()}) self.assertEqual(res.type(), cel.Type.ERROR) self.assertIn( - "Non-CEL value type for 'var_dyn': CelTest", + "Non-CEL value type for 'var_dyn': NonCelValue", res.value(), ) @@ -768,18 +774,26 @@ def testCompilationErrorHandling(self): # Check parser error. with self.assertRaises(Exception) as e: self.env.compile("'Hello,' # 'World!'", disable_check=True) - self.assertIn( - "1:10: Syntax error: token recognition error at: '#'\n " - "| 'Hello,' # 'World!'\n " - "| .........^", - str(e.exception), - ) - self.assertIn( - "1:12: Syntax error: extraneous input ''World!'' expecting \n " - "| 'Hello,' # 'World!'\n " - "| ...........^", - str(e.exception), - ) + if self.options.enable_pratt_parser: + self.assertIn( + "1:10: unexpected character\n" + " | 'Hello,' # 'World!'\n" + " | .........^", + str(e.exception), + ) + else: + self.assertIn( + "1:10: Syntax error: token recognition error at: '#'\n " + "| 'Hello,' # 'World!'\n " + "| .........^", + str(e.exception), + ) + self.assertIn( + "1:12: Syntax error: extraneous input ''World!'' expecting \n " + "| 'Hello,' # 'World!'\n " + "| ...........^", + str(e.exception), + ) # Check type-checker error. with self.assertRaises(Exception) as e: @@ -793,7 +807,11 @@ def testCompilationErrorHandling(self): ) def testErrorHandling(self): - bad_env = cel.NewEnv(_BadDescriptorPool(), variables={}) + bad_env = cel.NewEnv( + _BadDescriptorPool(), + variables={}, + options=self.options, + ) with self.assertRaises(Exception) as e: bad_env.compile("cel.expr.conformance.proto2.TestSomeTypes{}") self.assertRegex( @@ -929,5 +947,14 @@ def testErrorOnProtoCreation(self): ) +class CelTest(_CelTestBase): + # Default options. + pass + + +class CelPrattParserTest(_CelTestBase): + options = cel.Options(enable_pratt_parser=True) + + if __name__ == "__main__": absltest.main() diff --git a/cel_expr_python/py_cel_env.cc b/cel_expr_python/py_cel_env.cc index fb0fd69..30758cf 100644 --- a/cel_expr_python/py_cel_env.cc +++ b/cel_expr_python/py_cel_env.cc @@ -32,6 +32,7 @@ #include "cel_expr_python/py_cel_env_internal.h" #include "cel_expr_python/py_cel_expression.h" #include "cel_expr_python/py_cel_function_decl.h" +#include "cel_expr_python/py_cel_options.h" #include "cel_expr_python/py_cel_type.h" #include "cel_expr_python/py_error_status.h" #include @@ -81,7 +82,8 @@ void PyCelEnv::DefinePythonBindings(pybind11::module& m) { std::optional>>& functions, std::optional>& - function_impls) { + function_impls, + std::optional& options) { PyObject* pool_ptr; if (descriptor_pool.is_none()) { // Replicates python's `descriptor_pool.Default()` @@ -119,7 +121,10 @@ void PyCelEnv::DefinePythonBindings(pybind11::module& m) { } } - return PyCelEnv(config.value_or(PyCelEnvConfig()), pool_ptr, + PyCelOptions env_options = options.value_or(PyCelOptions()); + + return PyCelEnv(config.value_or(PyCelEnvConfig()), env_options, + pool_ptr, std::move(variables).value_or( std::unordered_map{}), ext_ptrs, std::move(expr_container), @@ -131,10 +136,12 @@ void PyCelEnv::DefinePythonBindings(pybind11::module& m) { py::arg("descriptor_pool") = py::none(), py::arg("config") = py::none(), py::arg("variables") = py::none(), py::arg("extensions") = py::none(), py::arg("container") = py::none(), py::arg("functions") = py::none(), - py::arg("function_impls") = py::none()); + py::arg("function_impls") = py::none(), py::arg("options") = py::none()); cel_class .def("config", [](PyCelEnv& self) { return self.GetEnv()->GetEnvConfig(); }) + .def("options", + [](PyCelEnv& self) { return self.GetEnv()->GetOptions(); }) .def("compile", &PyCelEnv::Compile, py::arg("expression"), py::arg("disable_check") = false) .def("deserialize", &PyCelEnv::Deserialize, py::arg("serialized")) @@ -165,14 +172,15 @@ void PyCelEnv::DefinePythonBindings(pybind11::module& m) { } PyCelEnv::PyCelEnv( - const PyCelEnvConfig& config, PyObject* descriptor_pool, + const PyCelEnvConfig& config, const PyCelOptions& options, + PyObject* descriptor_pool, const std::unordered_map& variable_types, const std::vector& extensions, cel::ExpressionContainer container, const std::vector>& functions, const std::unordered_map& function_impls) { env_ = ThrowIfError(PyCelEnvInternal::NewCelEnvInternal( - config, descriptor_pool, std::move(variable_types), extensions, + config, options, descriptor_pool, std::move(variable_types), extensions, std::move(container), std::move(functions), std::move(function_impls))); ABSL_CHECK(PyGILState_Check()); } diff --git a/cel_expr_python/py_cel_env.h b/cel_expr_python/py_cel_env.h index 2eac651..b6fc4a0 100644 --- a/cel_expr_python/py_cel_env.h +++ b/cel_expr_python/py_cel_env.h @@ -30,6 +30,7 @@ #include "cel_expr_python/py_cel_expression.h" #include "cel_expr_python/py_cel_function.h" #include "cel_expr_python/py_cel_function_decl.h" +#include "cel_expr_python/py_cel_options.h" #include "cel_expr_python/py_cel_type.h" #include @@ -68,7 +69,8 @@ class PyCelEnv { private: // Private constructor. Use `py_cel.NewEnv()` in python to obtain an instance. - PyCelEnv(const PyCelEnvConfig& config, PyObject* descriptor_pool, + PyCelEnv(const PyCelEnvConfig& config, const PyCelOptions& options, + PyObject* descriptor_pool, const std::unordered_map& variable_types, const std::vector& extensions, cel::ExpressionContainer container, diff --git a/cel_expr_python/py_cel_env_internal.cc b/cel_expr_python/py_cel_env_internal.cc index af0391b..8255975 100644 --- a/cel_expr_python/py_cel_env_internal.cc +++ b/cel_expr_python/py_cel_env_internal.cc @@ -45,6 +45,7 @@ #include "cel_expr_python/py_cel_env_config.h" #include "cel_expr_python/py_cel_function.h" #include "cel_expr_python/py_cel_function_decl.h" +#include "cel_expr_python/py_cel_options.h" #include "cel_expr_python/py_cel_overload.h" #include "cel_expr_python/py_cel_python_extension.h" #include "cel_expr_python/py_cel_type.h" @@ -66,10 +67,12 @@ static const cel::FunctionDescriptorOptions kFunctionDescriptorOptions = { } // namespace PyCelEnvInternal::PyCelEnvInternal( - const PyCelEnvConfig& env_config, PyObject* py_descriptor_pool, + const PyCelEnvConfig& env_config, const PyCelOptions& options, + PyObject* py_descriptor_pool, std::vector extension_handles, absl::flat_hash_map& function_impls) : env_config_(env_config), + options_(options), py_descriptor_database_(py_descriptor_pool), descriptor_pool_( std::make_shared(&py_descriptor_database_)), @@ -78,6 +81,9 @@ PyCelEnvInternal::PyCelEnvInternal( std::make_shared(py_descriptor_pool)), extensions_(std::move(extension_handles)), function_impls_(std::move(function_impls)) { + cel_env_.GetCompilerOptions().parser_options.enable_pratt_parser = + options.enable_pratt_parser; + cel_env_.SetDescriptorPool(descriptor_pool_); cel_env_.SetConfig(env_config_.GetConfig()); cel::RegisterStandardExtensions(cel_env_); @@ -105,7 +111,8 @@ PyCelEnvInternal::PyCelEnvInternal( absl::StatusOr> PyCelEnvInternal::NewCelEnvInternal( - const PyCelEnvConfig& env_config, PyObject* py_descriptor_pool, + const PyCelEnvConfig& env_config, const PyCelOptions& options, + PyObject* py_descriptor_pool, const std::unordered_map& variable_types, const std::vector& extensions, cel::ExpressionContainer container, @@ -219,7 +226,7 @@ PyCelEnvInternal::NewCelEnvInternal( } } return std::shared_ptr( - new PyCelEnvInternal(PyCelEnvConfig(config), py_descriptor_pool, + new PyCelEnvInternal(PyCelEnvConfig(config), options, py_descriptor_pool, std::move(extension_handles), impls)); } diff --git a/cel_expr_python/py_cel_env_internal.h b/cel_expr_python/py_cel_env_internal.h index ff82e4f..03e3c13 100644 --- a/cel_expr_python/py_cel_env_internal.h +++ b/cel_expr_python/py_cel_env_internal.h @@ -36,6 +36,7 @@ #include "cel_expr_python/py_cel_env_config.h" #include "cel_expr_python/py_cel_function.h" #include "cel_expr_python/py_cel_function_decl.h" +#include "cel_expr_python/py_cel_options.h" #include "cel_expr_python/py_cel_type.h" #include "cel_expr_python/py_descriptor_database.h" #include "cel_expr_python/py_message_factory.h" @@ -73,7 +74,8 @@ class PyCelEnvInternal { public: ~PyCelEnvInternal() = default; static absl::StatusOr> NewCelEnvInternal( - const PyCelEnvConfig& env_config, PyObject* py_descriptor_pool, + const PyCelEnvConfig& env_config, const PyCelOptions& options, + PyObject* py_descriptor_pool, const std::unordered_map& variable_types, const std::vector& extensions, cel::ExpressionContainer container, @@ -81,6 +83,7 @@ class PyCelEnvInternal { const std::unordered_map& function_impls); const PyCelEnvConfig& GetEnvConfig() const { return env_config_; } + const PyCelOptions& GetOptions() const { return options_; } static absl::StatusOr GetCompiler( const std::shared_ptr& env); @@ -113,8 +116,8 @@ class PyCelEnvInternal { private: // Use NewCelEnvInternal() to create an instance. PyCelEnvInternal( - const PyCelEnvConfig& env_config, PyObject* py_descriptor_pool, - std::vector extensions, + const PyCelEnvConfig& env_config, const PyCelOptions& options, + PyObject* py_descriptor_pool, std::vector extensions, absl::flat_hash_map& function_impls); absl::Status ConfigureStandardExtension( @@ -128,6 +131,7 @@ class PyCelEnvInternal { cel::Env cel_env_; cel::EnvRuntime cel_env_runtime_; PyCelEnvConfig env_config_; + PyCelOptions options_; PyDescriptorDatabase py_descriptor_database_; std::shared_ptr descriptor_pool_; google::protobuf::DynamicMessageFactory message_factory_; diff --git a/cel_expr_python/py_cel_module.cc b/cel_expr_python/py_cel_module.cc index d6766fe..aca47fd 100644 --- a/cel_expr_python/py_cel_module.cc +++ b/cel_expr_python/py_cel_module.cc @@ -19,6 +19,7 @@ #include "cel_expr_python/py_cel_expression.h" #include "cel_expr_python/py_cel_function.h" #include "cel_expr_python/py_cel_function_decl.h" +#include "cel_expr_python/py_cel_options.h" #include "cel_expr_python/py_cel_overload.h" #include "cel_expr_python/py_cel_python_extension.h" #include "cel_expr_python/py_cel_type.h" @@ -39,6 +40,7 @@ PYBIND11_MODULE(cel, m) { PyCelFunctionDecl::DefinePythonBindings(m); PyCelPythonExtension::DefinePythonBindings(m); PyCelFunction::DefinePythonBindings(m); + PyCelOptions::DefinePythonBindings(m); PyCelEnvConfig::DefinePythonBindings(m); PyCelEnv::DefinePythonBindings(m); } diff --git a/cel_expr_python/py_cel_options.cc b/cel_expr_python/py_cel_options.cc new file mode 100644 index 0000000..853c282 --- /dev/null +++ b/cel_expr_python/py_cel_options.cc @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cel_expr_python/py_cel_options.h" + +#include +#include +#include +#include + +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include + +namespace cel_python { + +namespace py = ::pybind11; + +namespace { + +std::string GenericRepr(py::handle self) { + std::vector parts; + py::handle cls = self.get_type(); + std::string class_name = cls.attr("__name__").cast(); + py::dict dict = cls.attr("__dict__"); + for (const auto& item : dict) { + std::string name = item.first.cast(); + if (!name.empty() && name[0] == '_') { + continue; + } + py::object val = self.attr(item.first); + parts.push_back( + absl::StrFormat("%s=%s", name, py::repr(val).cast())); + } + std::sort(parts.begin(), parts.end()); + return absl::StrFormat("%s(%s)", class_name, absl::StrJoin(parts, ", ")); +} + +} // namespace + +void PyCelOptions::DefinePythonBindings(pybind11::module& m) { + py::class_>(m, "Options") + .def(py::init([](bool enable_pratt_parser) { + return PyCelOptions{.enable_pratt_parser = enable_pratt_parser}; + }), + py::arg("enable_pratt_parser") = false) + .def_readwrite("enable_pratt_parser", &PyCelOptions::enable_pratt_parser) + .def("__repr__", &GenericRepr); +} + +} // namespace cel_python diff --git a/cel_expr_python/py_cel_options.h b/cel_expr_python/py_cel_options.h new file mode 100644 index 0000000..5aac31c --- /dev/null +++ b/cel_expr_python/py_cel_options.h @@ -0,0 +1,32 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef THIRD_PARTY_CEL_PYTHON_PY_CEL_OPTIONS_H_ +#define THIRD_PARTY_CEL_PYTHON_PY_CEL_OPTIONS_H_ + +#include + +namespace cel_python { + +struct PyCelOptions { + static void DefinePythonBindings(pybind11::module& m); + + bool enable_pratt_parser = false; +}; + +} // namespace cel_python + +#endif // THIRD_PARTY_CEL_PYTHON_PY_CEL_OPTIONS_H_ From 92989cf2dd067735165ffafcbf4a8f0005e20dbb Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 18 Aug 2026 11:46:30 -0700 Subject: [PATCH 19/23] Bump cel-spec version. Re-enable fixed conformance tests. PiperOrigin-RevId: 966718346 --- MODULE.bazel | 7 ++++++- conformance/BUILD | 1 + conformance/conformance_test.py | 13 ++----------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index a47ae41..c233a1e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,7 +24,12 @@ git_override( ) # https://registry.bazel.build/modules/cel-spec -bazel_dep(name = "cel-spec", version = "0.25.1", repo_name = "com_google_cel_spec") +bazel_dep(name = "cel-spec", version = "0.25.3", repo_name = "com_google_cel_spec") +git_override( + module_name = "cel-spec", + commit = "ba58ae5007845f3a1279b488cdeb79645ce958bb", + remote = "https://github.com/cel-expr/cel-spec", +) # https://registry.bazel.build/modules/googletest bazel_dep(name = "googletest", version = "1.17.0.bcr.2", repo_name = "com_google_googletest") diff --git a/conformance/BUILD b/conformance/BUILD index 79bb661..00291fc 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -14,6 +14,7 @@ _ALL_TESTS = [ "@com_google_cel_spec//tests/simple:testdata/fp_math.textproto", "@com_google_cel_spec//tests/simple:testdata/integer_math.textproto", "@com_google_cel_spec//tests/simple:testdata/lists.textproto", + "@com_google_cel_spec//tests/simple:testdata/lists_ext.textproto", "@com_google_cel_spec//tests/simple:testdata/logic.textproto", "@com_google_cel_spec//tests/simple:testdata/macros.textproto", "@com_google_cel_spec//tests/simple:testdata/math_ext.textproto", diff --git a/conformance/conformance_test.py b/conformance/conformance_test.py index d0b30e6..529bba2 100644 --- a/conformance/conformance_test.py +++ b/conformance/conformance_test.py @@ -60,13 +60,6 @@ class ConformanceTestSuite(unittest.TestSuite): "fields/qualified_identifier_resolution/qualified_identifier_resolution_unchecked", "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", "namespace/namespace/self_eval_container_lookup_unchecked", - # TODO(b/453051120): fix support for FieldMask->toJson - "wrappers/field_mask/to_json", - # TODO(b/464071224): fix support for triple-quoted unescaped punctuation. - "parse/bytes_literals/triple_double_quoted_unescaped_punctuation", - "parse/bytes_literals/triple_single_quoted_unescaped_punctuation", - "parse/string_literals/triple_double_quoted_unescaped_punctuation", - "parse/string_literals/triple_single_quoted_unescaped_punctuation", # Recent changes "proto2/set_null/repeated_field_timestamp_null_pruned", "proto2/set_null/repeated_field_duration_null_pruned", @@ -80,10 +73,8 @@ class ConformanceTestSuite(unittest.TestSuite): "proto3/set_null/map_timestamp_null_pruned", "proto3/set_null/map_duration_null_pruned", "proto3/set_null/map_wrapper_null_pruned", - "string_ext/format/default precision for fixed-point clause with int", - "string_ext/format/default precision for fixed-point clause with uint", - "string_ext/format/default precision for scientific notation with int", - "string_ext/format/default precision for scientific notation with uint", + # TODO(b/548571767): add lists_ext support + "lists_ext/.*", ] if sys.platform == "win32": From b42974063326901fefea650dc4e32b832bf337ee Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 19 Aug 2026 17:16:07 -0700 Subject: [PATCH 20/23] Wrap the "lists" extension library PiperOrigin-RevId: 967488650 --- cel_expr_python/BUILD | 2 ++ cel_expr_python/ext/BUILD | 21 ++++++++++++ cel_expr_python/ext/ext_lists.cc | 54 +++++++++++++++++++++++++++++++ cel_expr_python/ext/ext_lists.pyi | 4 +++ conformance/BUILD | 2 ++ conformance/conformance_test.py | 4 +-- release/setup.py | 4 +++ 7 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 cel_expr_python/ext/ext_lists.cc create mode 100644 cel_expr_python/ext/ext_lists.pyi diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index acb75ab..3b60244 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -172,6 +172,7 @@ py_test( data = [ ":cel", "//cel_expr_python/ext:ext_bindings", + "//cel_expr_python/ext:ext_lists", "//cel_expr_python/ext:ext_math", "//cel_expr_python/ext:ext_optional", "//cel_expr_python/ext:ext_strings", @@ -184,6 +185,7 @@ py_test( "//conditions:default": [ ":cel", "//cel_expr_python/ext:ext_bindings", + "//cel_expr_python/ext:ext_lists", "//cel_expr_python/ext:ext_math", "//cel_expr_python/ext:ext_optional", "//cel_expr_python/ext:ext_strings", diff --git a/cel_expr_python/ext/BUILD b/cel_expr_python/ext/BUILD index 1e879c1..4161b32 100644 --- a/cel_expr_python/ext/BUILD +++ b/cel_expr_python/ext/BUILD @@ -39,6 +39,27 @@ pybind_extension( ], ) +pybind_extension( + name = "ext_lists", + srcs = [ + "ext_lists.cc", + ], + data = [ + "ext_lists.pyi", + "//cel_expr_python:cel", + ], + visibility = ["//visibility:public"], + deps = [ + "//cel_expr_python:cel_extension", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_cel_cpp//compiler", + "@com_google_cel_cpp//extensions:lists_functions", + "@com_google_cel_cpp//runtime:runtime_builder", + "@com_google_cel_cpp//runtime:runtime_options", + ], +) + pybind_extension( name = "ext_math", srcs = [ diff --git a/cel_expr_python/ext/ext_lists.cc b/cel_expr_python/ext/ext_lists.cc new file mode 100644 index 0000000..888c252 --- /dev/null +++ b/cel_expr_python/ext/ext_lists.cc @@ -0,0 +1,54 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "compiler/compiler.h" +#include "extensions/lists_functions.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "cel_expr_python/cel_extension.h" +#include "cel_expr_python/py_error_status.h" + +namespace cel_python { + +class ExtLists : public CelExtension { + public: + explicit ExtLists(int version) + : CelExtension("cel.lib.ext.lists", "lists", version) { + if (version < 0 || + version > cel::extensions::kListsExtensionLatestVersion) { + throw StatusToException(absl::InvalidArgumentError(absl::StrCat( + "'lists' extension version: ", version, " not in range [0, ", + cel::extensions::kListsExtensionLatestVersion, "]"))); + } + } + + ExtLists() : ExtLists(cel::extensions::kListsExtensionLatestVersion) {} + + cel::CompilerLibrary GetCompilerLibrary() override { + return cel::extensions::ListsCompilerLibrary(version()); + } + + absl::Status ConfigureRuntime(cel::RuntimeBuilder& runtime_builder, + const cel::RuntimeOptions& opts) override { + return cel::extensions::RegisterListsFunctions( + runtime_builder.function_registry(), opts, + cel::extensions::ListsExtensionOptions{.version = version()}); + } +}; + +CEL_VERSIONED_EXTENSION_MODULE(ext_lists, ExtLists); + +} // namespace cel_python diff --git a/cel_expr_python/ext/ext_lists.pyi b/cel_expr_python/ext/ext_lists.pyi new file mode 100644 index 0000000..68b199f --- /dev/null +++ b/cel_expr_python/ext/ext_lists.pyi @@ -0,0 +1,4 @@ +from cel_expr_python import cel + +class ExtLists(cel.CelExtensionBase): + def __init__(self, version: int = ...) -> None: ... diff --git a/conformance/BUILD b/conformance/BUILD index 00291fc..1897cc8 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -40,6 +40,7 @@ py_test( "//cel_expr_python:cel", "//cel_expr_python/ext:ext_bindings", "//cel_expr_python/ext:ext_encoders", + "//cel_expr_python/ext:ext_lists", "//cel_expr_python/ext:ext_math", "//cel_expr_python/ext:ext_optional", "//cel_expr_python/ext:ext_proto", @@ -63,6 +64,7 @@ py_test( "//cel_expr_python:cel", "//cel_expr_python/ext:ext_bindings", "//cel_expr_python/ext:ext_encoders", + "//cel_expr_python/ext:ext_lists", "//cel_expr_python/ext:ext_math", "//cel_expr_python/ext:ext_optional", "//cel_expr_python/ext:ext_proto", diff --git a/conformance/conformance_test.py b/conformance/conformance_test.py index 529bba2..de54d1b 100644 --- a/conformance/conformance_test.py +++ b/conformance/conformance_test.py @@ -32,6 +32,7 @@ from cel_expr_python import cel from cel_expr_python.ext import ext_bindings from cel_expr_python.ext import ext_encoders +from cel_expr_python.ext import ext_lists from cel_expr_python.ext import ext_math from cel_expr_python.ext import ext_optional from cel_expr_python.ext import ext_proto @@ -73,8 +74,6 @@ class ConformanceTestSuite(unittest.TestSuite): "proto3/set_null/map_timestamp_null_pruned", "proto3/set_null/map_duration_null_pruned", "proto3/set_null/map_wrapper_null_pruned", - # TODO(b/548571767): add lists_ext support - "lists_ext/.*", ] if sys.platform == "win32": @@ -147,6 +146,7 @@ class ConformanceTest(absltest.TestCase): EXTENSIONS_PER_TESTFILE = { "bindings_ext": [ext_bindings.ExtBindings()], "encoders_ext": [ext_encoders.ExtEncoders()], + "lists_ext": [ext_lists.ExtLists()], "math_ext": [ext_math.ExtMath()], "optionals": [ext_optional.ExtOptional()], "proto2_ext": [ext_proto.ExtProto()], diff --git a/release/setup.py b/release/setup.py index e2d0962..16babad 100644 --- a/release/setup.py +++ b/release/setup.py @@ -207,6 +207,10 @@ def platform_config_macos(self, cmd): 'cel_expr_python.ext.ext_encoders', '//cel_expr_python/ext:ext_encoders', ), + BazelExtension( + 'cel_expr_python.ext.ext_lists', + '//cel_expr_python/ext:ext_lists', + ), BazelExtension( 'cel_expr_python.ext.ext_math', '//cel_expr_python/ext:ext_math', From ce7b742d68cb1db75ed842bb501d978bc6fd84af Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 21 Aug 2026 12:33:12 -0700 Subject: [PATCH 21/23] Add support for Python 3.10 PiperOrigin-RevId: 968634685 --- release/kokoro/release_macos.sh | 2 +- release/pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/release/kokoro/release_macos.sh b/release/kokoro/release_macos.sh index 54b92fa..dd5f501 100644 --- a/release/kokoro/release_macos.sh +++ b/release/kokoro/release_macos.sh @@ -11,7 +11,7 @@ fi pip install -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel echo "Installing CPython Mac Frameworks..." -for pyver in "3.11.9" "3.12.4" "3.13.0" "3.14.3"; do +for pyver in "3.10.11" "3.11.9" "3.12.4" "3.13.0" "3.14.3"; do echo "Downloading and installing Python ${pyver}..." curl -LO "https://www.python.org/ftp/python/${pyver}/python-${pyver}-macos11.pkg" sudo installer -pkg "python-${pyver}-macos11.pkg" -target / diff --git a/release/pyproject.toml b/release/pyproject.toml index d7aa211..c90f404 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -17,7 +17,7 @@ name = "cel-expr-python" # $VERSION is replaced with the desired version in build_wheel.sh. version = "$VERSION" description = "The CEL Python runtime" -requires-python = ">=3.11" +requires-python = ">=3.10" classifiers = [ "Topic :: Software Development :: Interpreters", "Topic :: Software Development :: Libraries :: Python Modules", @@ -38,7 +38,7 @@ where = ["."] exclude = ["codelab*", "conformance*", "custom_ext*", "release*", "testing*", "wheelhouse*"] [tool.cibuildwheel] -build = "cp311-* cp312-* cp313-* cp314-*" +build = "cp310-* cp311-* cp312-* cp313-* cp314-*" skip = "*musllinux* *win32* *i686*" test-command = "python {project}/cel_basic_test.py" build-verbosity = 1 From 6041cbb6ad84e2df7266d953349a063697adc0c5 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 24 Aug 2026 21:15:56 -0700 Subject: [PATCH 22/23] Add multi-threaded compilation and evaluation tests and resolve compilation deadlock in CEL Python. This change adds cel_parallel_test.py to benchmark and validate compiling and evaluating a diverse set of CEL expressions concurrently across multiple worker threads using concurrent.futures.ThreadPoolExecutor as well as sequentially. In addition, this resolves an AB-BA deadlock between the Python GIL and Protobuf DescriptorPool C++ mutex during concurrent compilation by releasing the GIL in PyCelEnv::Compile and acquiring the GIL via py::gil_scoped_acquire on-demand in PyDescriptorDatabase callbacks, while protecting PyCelEnvInternal with a mutex. Test duration metrics: - Multi-threaded compilation (1,000 iterations): ~248 ms - Sequential compilation (1,000 iterations): ~394 ms - Multi-threaded evaluation (10,000 iterations): ~782 ms - Sequential evaluation (10,000 iterations): ~434 ms PiperOrigin-RevId: 970277102 --- MODULE.bazel | 2 +- cel_expr_python/BUILD | 18 +- cel_expr_python/cel_parallel_test.py | 204 ++++++++++++++++++++++ cel_expr_python/py_cel_env.cc | 12 ++ cel_expr_python/py_cel_env_internal.cc | 56 +++--- cel_expr_python/py_cel_env_internal.h | 36 ++-- cel_expr_python/py_cel_expression.cc | 9 +- cel_expr_python/py_descriptor_database.cc | 12 +- cel_expr_python/py_error_status.cc | 7 +- 9 files changed, 292 insertions(+), 64 deletions(-) create mode 100644 cel_expr_python/cel_parallel_test.py diff --git a/MODULE.bazel b/MODULE.bazel index c233a1e..d9ec59a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,7 +19,7 @@ bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "cel-cpp", version = "0.16.1", repo_name = "com_google_cel_cpp") git_override( module_name = "cel-cpp", - commit = "1dcff093b3ca3d575f982e2ca154f9c751e06055", + commit = "98a7da06d3b9492e10e22d624fe55d749ce137c0", remote = "https://github.com/cel-expr/cel-cpp", ) diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index 3b60244..cbc08ca 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -51,6 +51,7 @@ pybind_library( ":cel_extension", ":status_macros", "@com_google_absl//absl/base", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/functional:function_ref", @@ -60,6 +61,7 @@ pybind_library( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", @@ -94,7 +96,6 @@ pybind_library( "@com_google_cel_cpp//runtime:reference_resolver", "@com_google_cel_cpp//runtime:runtime_builder", "@com_google_cel_cpp//runtime:runtime_options", - "@com_google_cel_cpp//validator", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", @@ -166,6 +167,21 @@ py_test( }), ) +py_test( + name = "cel_parallel_test", + srcs = ["cel_parallel_test.py"], + data = [ + ":cel", + ], + deps = [ + "//testing:proto2_test_all_types_py_pb2", + "@com_google_absl_py//absl/testing:absltest", + ] + select({ + "@platforms//os:windows": [], + "//conditions:default": [":cel"], + }), +) + py_test( name = "cel_env_test", srcs = ["cel_env_test.py"], diff --git a/cel_expr_python/cel_parallel_test.py b/cel_expr_python/cel_parallel_test.py new file mode 100644 index 0000000..479f2aa --- /dev/null +++ b/cel_expr_python/cel_parallel_test.py @@ -0,0 +1,204 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multi-threaded tests for cel-python.""" + +import collections.abc +import concurrent.futures +import dataclasses +import gc +import logging +import time +from typing import Any + +from absl.testing import absltest +from cel_expr_python import cel +from cel.expr.conformance.proto2 import test_all_types_pb2 as test_all_types_pb + + +@dataclasses.dataclass(frozen=True) +class _TestCase: + expr: str + data: collections.abc.Callable[[int], dict[str, Any]] + expected: collections.abc.Callable[[int], Any] + + +_NUM_EVALUATIONS = 10000 +_NUM_COMPILATIONS = 1000 + +_TEST_MSG = test_all_types_pb.TestAllTypes(single_int64=100) + +_TEST_CASES = [ + _TestCase( + expr="var_int * var_int", + data=lambda n: {"var_int": n}, + expected=lambda n: n * n, + ), + _TestCase( + expr="var_str + '_' + string(var_int)", + data=lambda n: {"var_str": "num", "var_int": n}, + expected=lambda n: f"num_{n}", + ), + _TestCase( + expr="var_int % 2 == 0", + data=lambda n: {"var_int": n}, + expected=lambda n: n % 2 == 0, + ), + _TestCase( + expr="[var_int, var_int + 1, var_int + 2]", + data=lambda n: {"var_int": n}, + expected=lambda n: [n, n + 1, n + 2], + ), + _TestCase( + expr="var_int_map[var_int]", + data=lambda n: {"var_int_map": {n: f"val_{n}"}, "var_int": n}, + expected=lambda n: f"val_{n}", + ), + _TestCase( + expr="var_msg.single_int64 + var_int", + data=lambda n: {"var_msg": _TEST_MSG, "var_int": n}, + expected=lambda n: 100 + n, + ), + _TestCase( + expr=( + "cel.expr.conformance.proto2.TestAllTypes{" + " single_int64: var_int, single_string: var_str" + "}" + ), + data=lambda n: {"var_int": n, "var_str": f"msg_{n}"}, + expected=lambda n: test_all_types_pb.TestAllTypes( + single_int64=n, single_string=f"msg_{n}" + ), + ), + _TestCase( + expr="{'key': var_str, 'value': var_int}", + data=lambda n: {"var_str": f"val_{n}", "var_int": n}, + expected=lambda n: {"key": f"val_{n}", "value": n}, + ), + _TestCase( + expr="[var_int, var_int + 1, var_int + 2].all(x, x >= var_int)", + data=lambda n: {"var_int": n}, + expected=lambda n: True, + ), +] + + +class CelParallelTest(absltest.TestCase): + + def setUp(self): + super().setUp() + + self.env = cel.NewEnv( + variables={ + "var_int": cel.Type.INT, + "var_str": cel.Type.STRING, + "var_int_map": cel.Type.Map(cel.Type.INT, cel.Type.STRING), + "var_msg": cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + }, + ) + self.object_counts_before_test = self._grab_object_counts() + + def tearDown(self): + """Tears down the test environment.""" + super().tearDown() + + gc.collect() + # Assert that all Arenas have been garbage-collected + self.assertEqual(cel._InternalArena._get_instance_count(), 0) + self._check_for_leaks() + + def _grab_object_counts(self) -> dict[str, int]: + gc.collect() + all_objects = gc.get_objects() + type_counts = {} + for obj in all_objects: + obj_type = type(obj) + type_counts[obj_type.__name__] = type_counts.get(obj_type, 0) + 1 + return type_counts + + def _check_for_leaks(self): + type_counts = self._grab_object_counts() + for key, count in type_counts.items(): + if count != self.object_counts_before_test.get(key, 0): + self.fail( + f"Object count for {key} did not match expected count. " + f"Expected: {self.object_counts_before_test.get(key, 0)}, " + f"Actual: {count}", + ) + + def _test_eval(self, multi_threaded: bool): + compiled_exprs = [self.env.compile(tc.expr) for tc in _TEST_CASES] + + def eval_expr(n: int) -> Any: + idx = n % len(_TEST_CASES) + test_case = _TEST_CASES[idx] + expr = compiled_exprs[idx] + data = test_case.data(n) + return expr.eval(data=data).plain_value() + + start_time = time.perf_counter() + if multi_threaded: + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(eval_expr, range(_NUM_EVALUATIONS))) + else: + results = [eval_expr(n) for n in range(_NUM_EVALUATIONS)] + duration_ms = (time.perf_counter() - start_time) * 1000 + + mode = "Multi-threaded" if multi_threaded else "Sequential" + logging.info("%s evaluation duration: %.2f ms", mode, duration_ms) + + self.assertLen(results, _NUM_EVALUATIONS) + for i, res in enumerate(results): + test_case = _TEST_CASES[i % len(_TEST_CASES)] + self.assertEqual(res, test_case.expected(i)) + + def testMultiThreadedEval(self): + self._test_eval(multi_threaded=True) + + def testSequentialEval(self): + self._test_eval(multi_threaded=False) + + def _test_compile(self, multi_threaded: bool): + def compile_expr(n: int) -> cel.Expression: + test_case = _TEST_CASES[n % len(_TEST_CASES)] + return self.env.compile(test_case.expr) + + start_time = time.perf_counter() + if multi_threaded: + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(compile_expr, range(_NUM_COMPILATIONS))) + else: + results = [compile_expr(n) for n in range(_NUM_COMPILATIONS)] + duration_ms = (time.perf_counter() - start_time) * 1000 + + mode = "Multi-threaded" if multi_threaded else "Sequential" + logging.info("%s compilation duration: %.2f ms", mode, duration_ms) + + self.assertLen(results, _NUM_COMPILATIONS) + for i, expr in enumerate(results): + test_case = _TEST_CASES[i % len(_TEST_CASES)] + data = test_case.data(i) + self.assertEqual( + expr.eval(data=data).plain_value(), test_case.expected(i) + ) + + def testMultiThreadedCompilation(self): + self._test_compile(multi_threaded=True) + + def testSequentialCompilation(self): + self._test_compile(multi_threaded=False) + + +if __name__ == "__main__": + absltest.main() diff --git a/cel_expr_python/py_cel_env.cc b/cel_expr_python/py_cel_env.cc index 30758cf..68310be 100644 --- a/cel_expr_python/py_cel_env.cc +++ b/cel_expr_python/py_cel_env.cc @@ -196,6 +196,18 @@ std::shared_ptr PyCelEnv::NewActivation( PyCelExpression PyCelEnv::Compile(const std::string& cel_expr, bool disable_check) { + // Release the GIL before entering C++ compilation to prevent lock + // inversion/deadlock with DescriptorPool's internal mutex during concurrent + // multi-threaded compilation. + // + // When DescriptorPool performs a descriptor lookup on a cache miss, it calls + // back into Python via PyDescriptorDatabase (which re-acquires the GIL via + // PyGILState_Ensure). If another thread were to enter Compile() with the GIL + // held, it would block on DescriptorPool's internal C++ mutex while holding + // the GIL, causing an AB-BA deadlock with any thread inside + // PyDescriptorDatabase waiting for the GIL. Releasing the GIL here guarantees + // a strict one-way lock hierarchy (DescriptorPool Mutex -> Python GIL). + py::gil_scoped_release gil_release; return ThrowIfError(PyCelExpression::Compile(env_, cel_expr, disable_check)); } diff --git a/cel_expr_python/py_cel_env_internal.cc b/cel_expr_python/py_cel_env_internal.cc index 8255975..cd2a58e 100644 --- a/cel_expr_python/py_cel_env_internal.cc +++ b/cel_expr_python/py_cel_env_internal.cc @@ -25,6 +25,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" +#include "absl/synchronization/mutex.h" #include "checker/type_checker_builder.h" #include "common/container.h" #include "common/function_descriptor.h" @@ -40,7 +41,6 @@ #include "runtime/runtime.h" #include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" -#include "validator/validator.h" #include "cel_expr_python/cel_extension.h" #include "cel_expr_python/py_cel_env_config.h" #include "cel_expr_python/py_cel_function.h" @@ -230,19 +230,17 @@ PyCelEnvInternal::NewCelEnvInternal( std::move(extension_handles), impls)); } -absl::StatusOr PyCelEnvInternal::GetCompiler( - const std::shared_ptr& env) { - ABSL_CHECK(PyGILState_Check()); - - if (env->compiler_) { - return env->compiler_.get(); +absl::StatusOr PyCelEnvInternal::GetCompiler() const { + absl::MutexLock lock(mutex_); + if (compiler_) { + return compiler_.get(); } - const cel::Config& config = env->env_config_.GetConfig(); + const cel::Config& config = env_config_.GetConfig(); CEL_PYTHON_ASSIGN_OR_RETURN( std::unique_ptr compiler_builder, - env->cel_env_.NewCompilerBuilder()); + cel_env_.NewCompilerBuilder()); cel::TypeCheckerBuilder& checker_builder = compiler_builder->GetCheckerBuilder(); @@ -266,25 +264,25 @@ absl::StatusOr PyCelEnvInternal::GetCompiler( for (const cel::Config::VariableConfig& variable_config : config.GetVariableConfigs()) { CEL_PYTHON_ASSIGN_OR_RETURN( - cel::Type cel_type, - cel::TypeInfoToType(variable_config.type_info, - env->descriptor_pool_.get(), arena)); + cel::Type cel_type, cel::TypeInfoToType(variable_config.type_info, + descriptor_pool_.get(), arena)); PyCelType py_cel_type = PyCelType::FromCelType(cel_type); - env->variable_types_[variable_config.name] = py_cel_type; + variable_types_[variable_config.name] = py_cel_type; } - CEL_PYTHON_ASSIGN_OR_RETURN(env->compiler_, compiler_builder->Build()); - return env->compiler_.get(); + CEL_PYTHON_ASSIGN_OR_RETURN(compiler_, compiler_builder->Build()); + return compiler_.get(); } absl::StatusOr PyCelEnvInternal::GetRuntime( - const std::shared_ptr& env, RuntimeMode runtime_mode) { - if (auto it = env->runtimes_.find(runtime_mode); it != env->runtimes_.end()) { + RuntimeMode runtime_mode) const { + absl::MutexLock lock(mutex_); + if (auto it = runtimes_.find(runtime_mode); it != runtimes_.end()) { return it->second.get(); } - cel::RuntimeOptions& opts = env->cel_env_runtime_.mutable_runtime_options(); - opts.container = env->GetEnvConfig().GetConfig().GetContainerConfig().name; + cel::RuntimeOptions opts; + opts.container = env_config_.GetConfig().GetContainerConfig().name; opts.enable_empty_wrapper_null_unboxing = true; opts.enable_qualified_type_identifiers = true; opts.enable_timestamp_duration_overflow_errors = true; @@ -296,16 +294,16 @@ absl::StatusOr PyCelEnvInternal::GetRuntime( break; } CEL_PYTHON_ASSIGN_OR_RETURN(cel::RuntimeBuilder builder, - env->cel_env_runtime_.CreateRuntimeBuilder()); + cel_env_runtime_.CreateRuntimeBuilder(opts)); CEL_PYTHON_RETURN_IF_ERROR(cel::EnableReferenceResolver( builder, cel::ReferenceResolverEnabled::kAlways)); for (const cel::Config::FunctionConfig& function_config : - env->GetEnvConfig().GetConfig().GetFunctionConfigs()) { + GetEnvConfig().GetConfig().GetFunctionConfigs()) { for (const cel::Config::FunctionOverloadConfig& overload_config : function_config.overload_configs) { - auto it = env->function_impls_.find(overload_config.overload_id); - if (it == env->function_impls_.end()) { + auto it = function_impls_.find(overload_config.overload_id); + if (it == function_impls_.end()) { continue; } py::object py_function = it->second; @@ -315,8 +313,7 @@ absl::StatusOr PyCelEnvInternal::GetRuntime( overload_config.parameters) { CEL_PYTHON_ASSIGN_OR_RETURN( cel::Type type, - cel::TypeInfoToType(parameter, env->descriptor_pool_.get(), - &env->arena_)); + cel::TypeInfoToType(parameter, descriptor_pool_.get(), &arena_)); param_kinds.push_back(static_cast(type.kind())); } cel::FunctionDescriptor descriptor( @@ -325,7 +322,7 @@ absl::StatusOr PyCelEnvInternal::GetRuntime( CEL_PYTHON_ASSIGN_OR_RETURN( cel::Type return_type, cel::TypeInfoToType(overload_config.return_type, - env->descriptor_pool_.get(), &env->arena_)); + descriptor_pool_.get(), &arena_)); CEL_PYTHON_RETURN_IF_ERROR(builder.function_registry().Register( descriptor, std::make_unique( function_config.name, @@ -335,13 +332,13 @@ absl::StatusOr PyCelEnvInternal::GetRuntime( CEL_PYTHON_ASSIGN_OR_RETURN(std::unique_ptr runtime, std::move(builder).Build()); const cel::Runtime* runtime_ptr = runtime.get(); - env->runtimes_[runtime_mode] = std::move(runtime); + runtimes_[runtime_mode] = std::move(runtime); return runtime_ptr; } const PyCelType& PyCelEnvInternal::GetVariableType( const std::string& name) const { - ABSL_CHECK(PyGILState_Check()); + absl::MutexLock lock(mutex_); auto it = variable_types_.find(name); if (it != variable_types_.end()) { return it->second; @@ -363,9 +360,8 @@ CelExtensionHandle::CelExtensionHandle(CelExtensionHandle&& other) CelExtensionHandle::~CelExtensionHandle() { if (py_extension_ != nullptr) { - auto gil_state = PyGILState_Ensure(); + py::gil_scoped_acquire acquire; Py_DECREF(py_extension_); - PyGILState_Release(gil_state); } } diff --git a/cel_expr_python/py_cel_env_internal.h b/cel_expr_python/py_cel_env_internal.h index 03e3c13..fc69b34 100644 --- a/cel_expr_python/py_cel_env_internal.h +++ b/cel_expr_python/py_cel_env_internal.h @@ -22,9 +22,11 @@ #include #include +#include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/synchronization/mutex.h" #include "common/container.h" #include "compiler/compiler.h" #include "env/env.h" @@ -85,8 +87,7 @@ class PyCelEnvInternal { const PyCelEnvConfig& GetEnvConfig() const { return env_config_; } const PyCelOptions& GetOptions() const { return options_; } - static absl::StatusOr GetCompiler( - const std::shared_ptr& env); + absl::StatusOr GetCompiler() const; enum RuntimeMode { // Standard CEL runtime with warnings treated as errors. @@ -96,15 +97,15 @@ class PyCelEnvInternal { kStandardIgnoreWarnings, }; - static absl::StatusOr GetRuntime( - const std::shared_ptr& env, RuntimeMode runtime_mode); + absl::StatusOr GetRuntime( + RuntimeMode runtime_mode) const; const google::protobuf::DescriptorPool* GetDescriptorPool() const { return descriptor_pool_.get(); } google::protobuf::MessageFactory* GetMessageFactory() const { - return const_cast(&message_factory_); + return &message_factory_; } std::shared_ptr GetPyMessageFactory() const { @@ -117,32 +118,27 @@ class PyCelEnvInternal { // Use NewCelEnvInternal() to create an instance. PyCelEnvInternal( const PyCelEnvConfig& env_config, const PyCelOptions& options, - PyObject* py_descriptor_pool, std::vector extensions, + PyObject* py_descriptor_pool, + std::vector extension_handles, absl::flat_hash_map& function_impls); - absl::Status ConfigureStandardExtension( - cel::CompilerBuilder& compiler_builder, const std::string& extension); - - absl::Status ConfigureStandardExtension(cel::RuntimeBuilder& runtime_builder, - const std::string& extension, - const cel::RuntimeOptions& opts); - - google::protobuf::Arena arena_; + mutable absl::Mutex mutex_; + mutable google::protobuf::Arena arena_ ABSL_GUARDED_BY(mutex_); cel::Env cel_env_; cel::EnvRuntime cel_env_runtime_; PyCelEnvConfig env_config_; PyCelOptions options_; PyDescriptorDatabase py_descriptor_database_; std::shared_ptr descriptor_pool_; - google::protobuf::DynamicMessageFactory message_factory_; + mutable google::protobuf::DynamicMessageFactory message_factory_; std::shared_ptr py_message_factory_; - // Synchronized by the GIL. - absl::flat_hash_map variable_types_; + mutable absl::flat_hash_map variable_types_ + ABSL_GUARDED_BY(mutex_); std::vector extensions_; absl::flat_hash_map function_impls_; - std::unique_ptr compiler_; - absl::flat_hash_map> - runtimes_; + mutable std::unique_ptr compiler_ ABSL_GUARDED_BY(mutex_); + mutable absl::flat_hash_map> + runtimes_ ABSL_GUARDED_BY(mutex_); }; } // namespace cel_python diff --git a/cel_expr_python/py_cel_expression.cc b/cel_expr_python/py_cel_expression.cc index bc36244..1a7b557 100644 --- a/cel_expr_python/py_cel_expression.cc +++ b/cel_expr_python/py_cel_expression.cc @@ -105,10 +105,8 @@ void PyCelExpression::DefinePythonBindings(py::module& m) { absl::StatusOr PyCelExpression::Compile( const std::shared_ptr& env, const std::string& cel_expr, bool disable_check) { - ABSL_CHECK(PyGILState_Check()); - CEL_PYTHON_ASSIGN_OR_RETURN(const cel::Compiler* compiler, - PyCelEnvInternal::GetCompiler(env)); + env->GetCompiler()); if (disable_check) { CEL_PYTHON_ASSIGN_OR_RETURN(auto s, cel::NewSource(cel_expr, "")); @@ -153,15 +151,14 @@ absl::StatusOr PyCelExpression::Eval( if (std::holds_alternative(expr_)) { PY_CEL_PYTHON_ASSIGN_OR_RETURN( const cel::Runtime* runtime, - PyCelEnvInternal::GetRuntime( - env_, PyCelEnvInternal::kStandardIgnoreWarnings)); + env_->GetRuntime(PyCelEnvInternal::kStandardIgnoreWarnings)); PY_CEL_PYTHON_ASSIGN_OR_RETURN( cel_program_, cel::extensions::ProtobufRuntimeAdapter::CreateProgram( *runtime, std::get(expr_))); } else { PY_CEL_PYTHON_ASSIGN_OR_RETURN( const cel::Runtime* runtime, - PyCelEnvInternal::GetRuntime(env_, PyCelEnvInternal::kStandard)); + env_->GetRuntime(PyCelEnvInternal::kStandard)); PY_CEL_PYTHON_ASSIGN_OR_RETURN( cel_program_, cel::extensions::ProtobufRuntimeAdapter::CreateProgram( *runtime, std::get(expr_))); diff --git a/cel_expr_python/py_descriptor_database.cc b/cel_expr_python/py_descriptor_database.cc index ea36039..f6e54ef 100644 --- a/cel_expr_python/py_descriptor_database.cc +++ b/cel_expr_python/py_descriptor_database.cc @@ -25,9 +25,12 @@ #include "common/minimal_descriptor_pool.h" #include "cel_expr_python/py_error_status.h" #include "google/protobuf/descriptor.h" +#include namespace cel_python { +namespace py = pybind11; + PyDescriptorDatabase::PyDescriptorDatabase(PyObject* py_descriptor_pool) : py_descriptor_pool_(py_descriptor_pool), standard_pool_(cel::GetMinimalDescriptorPool()) { @@ -36,16 +39,14 @@ PyDescriptorDatabase::PyDescriptorDatabase(PyObject* py_descriptor_pool) } PyDescriptorDatabase::~PyDescriptorDatabase() { - auto gil_state = PyGILState_Ensure(); + py::gil_scoped_acquire acquire; Py_XDECREF(py_descriptor_pool_); - PyGILState_Release(gil_state); } // Find a file by file name. Fills in in *output and returns true if found. // Otherwise, returns false, leaving the contents of *output undefined. bool PyDescriptorDatabase::FindFileByName(StringViewArg filename, google::protobuf::FileDescriptorProto* output) { - ABSL_CHECK(PyGILState_Check()); const google::protobuf::FileDescriptor* file = standard_pool_.FindFileByName(filename); if (file != nullptr) { file->CopyTo(output); @@ -56,6 +57,7 @@ bool PyDescriptorDatabase::FindFileByName(StringViewArg filename, return false; } + py::gil_scoped_acquire acquire; PyObject* pyfile = PyObject_CallMethod( py_descriptor_pool_, "FindFileByName", "s#", filename.data(), static_cast(filename.size())); @@ -94,7 +96,6 @@ bool PyDescriptorDatabase::FindFileByName(StringViewArg filename, // and leaves *output undefined. bool PyDescriptorDatabase::FindFileContainingSymbol( StringViewArg symbol_name, google::protobuf::FileDescriptorProto* output) { - ABSL_CHECK(PyGILState_Check()); const google::protobuf::FileDescriptor* file = standard_pool_.FindFileContainingSymbol(symbol_name); if (file != nullptr) { @@ -106,6 +107,7 @@ bool PyDescriptorDatabase::FindFileContainingSymbol( return false; } + py::gil_scoped_acquire acquire; PyObject* pyfile = PyObject_CallMethod( py_descriptor_pool_, "FindFileContainingSymbol", "s#", symbol_name.data(), static_cast(symbol_name.size())); @@ -149,7 +151,7 @@ bool PyDescriptorDatabase::FindFileContainingExtension( return false; } - ABSL_CHECK(PyGILState_Check()); + py::gil_scoped_acquire acquire; PyObject* py_containing_type = PyObject_CallMethod( py_descriptor_pool_, "FindMessageTypeByName", "s#", containing_type.data(), static_cast(containing_type.size())); diff --git a/cel_expr_python/py_error_status.cc b/cel_expr_python/py_error_status.cc index 83e03b9..4565c0e 100644 --- a/cel_expr_python/py_error_status.cc +++ b/cel_expr_python/py_error_status.cc @@ -29,6 +29,8 @@ namespace cel_python { +namespace py = pybind11; + static absl::Status PyErrorToStatus(PyObject* py_type, PyObject* py_error) { // Loose mapping from Python exceptions to absl::Status codes, consistent with // the pybind11 mapping. @@ -96,11 +98,13 @@ std::runtime_error StatusToException(const absl::Status& status) { } static absl::Status& PendingPyError() { - static absl::NoDestructor pending_py_error(absl::OkStatus()); + static thread_local absl::NoDestructor pending_py_error( + absl::OkStatus()); return *pending_py_error; } absl::Status PyErr_toStatus() { + py::gil_scoped_acquire acquire; PyObject* py_error = PyErr_Occurred(); if (!py_error) { absl::Status status = PendingPyError(); @@ -132,6 +136,7 @@ absl::Status PyErr_toStatus() { } void PyErr_noteAndClear() { + py::gil_scoped_acquire acquire; if (!PyErr_Occurred()) { return; } From 4e64447671ae530661c946c2e4f6b863213c7b33 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 25 Aug 2026 18:48:36 -0700 Subject: [PATCH 23/23] Enable free-threaded Python and improve concurrency in CEL Python. - Declare free-threading compatibility on pybind11 modules (`mod_gil_not_used`) and BUILD targets (`freethreading_compatible`). - Introduce `FreeThreadingMutex`, a zero-cost abstraction that compiles away in GIL-enabled builds and uses `absl::Mutex` in free-threaded mode (`Py_GIL_DISABLED`). - Protect lazy `cel_program_` compilation in `PyCelExpression` with `FreeThreadingMutex`. - Protect Python object caching and lazy element resolution in `PyCelValue` and list/map accessors with `FreeThreadingMutex`. - Protect `PyMessageFactory::message_classes_` with `FreeThreadingMutex`. - Extract `BuildCompiler` and `BuildRuntime` to construct compiler/runtime objects without holding mutex locks, using double-checked locking in `GetCompiler` and `GetRuntime`. - Eagerly resolve `variable_types_` during `PyCelEnvInternal` initialization, making `GetVariableType` lock-free and thread-safe. - Release GIL during CPU-bound policy compilation and YAML parsing. - Avoid unconditional `gil_scoped_acquire` across destructors and error conversions when GIL state is already held (`PyGILState_Check`). - Initialize pybind11 internals at module import time to prevent DSO internals races in multi-threaded environments. - Prevent reference leaks in custom function invocation by releasing arguments and temporary results. - Add `test_freethreaded.sh` script and concurrency tests in `cel_parallel_test.py` and `cel_test.py`. ### Performance Benchmarks (`cel_parallel_test`, `-c opt`) | Benchmark Stage / Test Case | Standard Mode (GIL) | Free-Threaded Mode (No GIL) | Speedup / Impact | | :--- | :---: | :---: | :---: | | **Multi-Threaded Compilation** (1,000 items, 8 threads) | 75.74 ms | 23.14 ms | **3.27x faster** (69.5% faster) | | **Sequential Compilation** (1,000 items) | 61.77 ms | 63.28 ms | 0.98x | | **Multi-Threaded Evaluation** (10,000 items, 8 threads) | 234.79 ms | 138.88 ms | **1.69x faster** (40.9% faster) | | **Sequential Evaluation** (10,000 items) | 90.68 ms | 112.74 ms | 0.80x | PiperOrigin-RevId: 970936071 --- cel_expr_python/BUILD | 5 + cel_expr_python/cel_extension.h | 4 +- cel_expr_python/cel_parallel_test.py | 41 +++++++ cel_expr_python/cel_test.py | 27 +++++ cel_expr_python/free_threading_mutex.h | 85 +++++++++++++++ cel_expr_python/py_cel_env.cc | 1 + cel_expr_python/py_cel_env_internal.cc | 114 +++++++++++++------- cel_expr_python/py_cel_env_internal.h | 27 ++--- cel_expr_python/py_cel_expression.cc | 59 ++++++---- cel_expr_python/py_cel_expression.h | 14 ++- cel_expr_python/py_cel_function.cc | 8 +- cel_expr_python/py_cel_function_decl.cc | 3 - cel_expr_python/py_cel_module.cc | 4 +- cel_expr_python/py_cel_type.cc | 1 + cel_expr_python/py_cel_value.cc | 125 ++++++++++++++++------ cel_expr_python/py_cel_value.h | 37 +++---- cel_expr_python/py_descriptor_database.cc | 11 +- cel_expr_python/py_descriptor_database.h | 1 + cel_expr_python/py_error_status.cc | 12 ++- cel_expr_python/py_error_status.h | 10 ++ cel_expr_python/py_message_factory.cc | 78 +++++++++----- cel_expr_python/py_message_factory.h | 6 +- test_freethreaded.sh | 64 +++++++++++ 23 files changed, 565 insertions(+), 172 deletions(-) create mode 100644 cel_expr_python/free_threading_mutex.h create mode 100755 test_freethreaded.sh diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index cbc08ca..4072870 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -29,6 +29,7 @@ pybind_library( "py_message_factory.cc", ], hdrs = [ + "free_threading_mutex.h", "py_cel_activation.h", "py_cel_arena.h", "py_cel_env.h", @@ -46,6 +47,7 @@ pybind_library( "py_descriptor_database.h", "py_message_factory.h", ], + tags = ["freethreading_compatible"], visibility = [":__subpackages__"], deps = [ ":cel_extension", @@ -113,6 +115,7 @@ pybind_extension( "//visibility:public", ], deps = [ + ":cel_extension", ":cel_pybind_lib", ], ) @@ -127,6 +130,7 @@ pybind_library( "cel_extension.h", "py_error_status.h", ], + tags = ["freethreading_compatible"], visibility = ["//visibility:public"], deps = [ ":status_macros", @@ -161,6 +165,7 @@ py_test( "//testing:proto2_test_all_types_py_pb2", "@com_google_absl_py//absl/testing:absltest", "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:protobuf_python", ] + select({ "@platforms//os:windows": [], "//conditions:default": [":cel"], diff --git a/cel_expr_python/cel_extension.h b/cel_expr_python/cel_extension.h index 48a7151..e8672cd 100644 --- a/cel_expr_python/cel_extension.h +++ b/cel_expr_python/cel_extension.h @@ -80,14 +80,14 @@ class CelExtension { // CEL_EXTENSION_MODULE(sample_cel_ext, SampleCelExtension); // #define CEL_EXTENSION_MODULE(module_name, class_name) \ - PYBIND11_MODULE(module_name, m) { \ + PYBIND11_MODULE(module_name, m, pybind11::mod_gil_not_used()) { \ pybind11::module_::import(CEL_MODULE_NAME); \ pybind11::class_(m, #class_name) \ .def(pybind11::init<>()); \ } #define CEL_VERSIONED_EXTENSION_MODULE(module_name, class_name) \ - PYBIND11_MODULE(module_name, m) { \ + PYBIND11_MODULE(module_name, m, pybind11::mod_gil_not_used()) { \ pybind11::module_::import(CEL_MODULE_NAME); \ pybind11::class_(m, #class_name) \ .def(pybind11::init<>()) \ diff --git a/cel_expr_python/cel_parallel_test.py b/cel_expr_python/cel_parallel_test.py index 479f2aa..5a41fbe 100644 --- a/cel_expr_python/cel_parallel_test.py +++ b/cel_expr_python/cel_parallel_test.py @@ -199,6 +199,47 @@ def testMultiThreadedCompilation(self): def testSequentialCompilation(self): self._test_compile(multi_threaded=False) + def testSharedValueConcurrentAccess(self): + expr_scalar = self.env.compile("var_int * 2") + expr_list = self.env.compile("[var_int, var_int + 1, var_int + 2]") + expr_map = self.env.compile("{'key': var_str, 'value': var_int}") + + def run_concurrent_value_test(n: int): + val_scalar = expr_scalar.eval(data={"var_int": n}) + val_list = expr_list.eval(data={"var_int": n}) + val_map = expr_map.eval(data={"var_str": f"k_{n}", "var_int": n}) + + def read_values(_): + # Concurrently access plain_value, value, type, and repr + # on shared instances + self.assertEqual(val_scalar.plain_value(), n * 2) + self.assertEqual(val_scalar.value(), n * 2) + self.assertEqual(val_scalar.type(), cel.Type.INT) + self.assertNotEmpty(str(val_scalar)) + + plain_list = val_list.plain_value() + self.assertEqual(plain_list, [n, n + 1, n + 2]) + list_accessors = val_list.value() + for idx, item in enumerate(list_accessors): + self.assertEqual(item.plain_value(), n + idx) + self.assertEqual(item.value(), n + idx) + self.assertEqual(item.type(), cel.Type.INT) + self.assertNotEmpty(str(item)) + + plain_map = val_map.plain_value() + self.assertEqual(plain_map, {"key": f"k_{n}", "value": n}) + map_accessors = val_map.value() + self.assertEqual(map_accessors["key"].plain_value(), f"k_{n}") + self.assertEqual(map_accessors["key"].value(), f"k_{n}") + self.assertEqual(map_accessors["value"].plain_value(), n) + self.assertEqual(map_accessors["value"].value(), n) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(read_values, range(50))) + + for i in range(10): + run_concurrent_value_test(i) + if __name__ == "__main__": absltest.main() diff --git a/cel_expr_python/cel_test.py b/cel_expr_python/cel_test.py index 2add977..9eb6100 100644 --- a/cel_expr_python/cel_test.py +++ b/cel_expr_python/cel_test.py @@ -21,6 +21,7 @@ import sys from typing import Any +from google.protobuf import message_factory from google.protobuf import duration_pb2 as duration_pb from google.protobuf import timestamp_pb2 as timestamp_pb from absl.testing import absltest @@ -337,6 +338,32 @@ def testProto_unexpectedType(self): r" .*. \(Expected cel.expr.conformance.proto2.TestAllTypes\)", ) + def testProto_cannotFindMessageClass(self): + orig_get_message_class = message_factory.GetMessageClass + + def failing_get_message_class(descriptor): + del descriptor + raise TypeError( + "Couldn't build proto class because dependency X is missing" + ) + + try: + message_factory.GetMessageClass = failing_get_message_class + env = cel.NewEnv(options=self.options) + expr = env.compile( + "cel.expr.conformance.proto2.TestAllTypes{single_string: 'hello'}" + ) + res = expr.eval(env.Activation()) + with self.assertRaisesRegex( + TypeError, "Couldn't find message class for type" + ): + res.plain_value() + del res + del expr + del env + finally: + message_factory.GetMessageClass = orig_get_message_class + def testEvalList(self): res: cel.Value = self._eval( "[1, 'CEL', true]", expected_return_type=cel.Type.LIST diff --git a/cel_expr_python/free_threading_mutex.h b/cel_expr_python/free_threading_mutex.h new file mode 100644 index 0000000..2501f46 --- /dev/null +++ b/cel_expr_python/free_threading_mutex.h @@ -0,0 +1,85 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_PYTHON_FREE_THREADING_MUTEX_H_ +#define THIRD_PARTY_CEL_PYTHON_FREE_THREADING_MUTEX_H_ + +#include // IWYU pragma: keep - Needed for Py_GIL_DISABLED + +#include "absl/base/attributes.h" +#include "absl/base/thread_annotations.h" + +#ifndef Py_GIL_DISABLED +#else +#include "absl/synchronization/mutex.h" +#endif + +namespace cel_python { + +// Zero-cost mutex wrapper that compiles away to nothing in standard GIL builds, +// and uses absl::Mutex in free-threaded builds (Py_GIL_DISABLED). +class ABSL_LOCKABLE ABSL_ATTRIBUTE_WARN_UNUSED FreeThreadingMutex { + public: + FreeThreadingMutex() = default; + FreeThreadingMutex(const FreeThreadingMutex&) = delete; + FreeThreadingMutex& operator=(const FreeThreadingMutex&) = delete; + +#ifndef Py_GIL_DISABLED + // In GIL-enabled builds, this mutex compiles away to zero-cost no-ops while + // retaining thread-safety annotations for Clang static analysis. + // + // Relationship with the GIL: + // - For operations accessing Python state or cached PyObjects (such as + // PyCelValue::Value() or PyMessageFactory::GetMessageClass()), mutual + // exclusion between threads is provided by the Python GIL itself. + // - For pure C++ operations (such as PyCelEnv::Compile() or deserialization), + // the GIL is intentionally released (via py::gil_scoped_release) to enable + // parallel execution and prevent deadlocks with DescriptorPool's mutex. + // Objects may therefore be constructed or moved while the GIL is NOT held. + // - Consequently, Lock() and Unlock() do not assert PyGILState_Check(), + // allowing lock acquisition and move semantics to function safely whether + // the GIL is currently held or released. Call sites that strictly require + // the GIL explicitly verify or acquire it themselves. + void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() {} + void Unlock() ABSL_UNLOCK_FUNCTION() {} +#else + // Free-threaded build: real mutex + void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { mutex_.Lock(); } + void Unlock() ABSL_UNLOCK_FUNCTION() { mutex_.Unlock(); } + + private: + absl::Mutex mutex_; +#endif +}; + +// RAII lock guard for FreeThreadingMutex. +class ABSL_SCOPED_LOCKABLE FreeThreadingLockGuard { + public: + explicit FreeThreadingLockGuard(FreeThreadingMutex& mutex) + ABSL_EXCLUSIVE_LOCK_FUNCTION(mutex) + : mutex_(mutex) { + mutex_.Lock(); + } + ~FreeThreadingLockGuard() ABSL_UNLOCK_FUNCTION() { mutex_.Unlock(); } + + FreeThreadingLockGuard(const FreeThreadingLockGuard&) = delete; + FreeThreadingLockGuard& operator=(const FreeThreadingLockGuard&) = delete; + + private: + FreeThreadingMutex& mutex_; +}; + +} // namespace cel_python + +#endif // THIRD_PARTY_CEL_PYTHON_FREE_THREADING_MUTEX_H_ diff --git a/cel_expr_python/py_cel_env.cc b/cel_expr_python/py_cel_env.cc index 68310be..6838bef 100644 --- a/cel_expr_python/py_cel_env.cc +++ b/cel_expr_python/py_cel_env.cc @@ -212,6 +212,7 @@ PyCelExpression PyCelEnv::Compile(const std::string& cel_expr, } PyCelExpression PyCelEnv::Deserialize(const std::string& serialized_expr) { + py::gil_scoped_release gil_release; return ThrowIfError(PyCelExpression::Deserialize(env_, serialized_expr)); } diff --git a/cel_expr_python/py_cel_env_internal.cc b/cel_expr_python/py_cel_env_internal.cc index cd2a58e..4bb32f8 100644 --- a/cel_expr_python/py_cel_env_internal.cc +++ b/cel_expr_python/py_cel_env_internal.cc @@ -20,12 +20,12 @@ #include #include +#include "absl/base/call_once.h" #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" -#include "absl/synchronization/mutex.h" #include "checker/type_checker_builder.h" #include "common/container.h" #include "common/function_descriptor.h" @@ -34,6 +34,7 @@ #include "compiler/compiler.h" #include "env/config.h" #include "env/env.h" +#include "env/env_runtime.h" #include "env/env_std_extensions.h" #include "env/runtime_std_extensions.h" #include "env/type_info.h" @@ -107,8 +108,24 @@ PyCelEnvInternal::PyCelEnvInternal( return extension->ConfigureRuntime(runtime_builder, runtime_options); }); } + + // PyCelType::FromCelType performs a deep copy and does not keep a + // reference to any of the arena backed cel::Type instances, so it is safe to + // use a local arena. + google::protobuf::Arena arena; + for (const cel::Config::VariableConfig& variable_config : + env_config_.GetConfig().GetVariableConfigs()) { + auto status_or_type = cel::TypeInfoToType(variable_config.type_info, + descriptor_pool_.get(), &arena); + if (status_or_type.ok()) { + variable_types_[variable_config.name] = + PyCelType::FromCelType(*status_or_type); + } + } } +PyCelEnvInternal::~PyCelEnvInternal() = default; + absl::StatusOr> PyCelEnvInternal::NewCelEnvInternal( const PyCelEnvConfig& env_config, const PyCelOptions& options, @@ -230,12 +247,8 @@ PyCelEnvInternal::NewCelEnvInternal( std::move(extension_handles), impls)); } -absl::StatusOr PyCelEnvInternal::GetCompiler() const { - absl::MutexLock lock(mutex_); - if (compiler_) { - return compiler_.get(); - } - +absl::StatusOr> PyCelEnvInternal::BuildCompiler() + const { const cel::Config& config = env_config_.GetConfig(); CEL_PYTHON_ASSIGN_OR_RETURN( @@ -259,28 +272,19 @@ absl::StatusOr PyCelEnvInternal::GetCompiler() const { } checker_builder.SetExpressionContainer(std::move(container)); - // Convert variable types from cel::TypeInfo to PyCelType. - google::protobuf::Arena* arena = checker_builder.arena(); - for (const cel::Config::VariableConfig& variable_config : - config.GetVariableConfigs()) { - CEL_PYTHON_ASSIGN_OR_RETURN( - cel::Type cel_type, cel::TypeInfoToType(variable_config.type_info, - descriptor_pool_.get(), arena)); - PyCelType py_cel_type = PyCelType::FromCelType(cel_type); - variable_types_[variable_config.name] = py_cel_type; - } - - CEL_PYTHON_ASSIGN_OR_RETURN(compiler_, compiler_builder->Build()); - return compiler_.get(); + return compiler_builder->Build(); } -absl::StatusOr PyCelEnvInternal::GetRuntime( - RuntimeMode runtime_mode) const { - absl::MutexLock lock(mutex_); - if (auto it = runtimes_.find(runtime_mode); it != runtimes_.end()) { - return it->second.get(); +absl::StatusOr PyCelEnvInternal::GetCompiler() const { + absl::call_once(compiler_once_, [this] { compiler_ = BuildCompiler(); }); + if (!compiler_.ok()) { + return compiler_.status(); } + return (*compiler_).get(); +} +absl::StatusOr> PyCelEnvInternal::BuildRuntime( + RuntimeMode runtime_mode) const { cel::RuntimeOptions opts; opts.container = env_config_.GetConfig().GetContainerConfig().name; opts.enable_empty_wrapper_null_unboxing = true; @@ -298,6 +302,10 @@ absl::StatusOr PyCelEnvInternal::GetRuntime( CEL_PYTHON_RETURN_IF_ERROR(cel::EnableReferenceResolver( builder, cel::ReferenceResolverEnabled::kAlways)); + // The local arena is only used as scratch space for intermediate cel::Type + // objects in TypeInfoToType. Parameters only retain cel::Kind (enum), and + // return types are converted to self-contained PyCelType value objects. + google::protobuf::Arena arena; for (const cel::Config::FunctionConfig& function_config : GetEnvConfig().GetConfig().GetFunctionConfigs()) { for (const cel::Config::FunctionOverloadConfig& overload_config : @@ -306,14 +314,20 @@ absl::StatusOr PyCelEnvInternal::GetRuntime( if (it == function_impls_.end()) { continue; } - py::object py_function = it->second; + py::object py_function; + if (!PyGILState_Check()) { + py::gil_scoped_acquire acquire; + py_function = it->second; + } else { + py_function = it->second; + } std::vector param_kinds; param_kinds.reserve(overload_config.parameters.size()); for (const cel::Config::TypeInfo& parameter : overload_config.parameters) { CEL_PYTHON_ASSIGN_OR_RETURN( cel::Type type, - cel::TypeInfoToType(parameter, descriptor_pool_.get(), &arena_)); + cel::TypeInfoToType(parameter, descriptor_pool_.get(), &arena)); param_kinds.push_back(static_cast(type.kind())); } cel::FunctionDescriptor descriptor( @@ -322,23 +336,41 @@ absl::StatusOr PyCelEnvInternal::GetRuntime( CEL_PYTHON_ASSIGN_OR_RETURN( cel::Type return_type, cel::TypeInfoToType(overload_config.return_type, - descriptor_pool_.get(), &arena_)); + descriptor_pool_.get(), &arena)); CEL_PYTHON_RETURN_IF_ERROR(builder.function_registry().Register( - descriptor, std::make_unique( - function_config.name, - PyCelType::FromCelType(return_type), py_function))); + descriptor, + std::make_unique( + function_config.name, PyCelType::FromCelType(return_type), + std::move(py_function)))); } } - CEL_PYTHON_ASSIGN_OR_RETURN(std::unique_ptr runtime, - std::move(builder).Build()); - const cel::Runtime* runtime_ptr = runtime.get(); - runtimes_[runtime_mode] = std::move(runtime); - return runtime_ptr; + return std::move(builder).Build(); +} + +absl::StatusOr PyCelEnvInternal::GetRuntime( + RuntimeMode runtime_mode) const { + switch (runtime_mode) { + case kStandard: + absl::call_once(standard_runtime_once_, + [this] { standard_runtime_ = BuildRuntime(kStandard); }); + if (!standard_runtime_.ok()) { + return standard_runtime_.status(); + } + return (*standard_runtime_).get(); + case kStandardIgnoreWarnings: + absl::call_once(standard_ignore_warnings_runtime_once_, [this] { + standard_ignore_warnings_runtime_ = + BuildRuntime(kStandardIgnoreWarnings); + }); + if (!standard_ignore_warnings_runtime_.ok()) { + return standard_ignore_warnings_runtime_.status(); + } + return (*standard_ignore_warnings_runtime_).get(); + } } const PyCelType& PyCelEnvInternal::GetVariableType( const std::string& name) const { - absl::MutexLock lock(mutex_); auto it = variable_types_.find(name); if (it != variable_types_.end()) { return it->second; @@ -360,8 +392,12 @@ CelExtensionHandle::CelExtensionHandle(CelExtensionHandle&& other) CelExtensionHandle::~CelExtensionHandle() { if (py_extension_ != nullptr) { - py::gil_scoped_acquire acquire; - Py_DECREF(py_extension_); + if (!PyGILState_Check()) { + py::gil_scoped_acquire acquire; + Py_DECREF(py_extension_); + } else { + Py_DECREF(py_extension_); + } } } diff --git a/cel_expr_python/py_cel_env_internal.h b/cel_expr_python/py_cel_env_internal.h index fc69b34..95bfd14 100644 --- a/cel_expr_python/py_cel_env_internal.h +++ b/cel_expr_python/py_cel_env_internal.h @@ -22,18 +22,15 @@ #include #include -#include "absl/base/thread_annotations.h" +#include "absl/base/call_once.h" #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/synchronization/mutex.h" #include "common/container.h" #include "compiler/compiler.h" #include "env/env.h" #include "env/env_runtime.h" #include "runtime/runtime.h" -#include "runtime/runtime_builder.h" -#include "runtime/runtime_options.h" #include "cel_expr_python/cel_extension.h" #include "cel_expr_python/py_cel_env_config.h" #include "cel_expr_python/py_cel_function.h" @@ -42,7 +39,6 @@ #include "cel_expr_python/py_cel_type.h" #include "cel_expr_python/py_descriptor_database.h" #include "cel_expr_python/py_message_factory.h" -#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" @@ -74,7 +70,7 @@ class CelExtensionHandle { // the python side. class PyCelEnvInternal { public: - ~PyCelEnvInternal() = default; + ~PyCelEnvInternal(); static absl::StatusOr> NewCelEnvInternal( const PyCelEnvConfig& env_config, const PyCelOptions& options, PyObject* py_descriptor_pool, @@ -122,8 +118,10 @@ class PyCelEnvInternal { std::vector extension_handles, absl::flat_hash_map& function_impls); - mutable absl::Mutex mutex_; - mutable google::protobuf::Arena arena_ ABSL_GUARDED_BY(mutex_); + absl::StatusOr> BuildCompiler() const; + absl::StatusOr> BuildRuntime( + RuntimeMode runtime_mode) const; + cel::Env cel_env_; cel::EnvRuntime cel_env_runtime_; PyCelEnvConfig env_config_; @@ -132,13 +130,16 @@ class PyCelEnvInternal { std::shared_ptr descriptor_pool_; mutable google::protobuf::DynamicMessageFactory message_factory_; std::shared_ptr py_message_factory_; - mutable absl::flat_hash_map variable_types_ - ABSL_GUARDED_BY(mutex_); + absl::flat_hash_map variable_types_; std::vector extensions_; absl::flat_hash_map function_impls_; - mutable std::unique_ptr compiler_ ABSL_GUARDED_BY(mutex_); - mutable absl::flat_hash_map> - runtimes_ ABSL_GUARDED_BY(mutex_); + mutable absl::once_flag compiler_once_; + mutable absl::StatusOr> compiler_; + mutable absl::once_flag standard_runtime_once_; + mutable absl::StatusOr> standard_runtime_; + mutable absl::once_flag standard_ignore_warnings_runtime_once_; + mutable absl::StatusOr> + standard_ignore_warnings_runtime_; }; } // namespace cel_python diff --git a/cel_expr_python/py_cel_expression.cc b/cel_expr_python/py_cel_expression.cc index 1a7b557..53084d4 100644 --- a/cel_expr_python/py_cel_expression.cc +++ b/cel_expr_python/py_cel_expression.cc @@ -43,6 +43,7 @@ #include "parser/parser_interface.h" #include "runtime/embedder_context.h" #include "runtime/runtime.h" +#include "cel_expr_python/free_threading_mutex.h" #include "cel_expr_python/py_cel_activation.h" #include "cel_expr_python/py_cel_arena.h" #include "cel_expr_python/py_cel_env_internal.h" @@ -110,7 +111,7 @@ absl::StatusOr PyCelExpression::Compile( if (disable_check) { CEL_PYTHON_ASSIGN_OR_RETURN(auto s, cel::NewSource(cel_expr, "")); - PY_CEL_PYTHON_ASSIGN_OR_RETURN(auto ast, compiler->GetParser().Parse(*s)); + CEL_PYTHON_ASSIGN_OR_RETURN(auto ast, compiler->GetParser().Parse(*s)); ParsedExpr parsed_expr; CEL_PYTHON_RETURN_IF_ERROR(cel::AstToParsedExpr(*ast, &parsed_expr)); return PyCelExpression(parsed_expr, env); @@ -127,6 +128,15 @@ absl::StatusOr PyCelExpression::Compile( return PyCelExpression(checked_expr, env); } +PyCelExpression::PyCelExpression(PyCelExpression&& other) noexcept { + FreeThreadingLockGuard lock(other.mutex_); + expr_ = std::move(other.expr_); + env_ = std::move(other.env_); + cel_program_ = std::move(other.cel_program_); +} + +PyCelExpression::~PyCelExpression() = default; + PyCelType PyCelExpression::GetReturnType() { if (!std::holds_alternative(expr_)) { return PyCelType::Dyn(); @@ -144,36 +154,42 @@ PyCelType PyCelExpression::GetReturnType() { return PyCelType::FromTypeProto(it->second); } +absl::StatusOr PyCelExpression::GetProgram() { + FreeThreadingLockGuard lock(mutex_); + if (cel_program_) { + return cel_program_.get(); + } + if (std::holds_alternative(expr_)) { + CEL_PYTHON_ASSIGN_OR_RETURN( + const cel::Runtime* runtime, + env_->GetRuntime(PyCelEnvInternal::kStandardIgnoreWarnings)); + CEL_PYTHON_ASSIGN_OR_RETURN( + cel_program_, cel::extensions::ProtobufRuntimeAdapter::CreateProgram( + *runtime, std::get(expr_))); + } else { + CEL_PYTHON_ASSIGN_OR_RETURN(const cel::Runtime* runtime, + env_->GetRuntime(PyCelEnvInternal::kStandard)); + CEL_PYTHON_ASSIGN_OR_RETURN( + cel_program_, cel::extensions::ProtobufRuntimeAdapter::CreateProgram( + *runtime, std::get(expr_))); + } + return cel_program_.get(); +} + absl::StatusOr PyCelExpression::Eval( const PyCelActivation& activation) { ABSL_CHECK(PyGILState_Check()); - if (cel_program_ == nullptr) { - if (std::holds_alternative(expr_)) { - PY_CEL_PYTHON_ASSIGN_OR_RETURN( - const cel::Runtime* runtime, - env_->GetRuntime(PyCelEnvInternal::kStandardIgnoreWarnings)); - PY_CEL_PYTHON_ASSIGN_OR_RETURN( - cel_program_, cel::extensions::ProtobufRuntimeAdapter::CreateProgram( - *runtime, std::get(expr_))); - } else { - PY_CEL_PYTHON_ASSIGN_OR_RETURN( - const cel::Runtime* runtime, - env_->GetRuntime(PyCelEnvInternal::kStandard)); - PY_CEL_PYTHON_ASSIGN_OR_RETURN( - cel_program_, cel::extensions::ProtobufRuntimeAdapter::CreateProgram( - *runtime, std::get(expr_))); - } - } + CEL_PYTHON_ASSIGN_OR_RETURN(const cel::Program* program, GetProgram()); std::shared_ptr arena = activation.GetArena(); std::shared_ptr env = activation.GetEnv(); cel::EmbedderContext embedder_context = cel::EmbedderContext::From(&env); cel::EvaluateOptions options; options.message_factory = env->GetMessageFactory(); options.embedder_context = &embedder_context; - PY_CEL_PYTHON_ASSIGN_OR_RETURN( + CEL_PYTHON_ASSIGN_OR_RETURN( cel::Value result, - cel_program_->Evaluate(arena->GetArena(), *activation.GetActivation(), - std::move(options))); + program->Evaluate(arena->GetArena(), *activation.GetActivation(), + std::move(options))); return PyCelValue(result, arena, std::move(env)); } @@ -190,7 +206,6 @@ std::string PyCelExpression::Serialize() const { absl::StatusOr PyCelExpression::Deserialize( const std::shared_ptr& env, const std::string& serialized_expr) { - ABSL_CHECK(PyGILState_Check()); google::protobuf::Any any; if (!any.ParseFromString(serialized_expr)) { return absl::InvalidArgumentError( diff --git a/cel_expr_python/py_cel_expression.h b/cel_expr_python/py_cel_expression.h index e4f59aa..acc1287 100644 --- a/cel_expr_python/py_cel_expression.h +++ b/cel_expr_python/py_cel_expression.h @@ -22,8 +22,10 @@ #include "cel/expr/checked.pb.h" #include "cel/expr/syntax.pb.h" +#include "absl/base/thread_annotations.h" #include "absl/status/statusor.h" #include "runtime/runtime.h" +#include "cel_expr_python/free_threading_mutex.h" #include "cel_expr_python/py_cel_activation.h" #include "cel_expr_python/py_cel_type.h" #include "cel_expr_python/py_cel_value.h" @@ -38,14 +40,15 @@ class PyCelExpression { public: static void DefinePythonBindings(pybind11::module& m); - PyCelExpression(PyCelExpression&& other) = default; + PyCelExpression(PyCelExpression&& other) noexcept; + ~PyCelExpression(); PyCelExpression(const cel::expr::ParsedExpr& parsed_expr, std::shared_ptr env) - : expr_(std::move(parsed_expr)), env_(std::move(env)) {} + : expr_(parsed_expr), env_(std::move(env)) {} PyCelExpression(const cel::expr::CheckedExpr& checked_expr, std::shared_ptr env) - : expr_(std::move(checked_expr)), env_(std::move(env)) {} + : expr_(checked_expr), env_(std::move(env)) {} PyCelType GetReturnType(); @@ -62,10 +65,13 @@ class PyCelExpression { const std::string& serialized_expr); private: + absl::StatusOr GetProgram(); + std::variant expr_; std::shared_ptr env_; - std::unique_ptr cel_program_; + mutable FreeThreadingMutex mutex_; + std::unique_ptr cel_program_ ABSL_GUARDED_BY(mutex_); }; } // namespace cel_python diff --git a/cel_expr_python/py_cel_function.cc b/cel_expr_python/py_cel_function.cc index c4e7a63..5ddd2b4 100644 --- a/cel_expr_python/py_cel_function.cc +++ b/cel_expr_python/py_cel_function.cc @@ -95,12 +95,14 @@ absl::StatusOr PyCelFunctionAdapter::Invoke( /*plain_value=*/true)); } PyObject* result = PyObject_CallObject(py_function_.ptr(), py_args); + Py_DECREF(py_args); absl::Status status = PyErr_toStatus(); if (!status.ok()) { + Py_XDECREF(result); return cel::ErrorValue(status); } - return PyObjectToCelValue( + absl::StatusOr cel_result = PyObjectToCelValue( result, return_type_, [this]() { return absl::StrFormat( @@ -108,6 +110,8 @@ absl::StatusOr PyCelFunctionAdapter::Invoke( PyUnicode_AsUTF8(PyObject_Repr(py_function_.ptr()))); }, env, context.arena()); -}; + Py_XDECREF(result); + return cel_result; +} } // namespace cel_python diff --git a/cel_expr_python/py_cel_function_decl.cc b/cel_expr_python/py_cel_function_decl.cc index 382fdae..b5f084b 100644 --- a/cel_expr_python/py_cel_function_decl.cc +++ b/cel_expr_python/py_cel_function_decl.cc @@ -16,13 +16,10 @@ #include #include -#include #include #include "env/config.h" -#include "env/type_info.h" #include "cel_expr_python/py_cel_overload.h" -#include "cel_expr_python/py_cel_type.h" #include #include diff --git a/cel_expr_python/py_cel_module.cc b/cel_expr_python/py_cel_module.cc index aca47fd..8879e00 100644 --- a/cel_expr_python/py_cel_module.cc +++ b/cel_expr_python/py_cel_module.cc @@ -24,11 +24,13 @@ #include "cel_expr_python/py_cel_python_extension.h" #include "cel_expr_python/py_cel_type.h" #include "cel_expr_python/py_cel_value.h" +#include "cel_expr_python/py_error_status.h" #include namespace cel_python { -PYBIND11_MODULE(cel, m) { +PYBIND11_MODULE(cel, m, pybind11::mod_gil_not_used()) { + InitPyErrorStatus(); m.doc() = "Python bindings for CEL."; PyCelArena::DefinePythonBindings(m); diff --git a/cel_expr_python/py_cel_type.cc b/cel_expr_python/py_cel_type.cc index be49498..52e1750 100644 --- a/cel_expr_python/py_cel_type.cc +++ b/cel_expr_python/py_cel_type.cc @@ -29,6 +29,7 @@ #include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" +#include "common/ast.h" #include "common/kind.h" #include "common/signature.h" #include "common/type.h" diff --git a/cel_expr_python/py_cel_value.cc b/cel_expr_python/py_cel_value.cc index 131de8d..df26c46 100644 --- a/cel_expr_python/py_cel_value.cc +++ b/cel_expr_python/py_cel_value.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -36,6 +37,7 @@ #include "common/type.h" #include "common/value.h" #include "common/value_kind.h" +#include "cel_expr_python/free_threading_mutex.h" #include "cel_expr_python/py_cel_arena.h" #include "cel_expr_python/py_cel_env_internal.h" #include "cel_expr_python/py_cel_type.h" @@ -58,11 +60,19 @@ void PyCelValue::DefinePythonBindings(py::module& m) { .def("type", &PyCelValue::Type) .def("value", [](PyCelValue& self) { - return py::reinterpret_borrow(self.Value()); + PyObject* obj = self.Value(); + if (obj == nullptr) { + throw py::error_already_set(); + } + return py::reinterpret_borrow(obj); }) .def("plain_value", [](PyCelValue& self) { - return py::reinterpret_borrow(self.PlainValue()); + PyObject* obj = self.PlainValue(); + if (obj == nullptr) { + throw py::error_already_set(); + } + return py::reinterpret_borrow(obj); }) .def("__repr__", &PyCelValue::ToString); @@ -79,12 +89,28 @@ PyCelValue::PyCelValue(cel::Value& cel_value, std::shared_ptr arena, arena_(std::move(arena)), env_(std::move(env)) {} +PyCelValue::PyCelValue(PyCelValue&& other) noexcept + : object_(nullptr), plain_object_(nullptr) { + FreeThreadingLockGuard lock(other.mutex_); + cel_value_ = std::move(other.cel_value_); + arena_ = std::move(other.arena_); + env_ = std::move(other.env_); + object_ = other.object_; + plain_object_ = other.plain_object_; + other.object_ = nullptr; + other.plain_object_ = nullptr; +} + PyCelValue::~PyCelValue() { if (object_ || plain_object_) { - auto gil_state = PyGILState_Ensure(); - Py_XDECREF(object_); - Py_XDECREF(plain_object_); - PyGILState_Release(gil_state); + if (!PyGILState_Check()) { + py::gil_scoped_acquire acquire; + Py_XDECREF(object_); + Py_XDECREF(plain_object_); + } else { + Py_XDECREF(object_); + Py_XDECREF(plain_object_); + } } } @@ -92,12 +118,13 @@ PyCelType PyCelValue::Type() { return PyCelType::ForCelValue(cel_value_); } PyObject* PyCelValue::Value() { ABSL_CHECK(PyGILState_Check()); + FreeThreadingLockGuard lock(mutex_); if (object_) { return object_; } object_ = CelValueToPyObject(cel_value_, env_, arena_, /*plain_value=*/false); - if (object_ == nullptr) { + if (object_ == nullptr && !PyErr_Occurred()) { PyErr_SetString(PyExc_AssertionError, "Cannot create object"); } @@ -106,12 +133,13 @@ PyObject* PyCelValue::Value() { PyObject* PyCelValue::PlainValue() { ABSL_CHECK(PyGILState_Check()); + FreeThreadingLockGuard lock(mutex_); if (plain_object_) { return plain_object_; } plain_object_ = CelValueToPyObject(cel_value_, env_, arena_, /*plain_value=*/true); - if (plain_object_ == nullptr) { + if (plain_object_ == nullptr && !PyErr_Occurred()) { PyErr_SetString(PyExc_AssertionError, "Cannot create object"); } @@ -127,9 +155,12 @@ PyCelValueProvider::PyCelValueProvider(std::string name, PyObject* value, } PyCelValueProvider::~PyCelValueProvider() { - auto gil_state = PyGILState_Ensure(); - Py_DECREF(py_object_); - PyGILState_Release(gil_state); + if (!PyGILState_Check()) { + py::gil_scoped_acquire acquire; + Py_DECREF(py_object_); + } else { + Py_DECREF(py_object_); + } } cel::Value PyCelValueProvider::Provide( @@ -145,7 +176,15 @@ cel::Value PyCelValueProvider::Provide( return *converted_value; } -void PyCelListItemAccessor::ResolveElement() { +PyCelListItemAccessor::PyCelListItemAccessor( + PyCelListItemAccessor&& other) noexcept + : PyCelValue(std::move(other)), index_(other.index_) { + FreeThreadingLockGuard lock(other.mutex_); + resolved_ = other.resolved_; + element_value_ = std::move(other.element_value_); +} + +void PyCelListItemAccessor::ResolveElementLocked() { if (resolved_) { return; } @@ -163,47 +202,61 @@ void PyCelListItemAccessor::ResolveElement() { } PyCelType PyCelListItemAccessor::Type() { - auto gil_state = PyGILState_Ensure(); - ResolveElement(); - PyGILState_Release(gil_state); + FreeThreadingLockGuard lock(mutex_); + ResolveElementLocked(); return PyCelType::ForCelValue(element_value_); } PyObject* PyCelListItemAccessor::Value() { ABSL_CHECK(PyGILState_Check()); + FreeThreadingLockGuard lock(mutex_); if (object_) { return object_; } - ResolveElement(); + ResolveElementLocked(); object_ = CelValueToPyObject(element_value_, env_, arena_, /*plain_value=*/false); + if (object_ == nullptr && !PyErr_Occurred()) { + PyErr_SetString(PyExc_AssertionError, "Cannot create object"); + } return object_; } PyObject* PyCelListItemAccessor::PlainValue() { ABSL_CHECK(PyGILState_Check()); + FreeThreadingLockGuard lock(mutex_); if (plain_object_) { return plain_object_; } - ResolveElement(); + ResolveElementLocked(); plain_object_ = CelValueToPyObject(element_value_, env_, arena_, /*plain_value=*/true); + if (plain_object_ == nullptr && !PyErr_Occurred()) { + PyErr_SetString(PyExc_AssertionError, "Cannot create object"); + } return plain_object_; } std::string PyCelListItemAccessor::ToString() { - auto gil_state = PyGILState_Ensure(); - ResolveElement(); - std::string result = element_value_.DebugString(); - PyGILState_Release(gil_state); - return result; + FreeThreadingLockGuard lock(mutex_); + ResolveElementLocked(); + return element_value_.DebugString(); +} + +PyCelMapItemAccessor::PyCelMapItemAccessor( + PyCelMapItemAccessor&& other) noexcept + : PyCelValue(std::move(other)) { + FreeThreadingLockGuard lock(other.mutex_); + key_ = std::move(other.key_); + element_value_ = std::move(other.element_value_); + resolved_ = other.resolved_; } -void PyCelMapItemAccessor::ResolveElement() { +void PyCelMapItemAccessor::ResolveElementLocked() { if (resolved_) { return; } @@ -220,44 +273,49 @@ void PyCelMapItemAccessor::ResolveElement() { } PyCelType PyCelMapItemAccessor::Type() { - auto gil_state = PyGILState_Ensure(); - ResolveElement(); - PyGILState_Release(gil_state); + FreeThreadingLockGuard lock(mutex_); + ResolveElementLocked(); return PyCelType::ForCelValue(element_value_); } PyObject* PyCelMapItemAccessor::Value() { ABSL_CHECK(PyGILState_Check()); + FreeThreadingLockGuard lock(mutex_); if (object_) { return object_; } - ResolveElement(); + ResolveElementLocked(); object_ = CelValueToPyObject(element_value_, env_, arena_, /*plain_value=*/false); + if (object_ == nullptr && !PyErr_Occurred()) { + PyErr_SetString(PyExc_AssertionError, "Cannot create object"); + } return object_; } PyObject* PyCelMapItemAccessor::PlainValue() { ABSL_CHECK(PyGILState_Check()); + FreeThreadingLockGuard lock(mutex_); if (plain_object_) { return plain_object_; } - ResolveElement(); + ResolveElementLocked(); plain_object_ = CelValueToPyObject(element_value_, env_, arena_, /*plain_value=*/true); + if (plain_object_ == nullptr && !PyErr_Occurred()) { + PyErr_SetString(PyExc_AssertionError, "Cannot create object"); + } return plain_object_; } std::string PyCelMapItemAccessor::ToString() { - auto gil_state = PyGILState_Ensure(); - ResolveElement(); - std::string result = element_value_.DebugString(); - PyGILState_Release(gil_state); - return result; + FreeThreadingLockGuard lock(mutex_); + ResolveElementLocked(); + return element_value_.DebugString(); } // This should be called with the GIL held. @@ -267,6 +325,7 @@ PyObject* CelValueToPyObject(const cel::Value& cel_value, bool plain_value) { switch (cel_value.kind()) { case cel::ValueKind::kNull: { + Py_INCREF(Py_None); return Py_None; } case cel::ValueKind::kBool: { diff --git a/cel_expr_python/py_cel_value.h b/cel_expr_python/py_cel_value.h index c79a680..5dae8e4 100644 --- a/cel_expr_python/py_cel_value.h +++ b/cel_expr_python/py_cel_value.h @@ -21,17 +21,17 @@ #include #include +#include "absl/base/thread_annotations.h" #include "absl/functional/function_ref.h" #include "absl/status/statusor.h" #include "common/value.h" +#include "cel_expr_python/free_threading_mutex.h" #include "cel_expr_python/py_cel_type.h" #include "google/protobuf/arena.h" #include namespace cel_python { -namespace py = ::pybind11; - class PyCelArena; class PyCelEnvInternal; class PyMessageFactory; @@ -47,13 +47,13 @@ class PyCelValue { PyCelValue(cel::Value& cel_value, std::shared_ptr arena, std::shared_ptr env); - // Move constructor and assignment. - PyCelValue(PyCelValue&& other) noexcept = default; - PyCelValue& operator=(PyCelValue&& other) noexcept = default; + // Move constructor. + PyCelValue(PyCelValue&& other) noexcept; - // Disallow copying. + // Disallow copying and move assignment. PyCelValue(const PyCelValue&) = delete; PyCelValue& operator=(const PyCelValue&) = delete; + PyCelValue& operator=(PyCelValue&&) = delete; virtual ~PyCelValue(); @@ -67,9 +67,10 @@ class PyCelValue { PyMessageFactory* py_message_factory); protected: + mutable FreeThreadingMutex mutex_; cel::Value cel_value_; - PyObject* object_; - PyObject* plain_object_; + PyObject* object_ ABSL_GUARDED_BY(mutex_); + PyObject* plain_object_ ABSL_GUARDED_BY(mutex_); std::shared_ptr arena_; std::shared_ptr env_; }; @@ -83,22 +84,21 @@ class PyCelListItemAccessor : public PyCelValue { : PyCelValue(celValue, std::move(arena), std::move(env)), index_(index) {} // Move constructor. - PyCelListItemAccessor(PyCelListItemAccessor&& other) noexcept = default; + PyCelListItemAccessor(PyCelListItemAccessor&& other) noexcept; ~PyCelListItemAccessor() override = default; - // Extracts the element at the given index from the list and caches the - // result. This is called on demand when the python side accesses the value. - void ResolveElement(); PyCelType Type() override; PyObject* Value() override; PyObject* PlainValue() override; std::string ToString() override; private: + void ResolveElementLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + int index_; - bool resolved_ = false; - cel::Value element_value_; + bool resolved_ ABSL_GUARDED_BY(mutex_) = false; + cel::Value element_value_ ABSL_GUARDED_BY(mutex_); }; // Variant of PyCelValue that is used to access a specific value from a map. @@ -111,20 +111,21 @@ class PyCelMapItemAccessor : public PyCelValue { key_(std::move(key)) {} // Move constructor. - PyCelMapItemAccessor(PyCelMapItemAccessor&& other) noexcept = default; + PyCelMapItemAccessor(PyCelMapItemAccessor&& other) noexcept; ~PyCelMapItemAccessor() override = default; - void ResolveElement(); PyCelType Type() override; PyObject* Value() override; PyObject* PlainValue() override; std::string ToString() override; private: + void ResolveElementLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + cel::Value key_; - cel::Value element_value_; - bool resolved_ = false; + cel::Value element_value_ ABSL_GUARDED_BY(mutex_); + bool resolved_ ABSL_GUARDED_BY(mutex_) = false; }; PyObject* CelValueToPyObject(const cel::Value& cel_value, diff --git a/cel_expr_python/py_descriptor_database.cc b/cel_expr_python/py_descriptor_database.cc index f6e54ef..bbbb29f 100644 --- a/cel_expr_python/py_descriptor_database.cc +++ b/cel_expr_python/py_descriptor_database.cc @@ -39,8 +39,15 @@ PyDescriptorDatabase::PyDescriptorDatabase(PyObject* py_descriptor_pool) } PyDescriptorDatabase::~PyDescriptorDatabase() { - py::gil_scoped_acquire acquire; - Py_XDECREF(py_descriptor_pool_); + if (py_descriptor_pool_ == nullptr) { + return; + } + if (!PyGILState_Check()) { + py::gil_scoped_acquire acquire; + Py_XDECREF(py_descriptor_pool_); + } else { + Py_XDECREF(py_descriptor_pool_); + } } // Find a file by file name. Fills in in *output and returns true if found. diff --git a/cel_expr_python/py_descriptor_database.h b/cel_expr_python/py_descriptor_database.h index 5f621f4..3a4a45e 100644 --- a/cel_expr_python/py_descriptor_database.h +++ b/cel_expr_python/py_descriptor_database.h @@ -22,6 +22,7 @@ #include // IWYU pragma: keep - Needed for string_view in OSS #include "google/protobuf/descriptor.pb.h" +#include "absl/strings/string_view.h" #include "google/protobuf/descriptor_database.h" namespace cel_python { diff --git a/cel_expr_python/py_error_status.cc b/cel_expr_python/py_error_status.cc index 4565c0e..06dbd59 100644 --- a/cel_expr_python/py_error_status.cc +++ b/cel_expr_python/py_error_status.cc @@ -103,8 +103,7 @@ static absl::Status& PendingPyError() { return *pending_py_error; } -absl::Status PyErr_toStatus() { - py::gil_scoped_acquire acquire; +static absl::Status PyErr_toStatusInternal() { PyObject* py_error = PyErr_Occurred(); if (!py_error) { absl::Status status = PendingPyError(); @@ -135,8 +134,15 @@ absl::Status PyErr_toStatus() { return status; } +absl::Status PyErr_toStatus() { + if (!PyGILState_Check()) { + py::gil_scoped_acquire acquire; + return PyErr_toStatusInternal(); + } + return PyErr_toStatusInternal(); +} + void PyErr_noteAndClear() { - py::gil_scoped_acquire acquire; if (!PyErr_Occurred()) { return; } diff --git a/cel_expr_python/py_error_status.h b/cel_expr_python/py_error_status.h index 2ecea98..574dbbe 100644 --- a/cel_expr_python/py_error_status.h +++ b/cel_expr_python/py_error_status.h @@ -31,6 +31,16 @@ namespace cel_python { +// Eagerly initializes pybind11 internal data structures (type casters, +// exception translation tables, and instance maps) during module +// initialization. +// +// This guarantees that pybind11's global internals are initialized on the main +// thread at import time, preventing data races or lazy-initialization +// contention when Status/exception conversions occur concurrently across +// multiple threads. +inline void InitPyErrorStatus() { pybind11::detail::get_internals(); } + std::runtime_error StatusToException(const absl::Status& status); void ThrowIfError(const absl::Status& status); diff --git a/cel_expr_python/py_message_factory.cc b/cel_expr_python/py_message_factory.cc index d3475f6..080b33b 100644 --- a/cel_expr_python/py_message_factory.cc +++ b/cel_expr_python/py_message_factory.cc @@ -21,9 +21,13 @@ #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "cel_expr_python/free_threading_mutex.h" +#include namespace cel_python { +namespace py = pybind11; + PyMessageFactory::PyMessageFactory(PyObject* descriptor_pool) { py_descriptor_pool_ = descriptor_pool; if (py_descriptor_pool_ == nullptr) { @@ -52,17 +56,26 @@ PyMessageFactory::~PyMessageFactory() { return; } - auto gil_state = PyGILState_Ensure(); - Py_XDECREF(py_descriptor_pool_); - Py_XDECREF(py_func_GetMessageClass_); - Py_XDECREF(py_func_MergeFromString_); - for (auto const& [key, py_obj] : message_classes_) { - Py_XDECREF(py_obj); + if (!PyGILState_Check()) { + py::gil_scoped_acquire acquire; + Py_XDECREF(py_descriptor_pool_); + Py_XDECREF(py_func_GetMessageClass_); + Py_XDECREF(py_func_MergeFromString_); + for (auto const& [key, py_obj] : message_classes_) { + Py_XDECREF(py_obj); + } + } else { + Py_XDECREF(py_descriptor_pool_); + Py_XDECREF(py_func_GetMessageClass_); + Py_XDECREF(py_func_MergeFromString_); + for (auto const& [key, py_obj] : message_classes_) { + Py_XDECREF(py_obj); + } } - PyGILState_Release(gil_state); } PyObject* PyMessageFactory::GetMessageClass(const std::string& message_type) { + ABSL_CHECK(PyGILState_Check()); if (py_descriptor_pool_ == nullptr) { PyErr_Format(PyExc_TypeError, "Message type not found: %s, descriptor pool is unavailable.", @@ -70,31 +83,38 @@ PyObject* PyMessageFactory::GetMessageClass(const std::string& message_type) { return nullptr; } - auto it = message_classes_.find(message_type); - if (it != message_classes_.end()) { - return it->second; - } else { - PyObject* descriptor = - PyObject_CallMethod(py_descriptor_pool_, "FindMessageTypeByName", "s", - message_type.c_str()); - if (!descriptor) { - PyErr_Format(PyExc_TypeError, "Message type not found: %s", - message_type.c_str()); - return nullptr; - } - PyObject* message_class = - PyObject_CallFunction(py_func_GetMessageClass_, "O", descriptor); - Py_DECREF(descriptor); - - if (!message_class) { - PyErr_Format(PyExc_TypeError, "Couldn't find message class for type: %s", - message_type.c_str()); - return nullptr; + { + FreeThreadingLockGuard lock(mutex_); + auto it = message_classes_.find(message_type); + if (it != message_classes_.end()) { + return it->second; } + } - message_classes_[message_type] = message_class; - return message_class; + PyObject* descriptor = PyObject_CallMethod( + py_descriptor_pool_, "FindMessageTypeByName", "s", message_type.c_str()); + if (!descriptor) { + PyErr_Format(PyExc_TypeError, "Message type not found: %s", + message_type.c_str()); + return nullptr; + } + PyObject* message_class = + PyObject_CallFunction(py_func_GetMessageClass_, "O", descriptor); + Py_DECREF(descriptor); + + if (!message_class) { + PyErr_Format(PyExc_TypeError, "Couldn't find message class for type: %s", + message_type.c_str()); + return nullptr; + } + + FreeThreadingLockGuard lock(mutex_); + auto [it, inserted] = message_classes_.emplace(message_type, message_class); + if (!inserted) { + Py_DECREF(message_class); + return it->second; } + return message_class; } PyObject* PyMessageFactory::FromString(const std::string& message_type, diff --git a/cel_expr_python/py_message_factory.h b/cel_expr_python/py_message_factory.h index 4811ed3..bb1991d 100644 --- a/cel_expr_python/py_message_factory.h +++ b/cel_expr_python/py_message_factory.h @@ -20,7 +20,9 @@ #include +#include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" +#include "cel_expr_python/free_threading_mutex.h" namespace cel_python { @@ -37,7 +39,9 @@ class PyMessageFactory { PyObject* py_descriptor_pool_; PyObject* py_func_GetMessageClass_; // NOLINT - Python function name. PyObject* py_func_MergeFromString_; // NOLINT - Python function name. - absl::flat_hash_map message_classes_; + mutable FreeThreadingMutex mutex_; + absl::flat_hash_map message_classes_ + ABSL_GUARDED_BY(mutex_); }; } // namespace cel_python diff --git a/test_freethreaded.sh b/test_freethreaded.sh new file mode 100755 index 0000000..a5c9f82 --- /dev/null +++ b/test_freethreaded.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs CEL Python tests in free-threaded mode (PEP 703 / Python 3.13t). +# Supports both Blaze (Google internal) and Bazel (open-source) toolchains. + +set -euo pipefail + +# Detect whether we're using Blaze or Bazel. +if [ -n "${BLAZE_BIN:-}" ]; then + BUILD_TOOL="${BLAZE_BIN}" + IS_BLAZE=true +elif [ -n "${BAZEL_BIN:-}" ]; then + BUILD_TOOL="${BAZEL_BIN}" + IS_BLAZE=false +elif command -v blaze >/dev/null 2>&1 && [ -d "third_party/cel/python" ]; then + BUILD_TOOL="blaze" + IS_BLAZE=true +elif command -v bazel >/dev/null 2>&1; then + BUILD_TOOL="bazel" + IS_BLAZE=false +elif command -v bazelisk >/dev/null 2>&1; then + BUILD_TOOL="bazelisk" + IS_BLAZE=false +elif command -v blaze >/dev/null 2>&1; then + BUILD_TOOL="blaze" + IS_BLAZE=true +else + echo "Error: Neither blaze nor bazel found in PATH." >&2 + exit 1 +fi + +if [ "${IS_BLAZE}" = true ]; then + FREETHREADED_FLAG="--//third_party/bazel_rules/rules_python/python/config_settings:py_freethreaded=yes" + DEFAULT_TARGETS=("//third_party/cel/python/...") +else + FREETHREADED_FLAG="--@rules_python//python/config_settings:py_freethreaded=yes" + DEFAULT_TARGETS=("//...") +fi + +if [ "$#" -eq 0 ]; then + exec "${BUILD_TOOL}" test \ + "${FREETHREADED_FLAG}" \ + --test_env=PYTHON_GIL=0 \ + "${DEFAULT_TARGETS[@]}" +else + exec "${BUILD_TOOL}" test \ + "${FREETHREADED_FLAG}" \ + --test_env=PYTHON_GIL=0 \ + "$@" +fi