From cfdb144ab329b51abb8b4387e9a5ad1fe8ce48ec Mon Sep 17 00:00:00 2001 From: cirilla-zmh Date: Tue, 24 Feb 2026 17:38:07 +0800 Subject: [PATCH 1/4] Add loongsuite baggage processor - LoongSuiteBaggageSpanProcessor with prefix matching and stripping - Reads baggage from parent context and adds to span attributes Change-Id: I016518ccab3210344c239297644e7e3b5596902a Co-developed-by: Cursor Co-authored-by: Cursor --- .../loongsuite-processor-baggage/CHANGELOG.md | 18 ++ .../loongsuite-processor-baggage/LICENSE | 201 ++++++++++++++++++ .../loongsuite-processor-baggage/README.rst | 81 +++++++ .../pyproject.toml | 46 ++++ .../src/loongsuite/__init__.py | 14 ++ .../src/loongsuite/processor/__init__.py | 14 ++ .../loongsuite/processor/baggage/__init__.py | 18 ++ .../loongsuite/processor/baggage/processor.py | 129 +++++++++++ .../loongsuite/processor/baggage/version.py | 15 ++ .../test-requirements.txt | 5 + .../tests/__init__.py | 14 ++ .../tests/test_baggage_processor.py | 171 +++++++++++++++ 12 files changed, 726 insertions(+) create mode 100644 processor/loongsuite-processor-baggage/CHANGELOG.md create mode 100644 processor/loongsuite-processor-baggage/LICENSE create mode 100644 processor/loongsuite-processor-baggage/README.rst create mode 100644 processor/loongsuite-processor-baggage/pyproject.toml create mode 100644 processor/loongsuite-processor-baggage/src/loongsuite/__init__.py create mode 100644 processor/loongsuite-processor-baggage/src/loongsuite/processor/__init__.py create mode 100644 processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/__init__.py create mode 100644 processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/processor.py create mode 100644 processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/version.py create mode 100644 processor/loongsuite-processor-baggage/test-requirements.txt create mode 100644 processor/loongsuite-processor-baggage/tests/__init__.py create mode 100644 processor/loongsuite-processor-baggage/tests/test_baggage_processor.py diff --git a/processor/loongsuite-processor-baggage/CHANGELOG.md b/processor/loongsuite-processor-baggage/CHANGELOG.md new file mode 100644 index 000000000..8b67f34c5 --- /dev/null +++ b/processor/loongsuite-processor-baggage/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +## Version 0.1.0 + +### Added + +- Initial release of LoongSuite Baggage Span Processor +- Support for prefix matching to filter baggage keys +- Support for prefix stripping to remove prefixes from baggage keys before adding to span attributes +- Integration with LoongSuite Configurator via environment variables + diff --git a/processor/loongsuite-processor-baggage/LICENSE b/processor/loongsuite-processor-baggage/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/processor/loongsuite-processor-baggage/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/processor/loongsuite-processor-baggage/README.rst b/processor/loongsuite-processor-baggage/README.rst new file mode 100644 index 000000000..054fae8fe --- /dev/null +++ b/processor/loongsuite-processor-baggage/README.rst @@ -0,0 +1,81 @@ +LoongSuite Baggage Span Processor +================================== + +The LoongSuite Baggage Span Processor reads entries stored in Baggage +from the parent context and adds the baggage entries' keys and values +to the span as attributes on span start. + +This processor supports: +- Prefix matching: Only process baggage keys that match specified prefixes +- Prefix stripping: Remove specified prefixes from baggage keys before adding to attributes + +Installation +------------ + +:: + + pip install loongsuite-processor-baggage + +Usage +----- + +Add the span processor when configuring the tracer provider. + +Example 1: Match specific prefixes and strip one of them + +:: + + from loongsuite.processor.baggage import LoongSuiteBaggageSpanProcessor + from opentelemetry.sdk.trace import TracerProvider + + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic.", "app."}, + strip_prefixes={"traffic."} + ) + ) + + # baggage: traffic.hello_key = "value" + # Result: attributes will have hello_key = "value" (traffic. prefix stripped) + + # baggage: app.user_id = "123" + # Result: attributes will have app.user_id = "123" (app. prefix not stripped) + +Example 2: Allow all prefixes but strip specific ones + +:: + + from loongsuite.processor.baggage import LoongSuiteBaggageSpanProcessor + from opentelemetry.sdk.trace import TracerProvider + + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes=None, # Allow all + strip_prefixes={"traffic.", "app."} + ) + ) + +Example 3: Only match specific prefixes without stripping + +:: + + from loongsuite.processor.baggage import LoongSuiteBaggageSpanProcessor + from opentelemetry.sdk.trace import TracerProvider + + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"loongsuite."}, + strip_prefixes=None # No stripping + ) + ) + +⚠ Warning ⚠️ + +Do not put sensitive information in Baggage. + +To repeat: a consequence of adding data to Baggage is that the keys and +values will appear in all outgoing HTTP headers from the application. + diff --git a/processor/loongsuite-processor-baggage/pyproject.toml b/processor/loongsuite-processor-baggage/pyproject.toml new file mode 100644 index 000000000..7f10eb7a9 --- /dev/null +++ b/processor/loongsuite-processor-baggage/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "loongsuite-processor-baggage" +dynamic = ["version"] +description = "LoongSuite Baggage Span Processor with prefix matching and stripping" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.9" +authors = [ + { name = "LoongSuite Python Agent Authors", email = "zmh405877@alibaba-inc.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "opentelemetry-api ~= 1.5", + "opentelemetry-sdk ~= 1.5", +] + +[project.urls] +Homepage = "https://github.com/alibaba/loongsuite-python-agent" +Repository = "https://github.com/alibaba/loongsuite-python-agent" + +[tool.hatch.version] +path = "src/loongsuite/processor/baggage/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/loongsuite"] diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/__init__.py b/processor/loongsuite-processor-baggage/src/loongsuite/__init__.py new file mode 100644 index 000000000..f87ce79b7 --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/__init__.py @@ -0,0 +1,14 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/processor/__init__.py b/processor/loongsuite-processor-baggage/src/loongsuite/processor/__init__.py new file mode 100644 index 000000000..f87ce79b7 --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/processor/__init__.py @@ -0,0 +1,14 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/__init__.py b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/__init__.py new file mode 100644 index 000000000..83844521d --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/__init__.py @@ -0,0 +1,18 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .processor import LoongSuiteBaggageSpanProcessor +from .version import __version__ + +__all__ = ["LoongSuiteBaggageSpanProcessor", "__version__"] diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/processor.py b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/processor.py new file mode 100644 index 000000000..6221bb941 --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/processor.py @@ -0,0 +1,129 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Set + +from opentelemetry.baggage import get_all as get_all_baggage +from opentelemetry.context import Context +from opentelemetry.sdk.trace import SpanProcessor +from opentelemetry.trace import Span + + +class LoongSuiteBaggageSpanProcessor(SpanProcessor): + """ + LoongSuite Baggage Span Processor + + Reads Baggage entries from the parent context and adds matching baggage + key-value pairs to span attributes based on configured prefix matching rules. + + Supported features: + 1. Prefix matching: Only process baggage keys that match specified prefixes + 2. Prefix stripping: Remove specified prefixes before writing to attributes + + Example: + # Configure matching prefixes: "traffic.", "app." + # Configure stripping prefix: "traffic." + # baggage: traffic.hello_key = "value" + # Result: attributes will have hello_key = "value" (prefix stripped) + + # baggage: app.user_id = "123" + # Result: attributes will have app.user_id = "123" (app. prefix not stripped) + + ⚠ Warning ⚠️ + + Do not put sensitive information in Baggage. + + To repeat: a consequence of adding data to Baggage is that the keys and + values will appear in all outgoing HTTP headers from the application. + """ + + def __init__( + self, + allowed_prefixes: Optional[Set[str]] = None, + strip_prefixes: Optional[Set[str]] = None, + ) -> None: + """ + Initialize LoongSuite Baggage Span Processor + + Args: + allowed_prefixes: Set of allowed baggage key prefixes. If None or empty, + all baggage keys are allowed. If specified, only keys + matching these prefixes will be processed. + strip_prefixes: Set of prefixes to strip. If a baggage key matches these + prefixes, they will be removed before writing to attributes. + """ + self._allowed_prefixes = allowed_prefixes or set() + self._strip_prefixes = strip_prefixes or set() + + # If allowed_prefixes is empty, allow all prefixes + self._allow_all = len(self._allowed_prefixes) == 0 + + def _should_process_key(self, key: str) -> bool: + """ + Determine whether this baggage key should be processed + + Args: + key: baggage key + + Returns: + True if the key should be processed, False otherwise + """ + if self._allow_all: + return True + + # Check if key matches any of the allowed prefixes + for prefix in self._allowed_prefixes: + if key.startswith(prefix): + return True + + return False + + def _strip_prefix(self, key: str) -> str: + """ + Strip matching prefix from key + + Args: + key: original baggage key + + Returns: + key with prefix stripped + """ + for prefix in self._strip_prefixes: + if key.startswith(prefix): + return key[len(prefix) :] + return key + + def on_start( + self, span: "Span", parent_context: Optional[Context] = None + ) -> None: + """ + Called when a span starts, adds matching baggage entries to span attributes + + Args: + span: span to add attributes to + parent_context: parent context used to retrieve baggage + """ + baggage = get_all_baggage(parent_context) + + for key, value in baggage.items(): + # Check if this key should be processed + if not self._should_process_key(key): + continue + + # Strip prefix if needed + attribute_key = self._strip_prefix(key) + + # Add to span attributes + # Baggage values are strings, which are valid AttributeValue + span.set_attribute(attribute_key, value) # type: ignore[arg-type] diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/version.py b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/version.py new file mode 100644 index 000000000..5fd301e2e --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.1.0" diff --git a/processor/loongsuite-processor-baggage/test-requirements.txt b/processor/loongsuite-processor-baggage/test-requirements.txt new file mode 100644 index 000000000..2fe69045f --- /dev/null +++ b/processor/loongsuite-processor-baggage/test-requirements.txt @@ -0,0 +1,5 @@ +pytest>=7.0.0 +pytest-cov>=4.0.0 +opentelemetry-api>=1.5.0 +opentelemetry-sdk>=1.5.0 + diff --git a/processor/loongsuite-processor-baggage/tests/__init__.py b/processor/loongsuite-processor-baggage/tests/__init__.py new file mode 100644 index 000000000..f87ce79b7 --- /dev/null +++ b/processor/loongsuite-processor-baggage/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/processor/loongsuite-processor-baggage/tests/test_baggage_processor.py b/processor/loongsuite-processor-baggage/tests/test_baggage_processor.py new file mode 100644 index 000000000..11225a504 --- /dev/null +++ b/processor/loongsuite-processor-baggage/tests/test_baggage_processor.py @@ -0,0 +1,171 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from loongsuite.processor.baggage import LoongSuiteBaggageSpanProcessor + +from opentelemetry.baggage import set_baggage +from opentelemetry.context import attach, detach +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SpanProcessor + + +class LoongSuiteBaggageSpanProcessorTest(unittest.TestCase): + def test_check_the_baggage_processor(self): + self.assertIsInstance(LoongSuiteBaggageSpanProcessor(), SpanProcessor) + + def test_allow_all_prefixes(self): + """Test allowing all prefixes""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor(allowed_prefixes=None) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("any_key", "any_value") + + with tracer.start_as_current_span(name="test", context=ctx) as span: + self.assertEqual(span._attributes["any_key"], "any_value") + + def test_prefix_matching(self): + """Test prefix matching functionality""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic.", "app."} + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("traffic.hello", "world") + ctx = set_baggage("app.user_id", "123", context=ctx) + ctx = set_baggage("other.key", "value", context=ctx) + + with tracer.start_as_current_span(name="test", context=ctx) as span: + # Matching prefixes should be added + self.assertEqual(span._attributes["traffic.hello"], "world") + self.assertEqual(span._attributes["app.user_id"], "123") + # Non-matching prefixes should not be added + self.assertNotIn("other.key", span._attributes) + + def test_prefix_stripping(self): + """Test prefix stripping functionality""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic.", "app."}, + strip_prefixes={"traffic."}, + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("traffic.hello_key", "value") + ctx = set_baggage("app.user_id", "123", context=ctx) + + with tracer.start_as_current_span(name="test", context=ctx) as span: + # traffic. prefix should be stripped + self.assertEqual(span._attributes["hello_key"], "value") + self.assertNotIn("traffic.hello_key", span._attributes) + # app. prefix should not be stripped (not in strip_prefixes) + self.assertEqual(span._attributes["app.user_id"], "123") + + def test_multiple_strip_prefixes(self): + """Test multiple strip prefixes""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes=None, strip_prefixes={"traffic.", "app."} + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("traffic.key1", "value1") + ctx = set_baggage("app.key2", "value2", context=ctx) + ctx = set_baggage("other.key3", "value3", context=ctx) + + with tracer.start_as_current_span(name="test", context=ctx) as span: + self.assertEqual(span._attributes["key1"], "value1") + self.assertEqual(span._attributes["key2"], "value2") + self.assertEqual(span._attributes["other.key3"], "value3") + + def test_nested_spans(self): + """Test nested spans""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic."}, strip_prefixes={"traffic."} + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("traffic.queen", "bee") + + with tracer.start_as_current_span( + name="parent", context=ctx + ) as parent_span: + self.assertEqual(parent_span._attributes["queen"], "bee") + + with tracer.start_as_current_span( + name="child", context=ctx + ) as child_span: + self.assertEqual(child_span._attributes["queen"], "bee") + + def test_context_token(self): + """Test using context token""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic."}, strip_prefixes={"traffic."} + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + token = attach(set_baggage("traffic.bumble", "bee")) + + try: + with tracer.start_as_current_span("parent") as span: + self.assertEqual(span._attributes["bumble"], "bee") + + token2 = attach(set_baggage("traffic.moar", "bee")) + try: + with tracer.start_as_current_span("child") as child_span: + self.assertEqual( + child_span._attributes["bumble"], "bee" + ) + self.assertEqual(child_span._attributes["moar"], "bee") + finally: + detach(token2) + finally: + detach(token) + + def test_empty_prefixes(self): + """Test empty prefix sets""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes=set(), # Empty set, should allow all + strip_prefixes=set(), + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("any_key", "any_value") + + with tracer.start_as_current_span(name="test", context=ctx) as span: + self.assertEqual(span._attributes["any_key"], "any_value") + + +if __name__ == "__main__": + unittest.main() From c0c5dc58493996c26fa6761a5b75a2f8a5b60642 Mon Sep 17 00:00:00 2001 From: cirilla-zmh Date: Tue, 24 Feb 2026 17:39:52 +0800 Subject: [PATCH 2/4] Add LoongSuite Python probe release scripts and pipelines - loongsuite-release.yml: release workflow (PyPI + GitHub Release) - loongsuite_{lint,test,misc}_0.yml: CI for distro and instrumentation - loongsuite-distro: bootstrap, baggage processor auto-config - instrumentation-loongsuite: agentscope, agno, dashscope, dify, langchain, mcp, mem0 - scripts: build, dry-run, bootstrap gen, readme gen - tox-loongsuite.ini: test matrix - docs/loongsuite-release.md Note: processor-baggage CI jobs run only when processor/loongsuite-processor-baggage exists (merge feat/loongsuite-baggage-processor first for full release) Co-authored-by: Cursor Change-Id: Ia0469ec89406d76b3bfd0e4d6381125073b1d9d6 Co-developed-by: Cursor --- .github/workflows/loongsuite-release.yml | 272 ++++ .github/workflows/loongsuite_lint_0.yml | 20 + .github/workflows/loongsuite_misc_0.yml | 53 + .github/workflows/loongsuite_test_0.yml | 100 ++ .gitignore | 3 +- docs/loongsuite-release.md | 620 ++++++++ instrumentation-loongsuite/README.md | 12 +- .../pyproject.toml | 4 +- .../pyproject.toml | 4 +- .../pyproject.toml | 6 +- .../pyproject.toml | 4 +- .../pyproject.toml | 4 +- .../pyproject.toml | 4 +- .../instrumentation/mcp/__init__.py | 9 +- .../instrumentation/mcp/utils.py | 21 +- .../pyproject.toml | 6 +- loongsuite-distro/BOOTSTRAP_REVIEW.md | 272 ++++ loongsuite-distro/README.rst | 77 + loongsuite-distro/pyproject.toml | 69 + loongsuite-distro/src/loongsuite/__init__.py | 15 + .../src/loongsuite/distro/__init__.py | 180 +++ .../src/loongsuite/distro/bootstrap.py | 1299 +++++++++++++++++ .../src/loongsuite/distro/bootstrap_gen.py | 262 ++++ .../src/loongsuite/distro/py.typed | 0 .../src/loongsuite/distro/version.py | 15 + pkg-requirements.txt | 11 + scripts/build_loongsuite_package.py | 675 +++++++++ scripts/dry_run_loongsuite_release.sh | 348 +++++ scripts/generate_loongsuite_bootstrap.py | 421 ++++++ scripts/generate_loongsuite_readme.py | 98 ++ scripts/loongsuite-build-config.json | 11 + tox-loongsuite.ini | 26 + 32 files changed, 4895 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/loongsuite-release.yml create mode 100644 .github/workflows/loongsuite_misc_0.yml create mode 100644 docs/loongsuite-release.md create mode 100644 loongsuite-distro/BOOTSTRAP_REVIEW.md create mode 100644 loongsuite-distro/README.rst create mode 100644 loongsuite-distro/pyproject.toml create mode 100644 loongsuite-distro/src/loongsuite/__init__.py create mode 100644 loongsuite-distro/src/loongsuite/distro/__init__.py create mode 100644 loongsuite-distro/src/loongsuite/distro/bootstrap.py create mode 100644 loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py create mode 100644 loongsuite-distro/src/loongsuite/distro/py.typed create mode 100644 loongsuite-distro/src/loongsuite/distro/version.py create mode 100644 pkg-requirements.txt create mode 100755 scripts/build_loongsuite_package.py create mode 100755 scripts/dry_run_loongsuite_release.sh create mode 100755 scripts/generate_loongsuite_bootstrap.py create mode 100755 scripts/generate_loongsuite_readme.py create mode 100644 scripts/loongsuite-build-config.json diff --git a/.github/workflows/loongsuite-release.yml b/.github/workflows/loongsuite-release.yml new file mode 100644 index 000000000..a7c7b2b27 --- /dev/null +++ b/.github/workflows/loongsuite-release.yml @@ -0,0 +1,272 @@ +# LoongSuite Release Workflow +# +# This workflow handles the complete LoongSuite release process: +# +# 1. PyPI packages (loongsuite-util-genai, loongsuite-distro) - .whl only, NOT loongsuite-python-agent.tar.gz +# 2. GitHub Release (instrumentation-genai/*, instrumentation-loongsuite/* as tar.gz) +# +# Trigger: +# - workflow_dispatch: Manual trigger with version inputs +# - push tags: Automatic trigger on v* tags +# +# PyPI / Test PyPI configuration: +# - Production PyPI: Set PYPI_API_TOKEN secret, or configure OIDC trusted publishing on pypi.org +# - Test PyPI: Set TEST_PYPI_TOKEN secret (pypi-xxx from https://test.pypi.org/manage/account/token/) +# - Use publish_target: testpypi to publish to Test PyPI instead of production +# +# IMPORTANT: Only loongsuite_util_genai-*.whl and loongsuite_distro-*.whl are uploaded to PyPI. +# loongsuite-python-agent-*.tar.gz is for GitHub Release only and must NOT be uploaded to PyPI. +# +name: LoongSuite Release + +run-name: "LoongSuite Release ${{ github.event.inputs.loongsuite_version || github.ref_name }}" + +on: + workflow_dispatch: + inputs: + loongsuite_version: + description: 'LoongSuite version (e.g., 0.1.0) - for loongsuite-* packages' + required: true + upstream_version: + description: 'Upstream OTel version (e.g., 0.60b1) - for opentelemetry-* packages in bootstrap_gen.py' + required: true + release_notes: + description: 'Release notes (optional, uses CHANGELOG-loongsuite.md Unreleased if empty)' + required: false + skip_pypi: + description: 'Skip PyPI publish (for testing)' + type: boolean + default: false + publish_target: + description: 'Publish target: pypi or testpypi' + type: choice + options: + - pypi + - testpypi + default: pypi + push: + tags: + - 'v*' + +permissions: + contents: read + +env: + PYTHON_VERSION: '3.11' + +jobs: + # Build all packages + build: + runs-on: ubuntu-latest + outputs: + loongsuite_version: ${{ steps.version.outputs.loongsuite_version }} + upstream_version: ${{ steps.version.outputs.upstream_version }} + steps: + - uses: actions/checkout@v4 + + - name: Set versions from tag or input + id: version + run: | + if [[ "${{ github.event_name }}" == "push" && "${{ github.ref_type }}" == "tag" ]]; then + # Tag-based release: extract version from tag (v0.1.0 -> 0.1.0) + tag="${GITHUB_REF#refs/tags/}" + loongsuite_version="${tag#v}" + # For tag-based release, upstream_version must be set via environment or default + upstream_version="${UPSTREAM_VERSION:-0.60b1}" + else + # Manual release: use inputs + loongsuite_version="${{ github.event.inputs.loongsuite_version }}" + upstream_version="${{ github.event.inputs.upstream_version }}" + fi + + if [[ -z "$loongsuite_version" ]]; then + echo "ERROR: loongsuite_version is required" + exit 1 + fi + if [[ -z "$upstream_version" ]]; then + echo "ERROR: upstream_version is required" + exit 1 + fi + + echo "loongsuite_version=$loongsuite_version" >> $GITHUB_OUTPUT + echo "upstream_version=$upstream_version" >> $GITHUB_OUTPUT + echo "LOONGSUITE_VERSION=$loongsuite_version" >> $GITHUB_ENV + echo "UPSTREAM_VERSION=$upstream_version" >> $GITHUB_ENV + + echo "LoongSuite version: $loongsuite_version" + echo "Upstream version: $upstream_version" + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r pkg-requirements.txt + + - name: Generate bootstrap_gen.py with versions + run: | + python scripts/generate_loongsuite_bootstrap.py \ + --upstream-version ${{ steps.version.outputs.upstream_version }} \ + --loongsuite-version ${{ steps.version.outputs.loongsuite_version }} + + echo "Generated bootstrap_gen.py:" + head -30 loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py + + - name: Build PyPI packages + run: | + python scripts/build_loongsuite_package.py \ + --build-pypi \ + --version ${{ steps.version.outputs.loongsuite_version }} + + echo "PyPI packages built:" + ls -la dist/*.whl + + - name: Build GitHub Release packages + run: | + python scripts/build_loongsuite_package.py \ + --build-github-release \ + --version ${{ steps.version.outputs.loongsuite_version }} + + echo "GitHub Release tar:" + ls -la dist/*.tar.gz + + - name: Upload PyPI artifacts + uses: actions/upload-artifact@v4 + with: + name: pypi-packages + path: | + dist/loongsuite_util_genai-*.whl + dist/loongsuite_distro-*.whl + + - name: Upload GitHub Release artifact + uses: actions/upload-artifact@v4 + with: + name: github-release + path: dist/loongsuite-python-agent-*.tar.gz + + # Publish to production PyPI + publish-pypi: + needs: build + runs-on: ubuntu-latest + if: | + (github.event_name != 'workflow_dispatch' || !inputs.skip_pypi) && + (github.event_name != 'workflow_dispatch' || github.event.inputs.publish_target != 'testpypi') + environment: + name: pypi + url: https://pypi.org/project/loongsuite-distro/ + permissions: + id-token: write # For OIDC; or use PYPI_API_TOKEN secret + steps: + - name: Download PyPI artifacts + uses: actions/download-artifact@v4 + with: + name: pypi-packages + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true + + # Publish to Test PyPI (for testing before production) + publish-testpypi: + needs: build + runs-on: ubuntu-latest + if: ${{ github.event_name == 'workflow_dispatch' && !inputs.skip_pypi && inputs.publish_target == 'testpypi' }} + steps: + - name: Download PyPI artifacts + uses: actions/download-artifact@v4 + with: + name: pypi-packages + path: dist/ + + - name: Publish to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + username: __token__ + password: ${{ secrets.TEST_PYPI_TOKEN }} + skip-existing: true + + # Create GitHub Release + github-release: + needs: build + runs-on: ubuntu-latest + permissions: + contents: write # Required for creating releases + steps: + - uses: actions/checkout@v4 + + - name: Download GitHub Release artifact + uses: actions/download-artifact@v4 + with: + name: github-release + path: dist/ + + - name: Generate release notes + id: release_notes + run: | + LOONGSUITE_VERSION="${{ needs.build.outputs.loongsuite_version }}" + UPSTREAM_VERSION="${{ needs.build.outputs.upstream_version }}" + + if [[ -n "${{ github.event.inputs.release_notes }}" ]]; then + echo "${{ github.event.inputs.release_notes }}" > /tmp/release-notes.txt + else + # Start with header + cat > /tmp/release-notes.txt << EOF + # LoongSuite Python Agent v${LOONGSUITE_VERSION} + + ## Installation + + \`\`\`bash + pip install loongsuite-distro==${LOONGSUITE_VERSION} + loongsuite-bootstrap -a install --version ${LOONGSUITE_VERSION} + \`\`\` + + ## Package Versions + + - loongsuite-* packages: ${LOONGSUITE_VERSION} + - opentelemetry-* packages: ${UPSTREAM_VERSION} + + --- + + EOF + + # Collect from root CHANGELOG-loongsuite.md + if [[ -f CHANGELOG-loongsuite.md ]]; then + echo "## loongsuite-distro" >> /tmp/release-notes.txt + echo "" >> /tmp/release-notes.txt + sed -n '/^## \[*Unreleased\]*$/,/^## /p' CHANGELOG-loongsuite.md | sed '/^## /d' >> /tmp/release-notes.txt + echo "" >> /tmp/release-notes.txt + fi + + # Collect from instrumentation-loongsuite/*/CHANGELOG.md + for changelog in instrumentation-loongsuite/*/CHANGELOG.md; do + if [[ -f "$changelog" ]]; then + pkg_dir=$(dirname "$changelog") + pkg_name=$(basename "$pkg_dir") + unreleased_content=$(sed -n '/^## \[*Unreleased\]*$/,/^## /p' "$changelog" | sed '/^## /d') + if [[ -n "$unreleased_content" && "$unreleased_content" =~ [^[:space:]] ]]; then + echo "## $pkg_name" >> /tmp/release-notes.txt + echo "" >> /tmp/release-notes.txt + echo "$unreleased_content" >> /tmp/release-notes.txt + echo "" >> /tmp/release-notes.txt + fi + fi + done + fi + + echo "Release notes:" + cat /tmp/release-notes.txt + + - name: Create GitHub release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION="${{ needs.build.outputs.loongsuite_version }}" + gh release create "v${VERSION}" \ + --title "LoongSuite Python Agent v${VERSION}" \ + --notes-file /tmp/release-notes.txt \ + dist/loongsuite-python-agent-${VERSION}.tar.gz diff --git a/.github/workflows/loongsuite_lint_0.yml b/.github/workflows/loongsuite_lint_0.yml index ec19cde2a..cc2e3c931 100644 --- a/.github/workflows/loongsuite_lint_0.yml +++ b/.github/workflows/loongsuite_lint_0.yml @@ -127,3 +127,23 @@ jobs: - name: Run tests run: tox -c tox-loongsuite.ini -e lint-loongsuite-instrumentation-mem0 + lint-loongsuite-processor-baggage: + name: LoongSuite loongsuite-processor-baggage + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e lint-loongsuite-processor-baggage + diff --git a/.github/workflows/loongsuite_misc_0.yml b/.github/workflows/loongsuite_misc_0.yml new file mode 100644 index 000000000..e728e253e --- /dev/null +++ b/.github/workflows/loongsuite_misc_0.yml @@ -0,0 +1,53 @@ +# Do not edit this file. +# This file is generated automatically by executing tox -e generate-workflows + +name: LoongSuite Misc 0 + +on: + push: + branches-ignore: + - 'release/*' + - 'otelbot/*' + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + # Set the SHA to the branch name if the PR has a label 'prepare-release' or 'backport' otherwise, set it to 'main' + # For PRs you can change the inner fallback ('main') + # For pushes you change the outer fallback ('main') + # The logic below is used during releases and depends on having an equivalent branch name in the core repo. + CORE_REPO_SHA: ${{ github.event_name == 'pull_request' && ( + contains(github.event.pull_request.labels.*.name, 'prepare-release') && github.event.pull_request.head.ref || + contains(github.event.pull_request.labels.*.name, 'backport') && github.event.pull_request.base.ref || + 'main' + ) || 'main' }} + CONTRIB_REPO_SHA: main + PIP_EXISTS_ACTION: w + +jobs: + + generate-loongsuite: + name: LoongSuite generate-loongsuite + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e generate-loongsuite + diff --git a/.github/workflows/loongsuite_test_0.yml b/.github/workflows/loongsuite_test_0.yml index 2ef853c67..5011b7737 100644 --- a/.github/workflows/loongsuite_test_0.yml +++ b/.github/workflows/loongsuite_test_0.yml @@ -868,3 +868,103 @@ jobs: - name: Run tests run: tox -c tox-loongsuite.ini -e py313-test-loongsuite-instrumentation-mem0-latest -- -ra + py39-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.9 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py39-test-loongsuite-processor-baggage -- -ra + + py310-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.10 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py310-test-loongsuite-processor-baggage -- -ra + + py311-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.11 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py311-test-loongsuite-processor-baggage -- -ra + + py312-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.12 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py312-test-loongsuite-processor-baggage -- -ra + + py313-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.13 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py313-test-loongsuite-processor-baggage -- -ra + diff --git a/.gitignore b/.gitignore index 841e7f158..47f8ff508 100644 --- a/.gitignore +++ b/.gitignore @@ -67,5 +67,6 @@ pyrightconfig.json # LoongSuite Extension .cursor/ +dist-*/ upload/ -upload_*_test/ \ No newline at end of file +upload_*_test/ diff --git a/docs/loongsuite-release.md b/docs/loongsuite-release.md new file mode 100644 index 000000000..57a0e8775 --- /dev/null +++ b/docs/loongsuite-release.md @@ -0,0 +1,620 @@ +# LoongSuite Python Agent 发布完整指南 + +本文档涵盖 LoongSuite Python Agent 的架构设计、发布流程、用户安装、开发调试及维护指南。 + +## 目录 + +- [1. 项目概述](#1-项目概述) + - [1.1 项目背景](#11-项目背景) + - [1.2 模块介绍](#12-模块介绍) + - [1.3 发布渠道](#13-发布渠道) +- [2. 发布原理](#2-发布原理) + - [2.1 发布架构](#21-发布架构) + - [2.2 核心脚本](#22-核心脚本) + - [2.3 构建流程详解](#23-构建流程详解) +- [3. 终端用户指南](#3-终端用户指南) + - [3.1 快速开始](#31-快速开始) + - [3.2 安装选项](#32-安装选项) + - [3.3 使用探针](#33-使用探针) +- [4. 开发者指南](#4-开发者指南) + - [4.1 环境准备](#41-环境准备) + - [4.2 本地开发](#42-本地开发) + - [4.3 运行测试](#43-运行测试) +- [5. 维护者指南：发布新版本](#5-维护者指南发布新版本) + - [5.1 版本号策略](#51-版本号策略) + - [5.2 本地验证 (Dry Run)](#52-本地验证-dry-run) + - [5.3 正式发布](#53-正式发布) +- [6. 维护者指南：同步上游代码](#6-维护者指南同步上游代码) + - [6.1 同步原理](#61-同步原理) + - [6.2 同步步骤](#62-同步步骤) + - [6.3 冲突处理](#63-冲突处理) +- [7. 故障排查](#7-故障排查) +- [8. 相关文件索引](#8-相关文件索引) + +--- + +## 1. 项目概述 + +### 1.1 项目背景 + +LoongSuite Python Agent 是基于 [OpenTelemetry Python Contrib](https://github.com/open-telemetry/opentelemetry-python-contrib) 的 Fork 项目，主要扩展了对大模型（GenAI）框架的可观测性支持。 + +**为什么需要 Fork？** + +- 上游 `opentelemetry-util-genai` 功能有限，我们需要扩展它 +- 我们新增的 `instrumentation-loongsuite/*` 依赖扩展后的 `util-genai` +- 为避免依赖冲突，我们将扩展后的包重命名为 `loongsuite-*` 前缀发布 + +### 1.2 模块介绍 + +| 模块类型 | 源目录 | 说明 | +|---------|--------|------| +| **util-genai** | `util/opentelemetry-util-genai` | GenAI 工具库，提供通用的 span 处理、token 计算等功能 | +| **distro** | `loongsuite-distro` | 发行版入口，提供 `loongsuite-bootstrap` 和 `loongsuite-instrument` 命令 | +| **GenAI instrumentations** | `instrumentation-genai/*` | 来自上游的 GenAI 插桩，如 OpenAI、VertexAI 等 | +| **LoongSuite instrumentations** | `instrumentation-loongsuite/*` | LoongSuite 自研插桩，如 DashScope、AgentScope、MCP 等 | +| **标准 instrumentations** | `instrumentation/*` | 标准微服务插桩（Flask、Django、Redis 等），由上游发布 | +| **processor** | `processor/loongsuite-processor-baggage` | Baggage 处理器 | + +### 1.3 发布渠道 + +LoongSuite 采用**双轨发布策略**： + +| 发布后包名 | 发布目标 | 来源 | 说明 | +|-----------|----------|------|------| +| `loongsuite-util-genai` | **PyPI** | `util/opentelemetry-util-genai` | 重命名后发布 | +| `loongsuite-distro` | **PyPI** | `loongsuite-distro` | 引导器 | +| `loongsuite-instrumentation-*` | **GitHub Release** | `instrumentation-genai/*` + `instrumentation-loongsuite/*` | 打包为 tar.gz | +| `opentelemetry-instrumentation-*` | **PyPI (上游)** | `instrumentation/*` | 由上游 OpenTelemetry 发布 | + +**依赖关系图：** + +``` +用户环境 +├── loongsuite-distro (PyPI) +│ ├── provides: loongsuite-bootstrap, loongsuite-instrument +│ └── depends: opentelemetry-api, opentelemetry-sdk +├── loongsuite-util-genai (PyPI) +│ └── GenAI 通用工具库 +├── loongsuite-instrumentation-* (GitHub Release) +│ ├── loongsuite-instrumentation-dashscope +│ ├── loongsuite-instrumentation-vertexai (renamed from opentelemetry-*) +│ └── ... (依赖 loongsuite-util-genai) +└── opentelemetry-instrumentation-* (PyPI 上游) + ├── opentelemetry-instrumentation-flask + ├── opentelemetry-instrumentation-redis + └── ... +``` + +--- + +## 2. 发布原理 + +### 2.1 发布架构 + +``` + ┌─────────────────────────────────────┐ + │ Release Trigger │ + │ (Manual dispatch / Git tag push) │ + └───────────────┬─────────────────────┘ + │ + ┌───────────────▼─────────────────────┐ + │ loongsuite-release.yml │ + │ (GitHub Actions Workflow) │ + └───────────────┬─────────────────────┘ + │ + ┌─────────────────────┼─────────────────────┐ + │ │ │ + ┌─────────▼───────────┐ ┌───────▼──────────┐ ┌────────▼─────────┐ + │ generate_loongsuite │ │ build_loongsuite │ │ build_loongsuite │ + │ _bootstrap.py │ │ _package.py │ │ _package.py │ + │ │ │ --build-pypi │ │ --build-github │ + └─────────┬───────────┘ └────────┬─────────┘ └────────┬─────────┘ + │ │ │ + ┌─────────▼────────────┐ ┌───────▼────────┐ ┌─────────▼───────┐ + │ bootstrap_gen.py │ │ dist-pypi/ │ │ dist/ │ + │ (version mapping) │ │ *.whl │ │ *.tar.gz │ + └──────────────────────┘ └───────┬────────┘ └────────┬────────┘ + │ │ + ┌────────▼────────┐ ┌────────▼────────┐ + │ PyPI │ │ GitHub Release │ + └─────────────────┘ └─────────────────┘ +``` + +### 2.2 核心脚本 + +| 脚本 | 作用 | +|------|------| +| `scripts/generate_loongsuite_bootstrap.py` | 生成 `bootstrap_gen.py`，定义包名映射和版本 | +| `scripts/build_loongsuite_package.py` | 构建 wheel 包，处理包名重命名和依赖替换 | +| `scripts/dry_run_loongsuite_release.sh` | 本地验证脚本，模拟完整发布流程 | +| `.github/workflows/loongsuite-release.yml` | GitHub Actions 发布工作流 | + +### 2.3 构建流程详解 + +#### Step 1: 生成 bootstrap_gen.py + +```bash +python scripts/generate_loongsuite_bootstrap.py \ + --upstream-version 0.60b1 \ + --loongsuite-version 0.1.0 +``` + +**处理逻辑：** +- 扫描 `instrumentation/`、`instrumentation-genai/`、`instrumentation-loongsuite/` 目录 +- 对 `instrumentation-genai/opentelemetry-*` 包进行重命名（→ `loongsuite-*`） +- 生成包名到版本的映射表 + +#### Step 2: 构建 PyPI 包 + +```bash +python scripts/build_loongsuite_package.py --build-pypi \ + --version 0.1.0 --util-genai-version 0.1.0 +``` + +**处理逻辑：** +- 构建 `util/opentelemetry-util-genai` → 输出 `loongsuite_util_genai-*.whl` + - 使用 TOML 解析修改 `pyproject.toml` 中的 `name` 字段 +- 构建 `loongsuite-distro` → 输出 `loongsuite_distro-*.whl` + +#### Step 3: 构建 GitHub Release 包 + +```bash +python scripts/build_loongsuite_package.py --build-github-release \ + --version 0.1.0 --util-genai-version 0.1.0 +``` + +**处理逻辑：** +- 遍历 `instrumentation-genai/` 目录： + - **规则 1**：`opentelemetry-*` 前缀的包重命名为 `loongsuite-*` + - **规则 2**：动态检测依赖，将 `opentelemetry-util-genai` 替换为 `loongsuite-util-genai` +- 遍历 `instrumentation-loongsuite/` 目录： + - 仅应用依赖替换规则 +- 遍历 `processor/loongsuite-processor-baggage/` +- 所有 `.whl` 打包为 `loongsuite-python-agent-{version}.tar.gz` + +**规则匹配（无需硬编码包名）：** + +```python +# 重命名规则：instrumentation-genai/ 下的 opentelemetry-* 包 +def should_rename_package(package_dir: Path) -> bool: + return "instrumentation-genai" in str(package_dir) and \ + package_dir.name.startswith("opentelemetry-") + +# 依赖替换规则：检测 pyproject.toml 中是否包含 opentelemetry-util-genai +def depends_on_util_genai(pyproject_path: Path) -> bool: + content = pyproject_path.read_text() + return "opentelemetry-util-genai" in content +``` + +--- + +## 3. 终端用户指南 + +### 3.1 快速开始 + +```bash +# 1. 安装 loongsuite-distro (从 PyPI) +pip install loongsuite-distro + +# 2. 安装所有 instrumentations +loongsuite-bootstrap -a install --version 0.1.0 + +# 3. 运行你的应用（自动注入探针） +loongsuite-instrument python app.py +``` + +### 3.2 安装选项 + +#### 完整安装 + +```bash +# 安装所有可用的 instrumentations +loongsuite-bootstrap -a install --version 0.1.0 +``` + +#### 按需安装 (推荐) + +```bash +# 只安装当前环境中已安装库对应的 instrumentations +loongsuite-bootstrap -a install --version 0.1.0 --auto-detect +``` + +#### 白名单安装 + +```bash +# 创建白名单 +cat > whitelist.txt << EOF +loongsuite-instrumentation-dashscope +loongsuite-instrumentation-langchain +opentelemetry-instrumentation-flask +opentelemetry-instrumentation-redis +EOF + +# 只安装白名单中的包 +loongsuite-bootstrap -a install --version 0.1.0 --whitelist whitelist.txt +``` + +### 3.3 使用探针 + +#### 方式 1: 命令行自动注入 + +```bash +# 自动加载所有已安装的 instrumentations +loongsuite-instrument python app.py + +# 指定 exporter +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \ +loongsuite-instrument python app.py +``` + +#### 方式 2: 代码中手动集成 + +```python +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + +# 配置 TracerProvider +provider = TracerProvider() +processor = BatchSpanProcessor(OTLPSpanExporter()) +provider.add_span_processor(processor) +trace.set_tracer_provider(provider) + +# 手动启用特定 instrumentation +from opentelemetry.instrumentation.dashscope import DashScopeInstrumentor +DashScopeInstrumentor().instrument() +``` + +### 安装流程说明 + +`loongsuite-bootstrap -a install` 执行两阶段安装： + +``` +Phase 1: 从 GitHub Release tar.gz 安装 loongsuite-* 包 + └── pip install --find-links loongsuite-instrumentation-* + +Phase 2: 从 PyPI 安装 opentelemetry-* 包 + └── pip install opentelemetry-instrumentation-flask==0.60b1 ... +``` + +--- + +## 4. 开发者指南 + +### 4.1 环境准备 + +开发环境**不需要**进行包名重命名，因为所有代码都在本地，可以直接使用 `opentelemetry-util-genai` 等原始包名。 + +```bash +# 克隆仓库 +git clone https://github.com/alibaba/loongsuite-python-agent.git +cd loongsuite-python-agent + +# 创建虚拟环境 +python -m venv .venv +source .venv/bin/activate + +# 安装开发依赖 +pip install -r dev-requirements.txt +``` + +### 4.2 本地开发 + +#### 安装核心包（editable 模式） + +```bash +# 安装 opentelemetry 核心包（从上游） +pip install opentelemetry-api opentelemetry-sdk opentelemetry-semantic-conventions + +# 安装本地 util-genai（使用原始包名，无需重命名） +pip install -e ./util/opentelemetry-util-genai + +# 安装你要开发的 instrumentation +pip install -e ./instrumentation-loongsuite/loongsuite-instrumentation-dashscope + +# 安装 loongsuite-distro（用于测试 bootstrap） +pip install -e ./loongsuite-distro +``` + +#### 开发新的 instrumentation + +```bash +# 复制模板 +cp -r instrumentation-loongsuite/_template \ + instrumentation-loongsuite/loongsuite-instrumentation-mylib + +# 修改 pyproject.toml 中的包名、依赖等 +# 实现 __init__.py 中的 Instrumentor 类 +``` + +### 4.3 运行测试 + +#### 使用 tox（推荐） + +```bash +# 激活 conda 环境（如果使用 conda） +conda activate loongsuite + +# 运行特定模块的测试 +tox -c tox-loongsuite.ini -e py312-test-loongsuite-instrumentation-dashscope-latest + +# 运行 lint +tox -c tox-loongsuite.ini -e lint-loongsuite-instrumentation-dashscope +``` + +#### 直接运行 pytest + +```bash +# 安装测试依赖 +pip install pytest pytest-cov + +# 安装被测模块 +pip install -e ./instrumentation-loongsuite/loongsuite-instrumentation-dashscope +pip install -r ./instrumentation-loongsuite/loongsuite-instrumentation-dashscope/tests/requirements.latest.txt + +# 运行测试 +pytest instrumentation-loongsuite/loongsuite-instrumentation-dashscope/tests/ +``` + +#### 测试环境配置参考 + +参考 `tox-loongsuite.ini` 中的配置，每个模块可以有两套测试依赖： + +- `requirements.oldest.txt`: 最低支持版本的依赖 +- `requirements.latest.txt`: 最新版本的依赖 + +--- + +## 5. 维护者指南：发布新版本 + +### 5.1 版本号策略 + +发布时需要指定两个版本号： + +| 参数 | 说明 | 示例 | +|------|------|------| +| `--loongsuite-version` | LoongSuite 包版本 | `0.1.0`, `0.2.0b0` | +| `--upstream-version` | 上游 OpenTelemetry 包版本 | `0.60b1`, `0.61b0` | + +**应用规则：** +- `loongsuite-version` → `loongsuite-util-genai`, `loongsuite-distro`, `loongsuite-instrumentation-*` +- `upstream-version` → `bootstrap_gen.py` 中的 `opentelemetry-instrumentation-*` 版本 + +### 5.2 本地验证 (Dry Run) + +```bash +# 完整验证 +./scripts/dry_run_loongsuite_release.sh \ + --loongsuite-version 0.1.0 \ + --upstream-version 0.60b1 + +# 快速验证（跳过安装测试） +./scripts/dry_run_loongsuite_release.sh \ + -l 0.1.0 -u 0.60b1 --skip-install +``` + +**验证步骤：** + +| 步骤 | 说明 | 产物 | +|------|------|------| +| 1 | 安装构建依赖 | - | +| 2 | 生成 bootstrap_gen.py | `loongsuite-distro/src/.../bootstrap_gen.py` | +| 3 | 构建 PyPI 包 | `dist-pypi/*.whl` | +| 4 | 构建 GitHub Release 包 | `dist/*.tar.gz` | +| 5 | 验证 tar 内容 | - | +| 6 | 生成 release notes | `dist/release-notes-dryrun.txt` | +| 7 | 安装验证 | 临时 venv 中测试 | + +**验证要点：** + +- ✅ `loongsuite-util-genai` 在 `dist-pypi/` 中（发布到 PyPI） +- ✅ `loongsuite-util-genai` 不在 `tar.gz` 中 +- ✅ `loongsuite-instrumentation-*` 在 `tar.gz` 中 +- ✅ `opentelemetry-util-genai` 不在任何产物中（避免冲突） +- ✅ 安装后依赖关系正确 + +### 5.3 正式发布 + +#### 方式 1: 手动触发 (推荐) + +1. 进入 GitHub 仓库 → **Actions** → **LoongSuite Release** +2. 点击 **Run workflow** +3. 填写参数： + - `loongsuite_version`: `0.1.0` + - `upstream_version`: `0.60b1` + - `release_notes`: 可选 + - `skip_pypi`: 测试时可勾选 +4. 执行 + +#### 方式 2: Tag 触发 + +```bash +git tag v0.1.0 +git push origin v0.1.0 +``` + +#### PyPI / Test PyPI 发布配置 + +**发布到生产 PyPI（二选一）：** + +1. **API Token**：在 GitHub 仓库 Settings → Secrets → Actions 中添加： + - `PYPI_API_TOKEN`：从 [pypi.org/manage/account/token/](https://pypi.org/manage/account/token/) 创建 + +2. **OIDC Trusted Publishing**（推荐）： + - PyPI 项目设置 → Publishing → Add a new pending publisher + - Owner: `alibaba`，Repository: `loongsuite-python-agent` + - Workflow: `loongsuite-release.yml`，Environment: `pypi` + - 在 GitHub 仓库中创建 Environment `pypi`（Settings → Environments） + +**发布到 Test PyPI（测试用）：** + +1. 在 [test.pypi.org/manage/account/token/](https://test.pypi.org/manage/account/token/) 创建 API Token +2. 在 GitHub Secrets 中添加：`TEST_PYPI_TOKEN`（值为 `pypi-xxx`） +3. 手动触发 workflow 时，将 `publish_target` 选为 **testpypi** + +**重要说明：** + +- 只有 `loongsuite_util_genai-*.whl` 和 `loongsuite_distro-*.whl` 会上传到 PyPI +- `loongsuite-python-agent-*.tar.gz` 仅用于 GitHub Release，**禁止**上传到 PyPI +- 若手动使用 `twine upload dist/*`，请先 `rm dist/loongsuite-python-agent-*.tar.gz`，否则会报错 `InvalidDistribution: Too many top-level members in sdist archive` + +#### 发布检查清单 + +- [ ] 本地 dry run 通过 +- [ ] `CHANGELOG-loongsuite.md` 已更新 +- [ ] 版本号格式正确（不带 `v` 前缀） +- [ ] `upstream_version` 与当前上游稳定版本匹配 +- [ ] PyPI 权限已配置 + +--- + +## 6. 维护者指南：同步上游代码 + +### 6.1 同步原理 + +本项目 Fork 自 [opentelemetry-python-contrib](https://github.com/open-telemetry/opentelemetry-python-contrib)，需要定期同步上游的更新。 + +**同步策略：** + +``` +upstream/main ──────────────────────────────────────► 上游主分支 + │ + │ git fetch upstream + │ git merge upstream/main + ▼ +origin/main ───────────────────────────────────────► 我们的主分支 + │ + │ feature branches + ▼ +origin/feature/* ──────────────────────────────────► 功能分支 +``` + +**需要注意的目录：** + +| 目录 | 同步策略 | +|------|----------| +| `instrumentation/` | 完全同步上游 | +| `instrumentation-genai/` | 完全同步上游 | +| `util/opentelemetry-util-genai/` | 同步上游，**保留我们的扩展** | +| `instrumentation-loongsuite/` | **我们独有**，不受上游影响 | +| `loongsuite-distro/` | **我们独有**，不受上游影响 | +| `scripts/generate_loongsuite_*.py` | **我们独有** | +| `scripts/build_loongsuite_*.py` | **我们独有** | + +### 6.2 同步步骤 + +```bash +# 1. 添加上游远程（如果未添加） +git remote add upstream https://github.com/open-telemetry/opentelemetry-python-contrib.git + +# 2. 获取上游更新 +git fetch upstream + +# 3. 切换到主分支 +git checkout main + +# 4. 合并上游更新 +git merge upstream/main + +# 5. 解决冲突（如有） +# ... + +# 6. 推送到我们的仓库 +git push origin main +``` + +### 6.3 冲突处理 + +**常见冲突场景：** + +1. **`util/opentelemetry-util-genai/` 冲突** + - 我们对这个模块有扩展 + - 需要手动合并，保留我们的扩展代码 + +2. **`scripts/` 目录冲突** + - 上游的 `scripts/generate_instrumentation_bootstrap.py` 等可能更新 + - 我们的 `scripts/generate_loongsuite_*.py` 依赖它们 + - 需要检查 API 兼容性 + +3. **`pyproject.toml` 冲突** + - 上游可能更新依赖版本 + - 需要验证兼容性 + +**冲突解决后验证：** + +```bash +# 运行测试确保兼容性 +tox -c tox-loongsuite.ini -e py312-test-loongsuite-instrumentation-dashscope-latest + +# 运行 dry run 确保发布流程正常 +./scripts/dry_run_loongsuite_release.sh -l 0.1.0 -u 0.60b1 --skip-install +``` + +--- + +## 7. 故障排查 + +### 构建问题 + +**问题**: `hatch version` 失败 +```bash +# 解决: 安装 hatch +pip install hatch +``` + +**问题**: 构建时找不到依赖 +```bash +# 解决: 安装构建依赖 +pip install -r pkg-requirements.txt +``` + +**问题**: tomlkit 相关错误 +```bash +# 解决: 安装 tomlkit +pip install tomlkit +``` + +### 安装问题 + +**问题**: `loongsuite-util-genai` 找不到 +```bash +# 原因: PyPI 包未正确构建 +# 解决: 检查 dry run step 3 的输出，确认包名为 loongsuite_util_genai +``` + +**问题**: `opentelemetry-util-genai` 和 `loongsuite-util-genai` 冲突 +```bash +# 解决: 卸载旧包 +pip uninstall opentelemetry-util-genai +pip install loongsuite-util-genai +``` + +### 发布问题 + +**问题**: PyPI 发布 403 Forbidden +```bash +# 解决: 检查 OIDC trusted publishing 配置或 API token +``` + +**问题**: 版本号已存在 +```bash +# 解决: PyPI 不允许覆盖版本，使用新版本号 +``` + +--- + +## 8. 相关文件索引 + +| 文件 | 说明 | +|------|------| +| `scripts/build_loongsuite_package.py` | 构建脚本，处理包名重命名和依赖替换 | +| `scripts/generate_loongsuite_bootstrap.py` | 生成 bootstrap_gen.py | +| `scripts/dry_run_loongsuite_release.sh` | 本地验证脚本 | +| `.github/workflows/loongsuite-release.yml` | GitHub Actions 发布工作流 | +| `loongsuite-distro/src/loongsuite/distro/bootstrap.py` | Bootstrap 安装逻辑 | +| `loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py` | 生成的包名映射配置 | +| `tox-loongsuite.ini` | 测试配置 | +| `pkg-requirements.txt` | 构建依赖 | +| `CHANGELOG-loongsuite.md` | 变更日志 | diff --git a/instrumentation-loongsuite/README.md b/instrumentation-loongsuite/README.md index 8327b2d99..f05e47fe8 100644 --- a/instrumentation-loongsuite/README.md +++ b/instrumentation-loongsuite/README.md @@ -1,10 +1,12 @@ | Instrumentation | Supported Packages | Metrics support | Semconv status | | --------------- | ------------------ | --------------- | -------------- | -| [loongsuite-instrumentation-agentscope](./loongsuite-instrumentation-agentscope) | agentscope >= 0.1.5.dev0 | No | development -| [loongsuite-instrumentation-agno](./loongsuite-instrumentation-agno) | agno >= 1.5.0 | No | development +| [loongsuite-instrumentation-agentscope](./loongsuite-instrumentation-agentscope) | agentscope >= 1.0.0 | No | development +| [loongsuite-instrumentation-agno](./loongsuite-instrumentation-agno) | agno | No | development +| [loongsuite-instrumentation-claude-agent-sdk](./loongsuite-instrumentation-claude-agent-sdk) | claude-agent-sdk >= 0.1.0 | No | development +| [loongsuite-instrumentation-dashscope](./loongsuite-instrumentation-dashscope) | dashscope >= 1.0.0 | No | development | [loongsuite-instrumentation-dify](./loongsuite-instrumentation-dify) | dify | No | development -| [loongsuite-instrumentation-google-adk](./loongsuite-instrumentation-google-adk) | google-adk >= 0.1.0 | Yes | experimental -| [loongsuite-instrumentation-langchain](./loongsuite-instrumentation-langchain) | langchain_core >= 0.1.0 | Yes | development -| [loongsuite-instrumentation-mcp](./loongsuite-instrumentation-mcp) | mcp>=1.3.0 | Yes | development +| [loongsuite-instrumentation-google-adk](./loongsuite-instrumentation-google-adk) | google-adk >= 0.1.0 | No | development +| [loongsuite-instrumentation-langchain](./loongsuite-instrumentation-langchain) | langchain_core >= 0.1.0 | No | development +| [loongsuite-instrumentation-mcp](./loongsuite-instrumentation-mcp) | mcp >= 1.3.0, <= 1.25.0 | No | development | [loongsuite-instrumentation-mem0](./loongsuite-instrumentation-mem0) | mem0ai >= 1.0.0 | No | development \ No newline at end of file diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml index 0a63ac5ab..c52a1ebb4 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml @@ -25,8 +25,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", "opentelemetry-util-genai", "wrapt", ] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agno/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-agno/pyproject.toml index c3d8fc16e..16bcf2fb7 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agno/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agno/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", ] [project.optional-dependencies] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-dashscope/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-dashscope/pyproject.toml index 472e829ae..87efc60e0 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-dashscope/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-dashscope/pyproject.toml @@ -26,9 +26,9 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", - "opentelemetry-util-genai ~= 0.2b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", + "opentelemetry-util-genai > 0.2b0", ] [project.optional-dependencies] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-dify/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-dify/pyproject.toml index b498abac2..5d492609d 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-dify/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-dify/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", ] [project.optional-dependencies] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-langchain/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-langchain/pyproject.toml index ea1dafe2d..2853b613f 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-langchain/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-langchain/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", ] [project.optional-dependencies] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/pyproject.toml index e1e548b4d..e106d2074 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "OpenTelemetry MCP (Model Context Protocol) instrumentation" readme = "README.md" license = "Apache-2.0" -requires-python = ">=3.10, <=3.13" +requires-python = ">=3.10" authors = [ { name = "LoongSuite Python Agent Authors"}, ] @@ -31,7 +31,7 @@ dependencies = [ [project.optional-dependencies] instruments = [ - "mcp >= 1.3.0, <= 1.13.1", + "mcp >= 1.3.0, <= 1.25.0", ] test = [ "opentelemetry-sdk", diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/__init__.py index 0f589b6ba..2213faff5 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/__init__.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/__init__.py @@ -19,6 +19,7 @@ ) from opentelemetry.instrumentation.mcp.utils import ( _get_logger, + _get_streamable_http_client_name, _is_version_supported, _is_ws_installed, ) @@ -53,6 +54,8 @@ for method_name, rpc_name in RPC_NAME_MAPPING.items() ] +_streamable_http_client_name = _get_streamable_http_client_name() + class MCPInstrumentor(BaseInstrumentor): """ @@ -99,7 +102,7 @@ def _instrument(self, **kwargs: Any) -> None: ) wrap_function_wrapper( module="mcp.client.streamable_http", - name="streamablehttp_client", + name=_streamable_http_client_name, wrapper=streamable_http_client_wrapper(), ) wrap_function_wrapper( @@ -140,10 +143,10 @@ def _uninstrument(self, **kwargs: Any) -> None: try: import mcp.client.streamable_http # noqa: PLC0415 - unwrap(mcp.client.streamable_http, "streamablehttp_client") + unwrap(mcp.client.streamable_http, _streamable_http_client_name) except Exception: logger.warning( - "Fail to uninstrument streamablehttp_client", exc_info=True + "Fail to uninstrument streamable_http_client", exc_info=True ) try: diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/utils.py b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/utils.py index 42655cdc3..23c33e243 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/utils.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/utils.py @@ -18,10 +18,17 @@ _has_mcp_types = False MIN_SUPPORTED_VERSION = (1, 3, 0) -MAX_SUPPORTED_VERSION = (1, 13, 1) +MAX_SUPPORTED_VERSION = (1, 25, 0) MCP_PACKAGE_NAME = "mcp" DEFAULT_MAX_ATTRIBUTE_LENGTH = 1024 * 1024 +# Version thresholds for API changes +# v1.24.0: streamable_http_client was added (streamablehttp_client deprecated) +STREAMABLE_HTTP_CLIENT_NEW_NAME_VERSION = (1, 24, 0) +# Streamable HTTP client function names +STREAMABLE_HTTP_CLIENT_NEW_NAME = "streamable_http_client" +STREAMABLE_HTTP_CLIENT_OLD_NAME = "streamablehttp_client" + _max_attributes_length = None @@ -77,6 +84,18 @@ def _is_version_supported() -> bool: ) +def _get_streamable_http_client_name() -> str: + """ + Get the correct streamable HTTP client function name based on MCP version. + - v1.24.0+: uses `streamable_http_client` (new name) + - v1.3.0 - v1.23.x: uses `streamablehttp_client` (old name) + """ + current_version = _get_mcp_version() + if current_version >= STREAMABLE_HTTP_CLIENT_NEW_NAME_VERSION: + return STREAMABLE_HTTP_CLIENT_NEW_NAME + return STREAMABLE_HTTP_CLIENT_OLD_NAME + + def _is_capture_content_enabled() -> bool: capture_content = environ.get( MCPEnvironmentVariables.CAPTURE_INPUT_ENABLED, "true" diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-mem0/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-mem0/pyproject.toml index 0e4a685c7..ff3c03bbe 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-mem0/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-mem0/pyproject.toml @@ -26,9 +26,9 @@ classifiers = [ dependencies = [ "wrapt >=1.17.3", "opentelemetry-api ~=1.37", - "opentelemetry-instrumentation ~=0.58b0", - "opentelemetry-semantic-conventions ~=0.58b0", - "opentelemetry-util-genai ~= 0.2b0", + "opentelemetry-instrumentation >=0.58b0", + "opentelemetry-semantic-conventions >=0.58b0", + "opentelemetry-util-genai >= 0.2b0", ] [project.optional-dependencies] diff --git a/loongsuite-distro/BOOTSTRAP_REVIEW.md b/loongsuite-distro/BOOTSTRAP_REVIEW.md new file mode 100644 index 000000000..2ebff4fb6 --- /dev/null +++ b/loongsuite-distro/BOOTSTRAP_REVIEW.md @@ -0,0 +1,272 @@ +# bootstrap.py 流程梳理与优化建议 + +## 一、基本流程梳理 + +### 1. 安装流程 (`install_from_tar`) + +``` +main() + └─> install_from_tar() + ├─> resolve_tar_path() # 解析 tar 路径，可能需要下载 + ├─> extract_tar() # 解压 tar 文件，获取所有 .whl 文件 + ├─> filter_packages() # 过滤包（核心逻辑） + │ ├─> get_package_name_from_whl() # 从 whl 文件名提取包名 + │ ├─> _is_instrumentation_in_bootstrap_gen() # 检查是否为 instrumentation + │ ├─> check_python_version_compatibility() # 检查 Python 版本兼容性 + │ ├─> check_dependency_compatibility() # 检查依赖版本兼容性 + │ └─> get_target_libraries_from_bootstrap_gen() + _is_library_installed() # 自动检测 + └─> install_packages() # 使用 pip 安装 +``` + +### 2. 卸载流程 (`uninstall_loongsuite_packages`) + +``` +main() + └─> uninstall_loongsuite_packages() + ├─> get_installed_loongsuite_packages() # 获取已安装的包列表 + └─> uninstall_packages() # 使用 pip 卸载 +``` + +### 3. 核心辅助函数 + +- **包名处理**： + - `get_package_name_from_whl()`: 从 whl 文件名提取包名 + - `get_installed_package_version()`: 获取已安装包的版本（处理下划线/连字符变体） + +- **元数据提取**： + - `get_metadata_from_whl()`: 从 whl 文件提取 METADATA + - `get_python_requirement_from_whl()`: 提取 Python 版本要求 + +- **兼容性检查**： + - `check_python_version_compatibility()`: 检查 Python 版本 + - `check_dependency_compatibility()`: 检查依赖版本 + +- **bootstrap_gen 查询**： + - `_is_instrumentation_in_bootstrap_gen()`: 检查是否为 instrumentation + - `get_target_libraries_from_bootstrap_gen()`: 获取目标库列表 + +- **库检测**： + - `_is_library_installed()`: 检查库是否已安装 + +## 二、发现的问题和优化建议 + +### 1. 🔴 包名规范化逻辑重复 + +**问题**： +- `get_installed_package_version()` 中三次尝试（原始名、下划线→连字符、连字符→下划线） +- `_is_library_installed()` 中也有类似逻辑 +- 多个地方都有 `normalized_name = package_name.replace("_", "-")` 的重复 + +**建议**： +```python +def normalize_package_name(package_name: str) -> str: + """统一规范化包名：将下划线转换为连字符""" + return package_name.replace("_", "-") + +def get_package_name_variants(package_name: str) -> List[str]: + """获取包名的所有可能变体（用于查找）""" + normalized = normalize_package_name(package_name) + variants = [package_name] + if normalized != package_name: + variants.append(normalized) + # 如果需要，也可以添加反向变体 + return variants +``` + +### 2. 🔴 从 requirement 字符串提取包名的逻辑重复 + +**问题**： +在 `_is_instrumentation_in_bootstrap_gen()` 和 `get_target_libraries_from_bootstrap_gen()` 中都有： +```python +default_pkg_name = ( + default_instr.split("==")[0] + .split(">=")[0] + .split("<=")[0] + .split("~=")[0] + .split("!=")[0] + .strip() +) +``` + +**建议**： +```python +def extract_package_name_from_requirement(req_str: str) -> str: + """从 requirement 字符串中提取包名""" + try: + return Requirement(req_str).name + except Exception: + # Fallback: 手动解析 + for op in ["==", ">=", "<=", "~=", "!=", ">", "<"]: + if op in req_str: + return req_str.split(op)[0].strip() + return req_str.strip() +``` + +### 3. 🟡 get_installed_package_version 中的重复代码 + +**问题**： +三个几乎相同的 try-except 块，只是包名不同。 + +**建议**： +```python +def get_installed_package_version(package_name: str) -> Optional[str]: + """获取已安装包的版本""" + variants = get_package_name_variants(package_name) + + for variant in variants: + version = _try_get_version(variant) + if version: + return version + return None + +def _try_get_version(package_name: str) -> Optional[str]: + """尝试获取单个包名变体的版本""" + cmd = [sys.executable, "-m", "pip", "show", package_name] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=True, timeout=5 + ) + for line in result.stdout.splitlines(): + if line.startswith("Version:"): + return line.split(":", 1)[1].strip() + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + pass + return None +``` + +### 4. 🟡 filter_packages 函数过长 + +**问题**： +`filter_packages()` 函数有 130+ 行，包含太多逻辑，可读性差。 + +**建议**： +拆分为多个小函数： +```python +def filter_packages(...): + """主函数，协调各个过滤步骤""" + base_packages = [] + instrumentation_packages = [] + + for whl_file in whl_files: + package_name = get_package_name_from_whl(whl_file) + + if _should_skip_package(package_name, whl_file, blacklist, whitelist, + skip_version_check, auto_detect): + continue + + if package_name in BASE_DEPENDENCIES: + base_packages.append(whl_file) + else: + if _should_install_instrumentation(package_name, whl_file, auto_detect): + instrumentation_packages.append(whl_file) + + return base_packages, instrumentation_packages + +def _should_skip_package(...) -> bool: + """检查是否应该跳过该包""" + # 黑名单/白名单检查 + # Python 版本检查 + # 依赖版本检查 + pass + +def _should_install_instrumentation(...) -> bool: + """检查是否应该安装该 instrumentation""" + # auto-detect 逻辑 + pass +``` + +### 5. 🟡 包名匹配逻辑重复 + +**问题**： +多处都有 `normalized_name == package_name or default_pkg_name == package_name` 这样的匹配。 + +**建议**： +```python +def package_names_match(name1: str, name2: str) -> bool: + """检查两个包名是否匹配（考虑规范化）""" + normalized1 = normalize_package_name(name1) + normalized2 = normalize_package_name(name2) + return (normalized1 == normalized2 or + name1 == name2 or + normalized1 == name2 or + name1 == normalized2) +``` + +### 6. 🟢 常量提取 + +**问题**： +`EXCLUDED_PACKAGES` 在函数内部定义，应该移到模块级别。 + +**建议**： +```python +# 在模块级别定义 +UNINSTALL_EXCLUDED_PACKAGES = { + "loongsuite-distro", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation", +} +``` + +### 7. 🟢 错误处理改进 + +**问题**： +多处使用 `except Exception: pass`，可能隐藏重要错误。 + +**建议**： +更具体地捕获异常，至少记录警告： +```python +except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.debug(f"Failed to get version for {package_name}: {e}") + return None +except Exception as e: + logger.warning(f"Unexpected error getting version for {package_name}: {e}") + return None +``` + +### 8. 🟢 使用 packaging 库解析 requirement + +**问题**： +手动解析 requirement 字符串（split("==")[0]...）不够健壮。 + +**建议**： +统一使用 `packaging.requirements.Requirement` 解析（已经在用，但有些地方还在手动解析）。 + +### 9. 🟡 模块化建议 + +**建议**将代码拆分为多个模块**： + +``` +loongsuite/distro/ + ├── bootstrap.py # 主入口和 CLI + ├── package_utils.py # 包名处理、版本获取等工具函数 + ├── metadata.py # whl 元数据提取 + ├── compatibility.py # 兼容性检查 + └── bootstrap_gen.py # bootstrap_gen 查询（已存在） +``` + +## 三、优先级建议 + +### 高优先级（立即优化） +1. ✅ 提取包名规范化函数（减少重复，提高一致性） +2. ✅ 提取 requirement 解析函数（多处使用，容易出错） +3. ✅ 简化 `get_installed_package_version()`（消除重复代码） + +### 中优先级（后续优化） +4. ⚠️ 拆分 `filter_packages()` 函数（提高可读性） +5. ⚠️ 提取包名匹配函数（统一匹配逻辑） +6. ⚠️ 改进错误处理（更好的调试体验） + +### 低优先级（可选） +7. 💡 模块化拆分（如果文件继续增长） +8. 💡 使用更专业的 metadata 解析库（如果遇到解析问题） + +## 四、总结 + +当前代码功能完整，但存在以下主要问题： +1. **代码重复**：包名规范化、requirement 解析等逻辑在多处重复 +2. **函数过长**：`filter_packages()` 函数包含太多逻辑 +3. **错误处理**：过于宽泛的异常捕获可能隐藏问题 + +建议优先解决代码重复问题，这将提高代码的可维护性和一致性。 + diff --git a/loongsuite-distro/README.rst b/loongsuite-distro/README.rst new file mode 100644 index 000000000..142d2d6fb --- /dev/null +++ b/loongsuite-distro/README.rst @@ -0,0 +1,77 @@ +LoongSuite Distro +================= + +LoongSuite Python Agent's Distro package, providing LoongSuite-specific configuration and tools. + +Installation +------------ + +:: + + pip install loongsuite-distro + +Optional dependencies: + +:: + + # Install with baggage processor support + pip install loongsuite-distro[baggage] + + # Install with OTLP exporter support + pip install loongsuite-distro[otlp] + + # Install with both + pip install loongsuite-distro[baggage,otlp] + +Features +-------- + +1. **LoongSuite Distro**: Provides LoongSuite-specific OpenTelemetry configuration +2. **LoongSuite Bootstrap**: Install all LoongSuite components from tar package +3. **Baggage Processor**: Optional baggage span processor with prefix matching and stripping support + +Usage +----- + +### Configure LoongSuite Distro + +Specify using LoongSuite Distro via environment variable:: + + export OTEL_PYTHON_DISTRO=loongsuite + +### Use LoongSuite Bootstrap + +Install all components from tar package:: + + loongsuite-bootstrap -t loongsuite-python-agent-1.0.0.tar.gz + +Install from GitHub Releases:: + + loongsuite-bootstrap -v 1.0.0 + +Install latest version:: + + loongsuite-bootstrap --latest + +### Configure Baggage Processor + +The baggage processor is automatically loaded if configured via environment variables. +First, install the optional dependency:: + + pip install loongsuite-distro[baggage] + +Then configure via environment variables:: + + export LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES="traffic.,app." + export LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES="traffic." + +The processor will only be loaded if ``LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES`` is set. + +For more usage, please refer to `LOONGSUITE_BOOTSTRAP_README.md`. + +References +---------- + +* `LoongSuite Python Agent `_ + + diff --git a/loongsuite-distro/pyproject.toml b/loongsuite-distro/pyproject.toml new file mode 100644 index 000000000..f7cba84f8 --- /dev/null +++ b/loongsuite-distro/pyproject.toml @@ -0,0 +1,69 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "loongsuite-distro" +dynamic = ["version"] +description = "LoongSuite Python Agent Distro" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.9" +authors = [ + { name = "LoongSuite Python Agent Authors", email = "qp467389@alibaba-inc.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Typing :: Typed", +] +dependencies = [ + "opentelemetry-api ~= 1.12", + "opentelemetry-sdk ~= 1.13", + "opentelemetry-instrumentation >= 0.58b0", + "packaging", +] + +[project.optional-dependencies] +otlp = [ + "opentelemetry-exporter-otlp ~= 1.40", +] +baggage = [ + "loongsuite-processor-baggage", +] + +[project.entry-points.opentelemetry_configurator] +loongsuite = "loongsuite.distro:LoongSuiteConfigurator" + +[project.entry-points.opentelemetry_distro] +loongsuite = "loongsuite.distro:LoongSuiteDistro" + +[project.scripts] +loongsuite-bootstrap = "loongsuite.distro.bootstrap:main" +loongsuite-instrument = "opentelemetry.instrumentation.auto_instrumentation:run" + +[project.urls] +Homepage = "https://github.com/alibaba/loongsuite-python-agent" +Repository = "https://github.com/alibaba/loongsuite-python-agent" + +[tool.hatch.version] +path = "src/loongsuite/distro/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/loongsuite"] + + diff --git a/loongsuite-distro/src/loongsuite/__init__.py b/loongsuite-distro/src/loongsuite/__init__.py new file mode 100644 index 000000000..476fb73aa --- /dev/null +++ b/loongsuite-distro/src/loongsuite/__init__.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/loongsuite-distro/src/loongsuite/distro/__init__.py b/loongsuite-distro/src/loongsuite/distro/__init__.py new file mode 100644 index 000000000..375f38909 --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/__init__.py @@ -0,0 +1,180 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from typing import TYPE_CHECKING, Any, Optional, Set, cast + +from opentelemetry import trace +from opentelemetry.environment_variables import ( + OTEL_LOGS_EXPORTER, + OTEL_METRICS_EXPORTER, + OTEL_TRACES_EXPORTER, +) +from opentelemetry.instrumentation.distro import BaseDistro +from opentelemetry.sdk._configuration import _OTelSDKConfigurator +from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_PROTOCOL +from opentelemetry.sdk.trace import TracerProvider + +if TYPE_CHECKING: + from opentelemetry.sdk.trace import SpanProcessor + +logger = logging.getLogger(__name__) + +# Environment variable names for baggage processor configuration +_LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES = ( + "LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES" +) +_LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES = ( + "LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES" +) + + +class LoongSuiteConfigurator(_OTelSDKConfigurator): + """ + LoongSuite configurator, inherits from OpenTelemetry SDK configurator + + Automatically adds LoongSuiteBaggageSpanProcessor if configured via environment variables. + Only loads the processor if LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES is set. + """ + + def _configure(self, **kwargs: Any) -> None: + # Call parent method to complete base initialization + super()._configure(**kwargs) # type: ignore[misc] + + # Get tracer provider + tracer_provider = trace.get_tracer_provider() + + if isinstance(tracer_provider, TracerProvider): + # Get additional processors + additional_processors = self._get_additional_span_processors( + **kwargs + ) + + # Add additional processors + for processor in additional_processors: + tracer_provider.add_span_processor(processor) + + def _get_additional_span_processors( + self, **kwargs: Any + ) -> list["SpanProcessor"]: + """ + Return additional span processors to add to trace provider + + Subclasses can override this method to provide custom processors. + + Supports configuration via environment variables for baggage processor: + - LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES: Comma-separated list of prefixes for matching baggage keys + - LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES: Comma-separated list of prefixes to strip from baggage keys + + The baggage processor is only loaded if LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES is set. + + Args: + **kwargs: Arguments passed to _configure + + Returns: + List of span processors to add + """ + processors: list["SpanProcessor"] = [] + + # Check if baggage allowed prefixes is configured + allowed_prefixes_str = os.getenv( + _LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES + ) + + if allowed_prefixes_str: + # Try to load loongsuite-processor-baggage + try: + # Dynamic import to avoid type checker errors + from loongsuite.processor.baggage import ( # noqa: PLC0415 + LoongSuiteBaggageSpanProcessor, + ) + + # Parse allowed prefixes + allowed_prefixes = self._parse_prefixes(allowed_prefixes_str) + + # Parse strip prefixes + strip_prefixes_str = os.getenv( + _LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES + ) + strip_prefixes = ( + self._parse_prefixes(strip_prefixes_str) + if strip_prefixes_str + else None + ) + + # Create processor + # LoongSuiteBaggageSpanProcessor inherits from SpanProcessor + processor_instance = LoongSuiteBaggageSpanProcessor( # type: ignore[misc] + allowed_prefixes=allowed_prefixes + if allowed_prefixes + else None, + strip_prefixes=strip_prefixes if strip_prefixes else None, + ) + # Type cast since LoongSuiteBaggageSpanProcessor inherits from SpanProcessor + processor = cast("SpanProcessor", processor_instance) + processors.append(processor) + + logger.info( + "Loaded LoongSuiteBaggageSpanProcessor with allowed_prefixes=%s, strip_prefixes=%s", + allowed_prefixes, + strip_prefixes, + ) + except ImportError as e: + logger.warning( + "Failed to import loongsuite.processor.baggage: %s. " + "Baggage processor will not be loaded. " + "Please install loongsuite-processor-baggage package.", + e, + ) + + return processors + + @staticmethod + def _parse_prefixes(prefixes_str: str) -> Optional[Set[str]]: + """ + Parse comma-separated prefix string + + Args: + prefixes_str: Comma-separated prefix string, e.g., "traffic.,app." + + Returns: + Set of prefixes, or None if input is empty + """ + if not prefixes_str or not prefixes_str.strip(): + return None + + # Split and strip whitespace + prefixes = { + prefix.strip() + for prefix in prefixes_str.split(",") + if prefix.strip() + } + + return prefixes if prefixes else None + + +class LoongSuiteDistro(BaseDistro): + """ + LoongSuite Distro configures default OpenTelemetry settings. + + This is the Distro provided by LoongSuite, which configures default exporters and protocols. + """ + + # pylint: disable=no-self-use + def _configure(self, **kwargs: Any) -> None: + os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp") + os.environ.setdefault(OTEL_METRICS_EXPORTER, "otlp") + os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp") + os.environ.setdefault(OTEL_EXPORTER_OTLP_PROTOCOL, "grpc") diff --git a/loongsuite-distro/src/loongsuite/distro/bootstrap.py b/loongsuite-distro/src/loongsuite/distro/bootstrap.py new file mode 100644 index 000000000..ab3126ac2 --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/bootstrap.py @@ -0,0 +1,1299 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +LoongSuite Bootstrap Tool + +Two-phase installation strategy: +1. Install loongsuite-* packages from GitHub Release tar.gz (GenAI instrumentations) +2. Install opentelemetry-* packages from PyPI (standard instrumentations) + +The installation source is determined by package name prefix: +- loongsuite-* -> GitHub Release tar.gz +- opentelemetry-* -> PyPI + +loongsuite-util-genai is installed from PyPI as a base dependency. +""" + +import argparse +import json as json_lib +import logging +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request +import zipfile +from pathlib import Path +from typing import Any, List, Optional, Set, Tuple, Union + +from loongsuite.distro.bootstrap_gen import ( + default_instrumentations as gen_default_instrumentations, +) +from loongsuite.distro.bootstrap_gen import libraries as gen_libraries +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet + +logger = logging.getLogger(__name__) + +# Base dependency packages installed from PyPI +# loongsuite-util-genai is published to PyPI and required by GenAI instrumentations +BASE_DEPENDENCIES_PYPI = { + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation", + "loongsuite-util-genai", + "opentelemetry-semantic-conventions", +} + +# Packages to exclude from uninstallation +UNINSTALL_EXCLUDED_PACKAGES = { + "loongsuite-distro", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation", +} + + +def normalize_package_name(package_name: str) -> str: + """ + Normalize package name by converting underscores to hyphens. + + Package names in PyPI use hyphens, but wheel filenames may use underscores. + This function ensures consistent package name format. + + Args: + package_name: Package name (may contain underscores or hyphens) + + Returns: + Normalized package name with hyphens + """ + return package_name.replace("_", "-") + + +def get_package_name_variants(package_name: str) -> List[str]: + """ + Get all possible variants of a package name for lookup. + + This is useful when checking if a package is installed, as package names + may be stored with either underscores or hyphens. + + Args: + package_name: Package name + + Returns: + List of package name variants to try + """ + variants = [package_name] + normalized = normalize_package_name(package_name) + if normalized != package_name: + variants.append(normalized) + # Also try reverse (hyphens to underscores) for completeness + reverse = package_name.replace("-", "_") + if reverse != package_name and reverse not in variants: + variants.append(reverse) + return variants + + +def extract_package_name_from_requirement(req_str: str) -> str: + """ + Extract package name from a requirement string. + + Examples: + "redis >= 2.6" -> "redis" + "opentelemetry-instrumentation==0.60b0" -> "opentelemetry-instrumentation" + "package-name~=1.0" -> "package-name" + + Args: + req_str: Requirement string + + Returns: + Package name extracted from requirement + """ + try: + return Requirement(req_str).name + except Exception: + # Fallback: manual parsing if Requirement parsing fails + for op in ["==", ">=", "<=", "~=", "!=", ">", "<"]: + if op in req_str: + return req_str.split(op)[0].strip() + return req_str.strip() + + +def package_names_match(name1: str, name2: str) -> bool: + """ + Check if two package names match (considering normalization). + + Args: + name1: First package name + name2: Second package name + + Returns: + True if names match (after normalization), False otherwise + """ + normalized1 = normalize_package_name(name1) + normalized2 = normalize_package_name(name2) + return ( + normalized1 == normalized2 + or name1 == name2 + or normalized1 == name2 + or name1 == normalized2 + ) + + +def load_list_file(file_path: Path) -> Set[str]: + """Load list from file (one package name per line)""" + if not file_path.exists(): + return set() + + packages = set() + with open(file_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + packages.add(line) + + return packages + + +def get_package_name_from_whl(whl_path: Path) -> str: + """ + Extract package name from whl filename + + Wheel filename format: {package_name}-{version}-{python_tag}-{abi_tag}-{platform_tag}.whl + Example: loongsuite_instrumentation_mem0-0.1.0-py3-none-any.whl + + Returns normalized package name with hyphens (e.g., "loongsuite-instrumentation-mem0") + """ + name = whl_path.stem # Remove .whl extension + parts = name.split("-") + + if len(parts) < 2: + # If no hyphens, return as-is (shouldn't happen for valid wheels) + return name.replace("_", "-") + + package_parts = [] + for part in parts: + # Check if this part looks like a version number + # Version numbers typically: + # - Start with a digit + # - Contain dots (e.g., "0.1.0", "1.2.3") + # - Or are build tags like "dev", "b0", etc. + # - Or are Python/ABI/platform tags + + # Check for version-like patterns: starts with digit and contains dot, or is a known tag + is_version_like = ( + ( + part and part[0].isdigit() and "." in part + ) # e.g., "0.1.0", "1.2.3" + or part in ("dev", "b0", "b1", "rc0", "rc1") # Build tags + or part.startswith("py") # Python tags: "py3", "py2", "py" + or part in ("none", "any") # ABI/platform tags + ) + + if is_version_like: + break + + package_parts.append(part) + + if not package_parts: + # Fallback: if we couldn't extract, use first part + result = parts[0] if parts else name + else: + # Join with hyphens + result = "-".join(package_parts) + + # Normalize: convert underscores to hyphens for package name consistency + # (wheel filenames may use underscores, but package names use hyphens) + result = result.replace("_", "-") + return result + + +def get_metadata_from_whl(whl_path: Path) -> Optional[dict[str, Any]]: + """ + Extract metadata from whl file + + Args: + whl_path: Path to whl file + + Returns: + Dictionary with metadata fields, or None if not found + """ + try: + with zipfile.ZipFile(whl_path, "r") as whl_zip: + # Look for METADATA file in the wheel + metadata_path = None + for name in whl_zip.namelist(): + if name.endswith("/METADATA") or name == "METADATA": + metadata_path = name + break + + if not metadata_path: + return None + + metadata = {} + current_field = None + # Read METADATA file + with whl_zip.open(metadata_path) as metadata_file: + for line in metadata_file: + line_str = line.decode("utf-8").strip() + if not line_str: + current_field = None + continue + + # Check for continuation line + if line_str.startswith(" ") or line_str.startswith("\t"): + if current_field and current_field in metadata: + if isinstance(metadata[current_field], list): + if metadata[current_field]: + metadata[current_field][-1] += ( + " " + line_str.strip() + ) + else: + metadata[current_field] += ( + " " + line_str.strip() + ) + continue + + # Parse field name and value + if ":" in line_str: + field_name, field_value = line_str.split(":", 1) + field_name = field_name.strip() + field_value = field_value.strip() + current_field = field_name + + if field_name == "Requires-Python": + metadata["requires_python"] = field_value + elif field_name == "Requires-Dist": + if "requires_dist" not in metadata: + metadata["requires_dist"] = [] + metadata["requires_dist"].append(field_value) + elif field_name == "Provides-Extra": + if "provides_extra" not in metadata: + metadata["provides_extra"] = [] + metadata["provides_extra"].append(field_value) + + return metadata if metadata else None + except Exception: + pass + + return None + + +def get_python_requirement_from_whl(whl_path: Path) -> Optional[str]: + """ + Extract Python version requirement from whl file metadata + + Args: + whl_path: Path to whl file + + Returns: + Python version requirement string (e.g., ">=3.10, <=3.13") or None if not found + """ + metadata = get_metadata_from_whl(whl_path) + return metadata.get("requires_python") if metadata else None + + +def _try_get_package_version(package_name: str) -> Optional[str]: + """ + Try to get version of a package using pip show. + + Args: + package_name: Package name to check + + Returns: + Version string if found, None otherwise + """ + cmd = [sys.executable, "-m", "pip", "show", package_name] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=True, timeout=5 + ) + for line in result.stdout.splitlines(): + if line.startswith("Version:"): + return line.split(":", 1)[1].strip() + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.debug(f"Failed to get version for {package_name}: {e}") + except Exception as e: + logger.warning( + f"Unexpected error getting version for {package_name}: {e}" + ) + return None + + +def get_installed_package_version(package_name: str) -> Optional[str]: + """ + Get installed version of a package. + + Tries multiple name variants (with underscores/hyphens) to handle + different naming conventions. + + Args: + package_name: Package name (may contain hyphens or underscores) + + Returns: + Installed version string, or None if not installed + """ + variants = get_package_name_variants(package_name) + for variant in variants: + version = _try_get_package_version(variant) + if version: + return version + return None + + +def _is_library_installed(req_str: str) -> bool: + """ + Check if a library is installed and version satisfies requirement. + + Similar to opentelemetry-bootstrap's _is_installed function. + + Args: + req_str: Requirement string (e.g., "redis >= 2.6") + + Returns: + True if library is installed and version satisfies requirement, False otherwise + """ + try: + req = Requirement(req_str) + package_name = req.name + + # get_installed_package_version already tries multiple variants + dist_version = get_installed_package_version(package_name) + + if dist_version is None: + return False + + # Check if installed version satisfies requirement + return req.specifier.contains(dist_version) + except Exception as e: + logger.debug( + f"Failed to check if library is installed for {req_str}: {e}" + ) + return False + + +def _is_instrumentation_in_bootstrap_gen(package_name: str) -> bool: + """ + Check if a package is an instrumentation listed in bootstrap_gen.py. + + Args: + package_name: Package name to check + + Returns: + True if the package is in bootstrap_gen.py (either in libraries or default_instrumentations) + """ + if not package_name: + return False + + # Check default instrumentations + for default_instr in gen_default_instrumentations: + if isinstance(default_instr, str): + default_pkg_name = extract_package_name_from_requirement( + default_instr + ) + if package_names_match(default_pkg_name, package_name): + return True + + # Check libraries mapping + for lib_mapping in gen_libraries: + instrumentation = lib_mapping.get("instrumentation", "") + if isinstance(instrumentation, str): + instr_pkg_name = extract_package_name_from_requirement( + instrumentation + ) + if package_names_match(instr_pkg_name, package_name): + return True + + return False + + +def _is_loongsuite_package(package_name: str) -> bool: + """Check if package is a loongsuite package (installed from GitHub Release tar)""" + return package_name.startswith("loongsuite-") + + +def _get_desired_instrumentation_requirements( + blacklist: Optional[Set[str]] = None, + whitelist: Optional[Set[str]] = None, + auto_detect: bool = False, +) -> Tuple[List[Tuple[str, str]], List[Tuple[str, str]]]: + """ + Get desired instrumentation packages from bootstrap_gen with filtering. + + Returns: + (tar_packages, pypi_packages) + - tar_packages: loongsuite-* packages to install from GitHub Release tar.gz + - pypi_packages: opentelemetry-* packages to install from PyPI + """ + blacklist = blacklist or set() + whitelist = whitelist or set() + tar_packages: List[Tuple[str, str]] = [] + pypi_packages: List[Tuple[str, str]] = [] + + def _should_include( + pkg_name: str, target_libraries: List[str], is_default: bool + ) -> bool: + if blacklist and pkg_name in blacklist: + return False + if whitelist and pkg_name not in whitelist: + return False + if is_default: + return True + if auto_detect and target_libraries: + return any(_is_library_installed(lib) for lib in target_libraries) + return not auto_detect + + seen: Set[str] = set() + for default_instr in gen_default_instrumentations: + if isinstance(default_instr, str): + pkg_name = extract_package_name_from_requirement(default_instr) + if pkg_name not in seen and _should_include(pkg_name, [], True): + seen.add(pkg_name) + if _is_loongsuite_package(pkg_name): + tar_packages.append((pkg_name, default_instr)) + else: + pypi_packages.append((pkg_name, default_instr)) + + for lib_mapping in gen_libraries: + instrumentation = lib_mapping.get("instrumentation", "") + if isinstance(instrumentation, str): + pkg_name = extract_package_name_from_requirement(instrumentation) + target_lib = lib_mapping.get("library", "") + target_libraries = [target_lib] if target_lib else [] + if pkg_name not in seen and _should_include( + pkg_name, target_libraries, False + ): + seen.add(pkg_name) + if _is_loongsuite_package(pkg_name): + tar_packages.append((pkg_name, instrumentation)) + else: + pypi_packages.append((pkg_name, instrumentation)) + + return tar_packages, pypi_packages + + +def get_target_libraries_from_bootstrap_gen( + package_name: str, +) -> Tuple[List[str], bool]: + """ + Get target library requirements from bootstrap_gen.py. + + This function uses the pre-generated bootstrap_gen.py file to get + target library information, similar to opentelemetry-bootstrap. + + Args: + package_name: Name of the instrumentation package (e.g., "opentelemetry-instrumentation-redis") + May contain hyphens or underscores, will be normalized + + Returns: + Tuple of (target_libraries list, is_default_instrumentation bool) + target_libraries contains library requirement strings (e.g., ["redis >= 2.6"]) + is_default_instrumentation is True if this is a default instrumentation + """ + if not package_name: + return [], False + + # Check if it's a default instrumentation + for default_instr in gen_default_instrumentations: + if isinstance(default_instr, str): + default_pkg_name = extract_package_name_from_requirement( + default_instr + ) + if package_names_match(default_pkg_name, package_name): + return [], True + + # Look up in libraries mapping + target_libraries = [] + for lib_mapping in gen_libraries: + instrumentation = lib_mapping.get("instrumentation", "") + if isinstance(instrumentation, str): + instr_pkg_name = extract_package_name_from_requirement( + instrumentation + ) + if package_names_match(instr_pkg_name, package_name): + target_lib = lib_mapping.get("library", "") + if target_lib and isinstance(target_lib, str): + target_libraries.append(target_lib) + + return target_libraries, False + + +def check_dependency_compatibility( + whl_path: Path, skip_version_check: bool = False +) -> Tuple[bool, Optional[str]]: + """ + Check if package dependencies are compatible with installed packages + + Args: + whl_path: Path to whl file + skip_version_check: If True, skip version compatibility check + + Returns: + (is_compatible, conflict_message) + is_compatible: True if compatible, False otherwise + conflict_message: Description of conflict if incompatible, None otherwise + """ + if skip_version_check: + return True, None + + metadata = get_metadata_from_whl(whl_path) + if not metadata or "requires_dist" not in metadata: + return True, None + + # Key packages to check compatibility + key_packages = { + "opentelemetry-instrumentation", + "opentelemetry-semantic-conventions", + } + + conflicts = [] + for req_str in metadata.get("requires_dist", []): + try: + req = Requirement(req_str) + if req.name.lower() in key_packages: + installed_version = get_installed_package_version(req.name) + if installed_version: + # Check if installed version satisfies requirement + if not req.specifier.contains(installed_version): + conflicts.append( + f"{req.name} {installed_version} does not satisfy {req_str}" + ) + except Exception: + # If parsing fails, assume compatible to avoid false positives + continue + + if conflicts: + conflict_msg = "; ".join(conflicts) + return False, conflict_msg + + return True, None + + +def check_python_version_compatibility( + whl_path: Path, current_version: Tuple[int, int] +) -> Tuple[bool, Optional[str]]: + """ + Check if current Python version is compatible with whl file requirements + + Args: + whl_path: Path to whl file + current_version: Current Python version as (major, minor) tuple + + Returns: + (is_compatible, requirement_string) + is_compatible: True if compatible, False otherwise + requirement_string: Python requirement string if found, None otherwise + """ + requirement_str = get_python_requirement_from_whl(whl_path) + + if not requirement_str: + # If no requirement found, assume compatible + return True, None + + try: + # Parse the requirement string + spec = SpecifierSet(requirement_str) + # Convert current version to string format + current_version_str = f"{current_version[0]}.{current_version[1]}" + # Check if current version satisfies the requirement + is_compatible = spec.contains(current_version_str) + return is_compatible, requirement_str + except Exception: + # If parsing fails, assume compatible to avoid false positives + return True, requirement_str + + +def download_file(url: str, dest: Path) -> Path: + """Download file to specified path""" + logger.info(f"Downloading file: {url}") + urllib.request.urlretrieve(url, dest) + logger.info(f"Download completed: {dest}") + return dest + + +def extract_tar(tar_path: Path, extract_dir: Path) -> List[Path]: + """Extract tar.gz file, return all whl file paths""" + logger.info(f"Extracting tar file: {tar_path} -> {extract_dir}") + + whl_files = [] + with tarfile.open(tar_path, "r:gz") as tar: + tar.extractall(extract_dir) + + # Find all whl files + for member in tar.getmembers(): + if member.name.endswith(".whl"): + whl_path = extract_dir / member.name + if whl_path.exists(): + whl_files.append(whl_path) + + logger.info(f"Extraction completed, found {len(whl_files)} whl files") + return sorted(whl_files) + + +def filter_packages( + whl_files: List[Path], + blacklist: Optional[Set[str]] = None, + whitelist: Optional[Set[str]] = None, + skip_version_check: bool = False, + auto_detect: bool = False, +) -> Tuple[List[Path], List[Path]]: + """ + Filter packages based on blacklist/whitelist, Python version compatibility, + dependency version compatibility, and optionally auto-detect installed libraries + + Args: + whl_files: List of whl file paths + blacklist: blacklist (do not install these packages) + whitelist: whitelist (only install these packages if specified) + skip_version_check: If True, skip dependency version compatibility check + auto_detect: If True, only install instrumentation packages if their target libraries are installed + + Returns: + (base dependency packages list, instrumentation packages list) + """ + base_packages = [] + instrumentation_packages = [] + + blacklist = blacklist or set() + whitelist = whitelist or set() + + # Get current Python version + current_version = (sys.version_info.major, sys.version_info.minor) + current_version_str = f"{current_version[0]}.{current_version[1]}" + + logger.info(f"Scanning {len(whl_files)} packages for installation...") + if auto_detect: + logger.info( + "Auto-detect mode enabled: will only install instrumentations for detected libraries" + ) + + for whl_file in whl_files: + package_name = get_package_name_from_whl(whl_file) + + # Check blacklist + if blacklist and package_name in blacklist: + logger.info(f"Skipping {package_name} (blacklist)") + continue + + # Check whitelist + if whitelist and package_name not in whitelist: + logger.info(f"Skipping {package_name} (not in whitelist)") + continue + + # Check Python version compatibility (only for instrumentations in bootstrap_gen.py) + # Base dependencies and utility packages are installed without Python version check + is_instrumentation = _is_instrumentation_in_bootstrap_gen(package_name) + if is_instrumentation: + is_compatible, requirement_str = ( + check_python_version_compatibility(whl_file, current_version) + ) + if not is_compatible: + logger.info( + f"Skipping {package_name} (Python version incompatible: requires {requirement_str}, current: {current_version_str})" + ) + continue + + # Check dependency version compatibility (only for base dependencies) + # Instrumentation packages will be checked by pip during installation + if package_name in BASE_DEPENDENCIES_PYPI: + is_dep_compatible, conflict_msg = check_dependency_compatibility( + whl_file, skip_version_check + ) + if not is_dep_compatible: + logger.warning( + f"Skipping {package_name} (dependency version incompatible: {conflict_msg})" + ) + continue + + # Classify: base dependencies vs instrumentation + if package_name in BASE_DEPENDENCIES_PYPI: + base_packages.append(whl_file) + else: + # For instrumentation packages, check if auto-detect is enabled + if auto_detect: + target_libraries, is_default = ( + get_target_libraries_from_bootstrap_gen(package_name) + ) + + # Default instrumentations are always installed (like opentelemetry-bootstrap) + if is_default: + logger.info( + f"Will install {package_name} (default instrumentation)" + ) + instrumentation_packages.append(whl_file) + elif target_libraries: + # Check if any target library is installed + library_installed = False + installed_libs = [] + not_installed_libs = [] + for lib_req in target_libraries: + if _is_library_installed(lib_req): + library_installed = True + try: + req = Requirement(lib_req) + installed_libs.append(req.name) + except Exception: + installed_libs.append(lib_req) + else: + try: + req = Requirement(lib_req) + not_installed_libs.append(req.name) + except Exception: + not_installed_libs.append(lib_req) + + if library_installed: + logger.info( + f"Will install {package_name} (detected libraries: {', '.join(installed_libs)})" + ) + instrumentation_packages.append(whl_file) + else: + logger.info( + f"Skipping {package_name} (required libraries not installed: {', '.join(not_installed_libs)})" + ) + continue + else: + # No mapping found in bootstrap_gen.py, skip it + logger.info( + f"Skipping {package_name} (no target libraries mapping in bootstrap_gen.py)" + ) + continue + else: + # Auto-detect disabled, install all instrumentation packages + logger.info(f"Will install {package_name}") + instrumentation_packages.append(whl_file) + + return base_packages, instrumentation_packages + + +def install_packages( + whl_files: List[Path], + find_links_dir: Path, + upgrade: bool = False, + extra_requirements: Optional[List[str]] = None, +): + """Install packages using pip. extra_requirements are installed from PyPI.""" + if not whl_files and not extra_requirements: + logger.warning("No packages to install") + return + + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--find-links", + str(find_links_dir), + ] + + if upgrade: + cmd.append("--upgrade") + + # Add whl files (from tar) and extra requirements (from PyPI) + cmd.extend([str(whl) for whl in whl_files]) + if extra_requirements: + cmd.extend(extra_requirements) + + logger.info(f"Executing install command: {' '.join(cmd)}") + try: + subprocess.run(cmd, check=True) + logger.info("Installation completed") + except subprocess.CalledProcessError as e: + logger.error(f"Installation failed: {e}") + raise + + +def get_installed_loongsuite_packages() -> List[str]: + """ + Get list of installed loongsuite and opentelemetry packages to uninstall + + Excludes: + - loongsuite-distro + - opentelemetry-api + - opentelemetry-sdk + - opentelemetry-instrumentation + + Returns: + List of installed package names to uninstall + """ + cmd = [sys.executable, "-m", "pip", "list", "--format=json"] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=True + ) + installed_packages = json_lib.loads(result.stdout) + + # Filter packages to uninstall + packages_to_uninstall = [] + for pkg in installed_packages: + name = pkg.get("name", "") + name_lower = name.lower() + + # Skip excluded packages + if name_lower in UNINSTALL_EXCLUDED_PACKAGES: + continue + + # Include loongsuite-* packages (except loongsuite-distro) + if name_lower.startswith("loongsuite-"): + packages_to_uninstall.append(name) + # Include opentelemetry-* packages (except opentelemetry-api and opentelemetry-sdk) + elif name_lower.startswith("opentelemetry-"): + packages_to_uninstall.append(name) + + return packages_to_uninstall + except subprocess.CalledProcessError as e: + logger.error(f"Failed to get installed packages: {e}") + raise + except json_lib.JSONDecodeError as e: + logger.error(f"Failed to parse pip list output: {e}") + raise + + +def uninstall_packages(package_names: List[str], yes: bool = False): + """Uninstall packages using pip""" + if not package_names: + logger.warning("No packages to uninstall") + return + + cmd = [ + sys.executable, + "-m", + "pip", + "uninstall", + ] + + if yes: + cmd.append("-y") + + # Add all package names + cmd.extend(package_names) + + logger.info(f"Executing uninstall command: {' '.join(cmd)}") + try: + subprocess.run(cmd, check=True) + logger.info("Uninstallation completed") + except subprocess.CalledProcessError as e: + logger.error(f"Uninstallation failed: {e}") + raise + + +def resolve_tar_path( + tar_path: Union[Path, str], +) -> Tuple[Path, Optional[Path]]: + """ + Resolve tar path, downloading from URI if necessary + + Args: + tar_path: tar file path or URI (can be Path or str) + + Returns: + (local_tar_path, temp_dir_to_cleanup) + local_tar_path: Path to local tar file + temp_dir_to_cleanup: Path to temporary directory to clean up (None if not downloaded) + """ + tar_path_str = str(tar_path) + if tar_path_str.startswith(("http://", "https://")): + # Download from URI + temp_dir = Path(tempfile.mkdtemp(prefix="loongsuite-download-")) + temp_tar = temp_dir / "loongsuite.tar.gz" + download_file(tar_path_str, temp_tar) + return temp_tar, temp_dir + else: + tar_path = Path(tar_path) + if not tar_path.exists(): + raise FileNotFoundError(f"Tar file does not exist: {tar_path}") + return tar_path, None + + +def get_package_names_from_tar( + tar_path: Path, + blacklist: Optional[Set[str]] = None, + whitelist: Optional[Set[str]] = None, +) -> List[str]: + """ + Extract package names from tar file + + Args: + tar_path: Path to tar file + blacklist: blacklist (do not include these packages) + whitelist: whitelist (only include these packages if specified) + + Returns: + List of package names + """ + temp_dir = Path(tempfile.mkdtemp(prefix="loongsuite-")) + try: + whl_files = extract_tar(tar_path, temp_dir) + if not whl_files: + raise ValueError("No whl files found in tar file") + + base_packages, instrumentation_packages = filter_packages( + whl_files, blacklist, whitelist, auto_detect=False + ) + + # Get package names + package_names = [] + for whl in base_packages + instrumentation_packages: + package_name = get_package_name_from_whl(whl) + package_names.append(package_name) + + return package_names + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + +def install_from_tar( + tar_path: Union[Path, str], + blacklist: Optional[Set[str]] = None, + whitelist: Optional[Set[str]] = None, + upgrade: bool = False, + keep_temp: bool = False, + skip_version_check: bool = False, + auto_detect: bool = False, +): + """ + Two-phase installation from tar package: + 1. Install loongsuite-* packages from GitHub Release tar.gz (GenAI instrumentations) + 2. Install opentelemetry-* packages from PyPI (standard instrumentations) + + Args: + tar_path: tar file path or URI (can be Path or str) + blacklist: blacklist (do not install these packages) + whitelist: whitelist (only install these packages if specified) + upgrade: whether to upgrade already installed packages + keep_temp: whether to keep temporary directory + skip_version_check: If True, skip dependency version compatibility check + auto_detect: If True, only install instrumentation packages if their target libraries are installed + """ + # Resolve tar path (download from URI if necessary) + local_tar_path, temp_tar_dir = resolve_tar_path(tar_path) + + # Create temporary directory for extraction + temp_dir = Path(tempfile.mkdtemp(prefix="loongsuite-")) + + try: + logger.info("Extracting packages from tar file...") + # Extract tar file + whl_files = extract_tar(local_tar_path, temp_dir) + + if not whl_files: + raise ValueError("No whl files found in tar file") + + logger.info(f"Found {len(whl_files)} packages in tar file") + + # Filter packages from tar (loongsuite-* packages) + logger.info("Filtering packages...") + base_packages, instrumentation_packages = filter_packages( + whl_files, blacklist, whitelist, skip_version_check, auto_detect + ) + + # Get desired packages from bootstrap_gen + tar_desired, pypi_desired = _get_desired_instrumentation_requirements( + blacklist, whitelist, auto_detect + ) + + # Build package name set from tar + tar_package_names = { + normalize_package_name(get_package_name_from_whl(w)) + for w in whl_files + } + + # loongsuite-* packages from tar (already filtered) + tar_packages = base_packages + instrumentation_packages + + # opentelemetry-* packages from PyPI (use requirement string with version) + pypi_requirements: List[str] = [] + for pkg_name, req_str in pypi_desired: + norm_name = normalize_package_name(pkg_name) + if norm_name not in tar_package_names: + # Use full requirement string (e.g., "opentelemetry-instrumentation-flask==0.60b1") + pypi_requirements.append(req_str) + + if not tar_packages and not pypi_requirements: + logger.warning("No packages to install after filtering") + return + + # Phase 1: Install from tar.gz (loongsuite-* packages) + logger.info("=" * 50) + logger.info("Phase 1: Installing loongsuite-* packages from tar...") + logger.info("=" * 50) + if tar_packages: + logger.info(f"Will install {len(tar_packages)} packages from tar:") + for pkg in tar_packages: + pkg_name = get_package_name_from_whl(pkg) + logger.info(f" - {pkg_name}") + install_packages(tar_packages, temp_dir, upgrade) + else: + logger.info("No loongsuite-* packages to install from tar") + + # Phase 2: Install from PyPI (opentelemetry-* packages) + logger.info("=" * 50) + logger.info( + "Phase 2: Installing opentelemetry-* packages from PyPI..." + ) + logger.info("=" * 50) + if pypi_requirements: + logger.info( + f"Will install {len(pypi_requirements)} packages from PyPI:" + ) + for req in pypi_requirements: + logger.info(f" - {req}") + install_packages( + [], temp_dir, upgrade, extra_requirements=pypi_requirements + ) + else: + logger.info("No opentelemetry-* packages to install from PyPI") + + logger.info("=" * 50) + logger.info("Installation completed successfully!") + logger.info("=" * 50) + + finally: + if not keep_temp: + shutil.rmtree(temp_dir, ignore_errors=True) + if temp_tar_dir and temp_tar_dir.exists(): + shutil.rmtree(temp_tar_dir, ignore_errors=True) + else: + logger.info(f"Temporary directory kept at: {temp_dir}") + if temp_tar_dir: + logger.info(f"Downloaded tar file kept at: {local_tar_path}") + + +def uninstall_loongsuite_packages( + blacklist: Optional[Set[str]] = None, + whitelist: Optional[Set[str]] = None, + yes: bool = False, +): + """ + Uninstall installed loongsuite packages + + Args: + blacklist: blacklist (do not uninstall these packages) + whitelist: whitelist (only uninstall these packages if specified) + yes: automatically confirm uninstallation + """ + # Get installed loongsuite packages + installed_packages = get_installed_loongsuite_packages() + + if not installed_packages: + logger.warning("No loongsuite packages found installed") + return + + # Apply blacklist/whitelist filters + blacklist = blacklist or set() + whitelist = whitelist or set() + + package_names = [] + for pkg in installed_packages: + # Check blacklist + if blacklist and pkg in blacklist: + logger.debug(f"Skipping package (blacklist): {pkg}") + continue + + # Check whitelist + if whitelist and pkg not in whitelist: + logger.debug(f"Skipping package (not in whitelist): {pkg}") + continue + + package_names.append(pkg) + + if not package_names: + logger.warning("No packages to uninstall after filtering") + return + + logger.info(f"Will uninstall {len(package_names)} packages:") + for name in package_names: + logger.info(f" - {name}") + + # Uninstall + uninstall_packages(package_names, yes) + + +def get_latest_release_url( + repo: str = "alibaba/loongsuite-python-agent", +) -> str: + """Get latest release tar.gz URL from GitHub API""" + api_url = f"https://api.github.com/repos/{repo}/releases/latest" + logger.info(f"Fetching latest release: {api_url}") + + try: + with urllib.request.urlopen(api_url) as response: + data = json_lib.loads(response.read()) + for asset in data.get("assets", []): + if asset["name"].endswith(".tar.gz"): + return asset["browser_download_url"] + + # If no asset found, try to build URL from tag + tag = data.get("tag_name", "").lstrip("v") + return f"https://github.com/{repo}/releases/download/{data.get('tag_name')}/loongsuite-python-agent-{tag}.tar.gz" + except Exception as e: + logger.error(f"Failed to fetch latest release: {e}") + raise + + +def main(): + parser = argparse.ArgumentParser( + description=""" + LoongSuite Bootstrap - Install/Uninstall loongsuite Python Agent from tar package + + This tool installs or uninstalls all loongsuite components from tar.gz file. + Supports blacklist/whitelist to control which instrumentations to install/uninstall. + """ + ) + + parser.add_argument( + "-a", + "--action", + choices=["install", "uninstall"], + required=True, + help="action type: install to install packages, uninstall to uninstall packages", + ) + + # Common arguments + parser.add_argument( + "--blacklist", + type=Path, + help="blacklist file path (one package name per line, do not install/uninstall these packages)", + ) + parser.add_argument( + "--whitelist", + type=Path, + help="whitelist file path (one package name per line, only install/uninstall these packages)", + ) + + # Install-specific arguments + install_group = parser.add_argument_group("install options") + install_group.add_argument( + "-t", + "--tar", + type=str, + help="tar package path or URI (required for install action, supports http:// and https://)", + ) + install_group.add_argument( + "-v", + "--version", + type=str, + help="version number, download from GitHub Releases (e.g., 1.0.0) (for install action)", + ) + install_group.add_argument( + "--latest", + action="store_true", + help="install latest version (from GitHub Releases) (for install action)", + ) + install_group.add_argument( + "--upgrade", + action="store_true", + help="upgrade already installed packages (for install action)", + ) + install_group.add_argument( + "--keep-temp", + action="store_true", + help="keep temporary directory (for debugging)", + ) + install_group.add_argument( + "--force", + action="store_true", + help="force installation even if dependency versions are incompatible", + ) + install_group.add_argument( + "--auto-detect", + action="store_true", + help="only install instrumentation packages if their target libraries are installed (similar to opentelemetry-bootstrap)", + ) + install_group.add_argument( + "--verbose", + action="store_true", + help="enable verbose debug logging", + ) + + # Uninstall-specific arguments + uninstall_group = parser.add_argument_group("uninstall options") + uninstall_group.add_argument( + "-y", + "--yes", + action="store_true", + help="automatically confirm uninstallation (for uninstall action)", + ) + + args = parser.parse_args() + + # Configure logging level + if args.verbose or ( + hasattr(args, "action") + and args.action == "install" + and hasattr(args, "auto_detect") + and args.auto_detect + ): + logging.basicConfig( + level=logging.DEBUG, + format="%(levelname)s: %(message)s", + force=True, + ) + logger.setLevel(logging.DEBUG) + else: + logging.basicConfig( + level=logging.INFO, format="%(levelname)s: %(message)s", force=True + ) + logger.setLevel(logging.INFO) + + # Load blacklist/whitelist + blacklist = load_list_file(args.blacklist) if args.blacklist else None + whitelist = load_list_file(args.whitelist) if args.whitelist else None + + if blacklist: + logger.info(f"Blacklist: {len(blacklist)} packages") + if whitelist: + logger.info(f"Whitelist: {len(whitelist)} packages") + + if args.action == "install": + # Determine tar file path + tar_path = None + if args.tar: + tar_path = args.tar + elif args.version: + tar_path = f"https://github.com/alibaba/loongsuite-python-agent/releases/download/v{args.version}/loongsuite-python-agent-{args.version}.tar.gz" + elif args.latest: + tar_path = get_latest_release_url() + else: + parser.error( + "For install action, must specify one of --tar, --version, or --latest" + ) + + # Install + install_from_tar( + tar_path, + blacklist=blacklist, + whitelist=whitelist, + upgrade=args.upgrade, + keep_temp=args.keep_temp, + skip_version_check=args.force, + auto_detect=args.auto_detect, + ) + + elif args.action == "uninstall": + # Uninstall installed loongsuite packages + uninstall_loongsuite_packages( + blacklist=blacklist, + whitelist=whitelist, + yes=args.yes, + ) + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(levelname)s: %(message)s", + ) + main() diff --git a/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py b/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py new file mode 100644 index 000000000..3a50509ab --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py @@ -0,0 +1,262 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT. THIS FILE WAS AUTOGENERATED FROM INSTRUMENTATION PACKAGES. +# RUN `python scripts/generate_loongsuite_bootstrap.py` TO REGENERATE. +# +# Generated with options: +# --upstream-version: (from source) +# --loongsuite-version: (from source) + +libraries = [ + { + "library": "openai >= 1.26.0", + "instrumentation": "loongsuite-instrumentation-openai-v2", + }, + { + "library": "google-cloud-aiplatform >= 1.64", + "instrumentation": "loongsuite-instrumentation-vertexai>=2.0b0", + }, + { + "library": "aio_pika >= 7.2.0, < 10.0.0", + "instrumentation": "opentelemetry-instrumentation-aio-pika==0.61b0.dev", + }, + { + "library": "aiohttp ~= 3.0", + "instrumentation": "opentelemetry-instrumentation-aiohttp-client==0.61b0.dev", + }, + { + "library": "aiohttp ~= 3.0", + "instrumentation": "opentelemetry-instrumentation-aiohttp-server==0.61b0.dev", + }, + { + "library": "aiokafka >= 0.8, < 1.0", + "instrumentation": "opentelemetry-instrumentation-aiokafka==0.61b0.dev", + }, + { + "library": "aiopg >= 0.13.0, < 2.0.0", + "instrumentation": "opentelemetry-instrumentation-aiopg==0.61b0.dev", + }, + { + "library": "asgiref ~= 3.0", + "instrumentation": "opentelemetry-instrumentation-asgi==0.61b0.dev", + }, + { + "library": "asyncclick ~= 8.0", + "instrumentation": "opentelemetry-instrumentation-asyncclick==0.61b0.dev", + }, + { + "library": "asyncpg >= 0.12.0", + "instrumentation": "opentelemetry-instrumentation-asyncpg==0.61b0.dev", + }, + { + "library": "boto~=2.0", + "instrumentation": "opentelemetry-instrumentation-boto==0.61b0.dev", + }, + { + "library": "boto3 ~= 1.0", + "instrumentation": "opentelemetry-instrumentation-boto3sqs==0.61b0.dev", + }, + { + "library": "botocore ~= 1.0", + "instrumentation": "opentelemetry-instrumentation-botocore==0.61b0.dev", + }, + { + "library": "cassandra-driver ~= 3.25", + "instrumentation": "opentelemetry-instrumentation-cassandra==0.61b0.dev", + }, + { + "library": "scylla-driver ~= 3.25", + "instrumentation": "opentelemetry-instrumentation-cassandra==0.61b0.dev", + }, + { + "library": "celery >= 4.0, < 6.0", + "instrumentation": "opentelemetry-instrumentation-celery==0.61b0.dev", + }, + { + "library": "click >= 8.1.3, < 9.0.0", + "instrumentation": "opentelemetry-instrumentation-click==0.61b0.dev", + }, + { + "library": "confluent-kafka >= 1.8.2, <= 2.11.0", + "instrumentation": "opentelemetry-instrumentation-confluent-kafka==0.61b0.dev", + }, + { + "library": "django >= 1.10", + "instrumentation": "opentelemetry-instrumentation-django==0.61b0.dev", + }, + { + "library": "elasticsearch >= 6.0", + "instrumentation": "opentelemetry-instrumentation-elasticsearch==0.61b0.dev", + }, + { + "library": "falcon >= 1.4.1, < 5.0.0", + "instrumentation": "opentelemetry-instrumentation-falcon==0.61b0.dev", + }, + { + "library": "fastapi ~= 0.92", + "instrumentation": "opentelemetry-instrumentation-fastapi==0.61b0.dev", + }, + { + "library": "flask >= 1.0", + "instrumentation": "opentelemetry-instrumentation-flask==0.61b0.dev", + }, + { + "library": "grpcio >= 1.42.0", + "instrumentation": "opentelemetry-instrumentation-grpc==0.61b0.dev", + }, + { + "library": "httpx >= 0.18.0", + "instrumentation": "opentelemetry-instrumentation-httpx==0.61b0.dev", + }, + { + "library": "jinja2 >= 2.7, < 4.0", + "instrumentation": "opentelemetry-instrumentation-jinja2==0.61b0.dev", + }, + { + "library": "kafka-python >= 2.0, < 3.0", + "instrumentation": "opentelemetry-instrumentation-kafka-python==0.61b0.dev", + }, + { + "library": "kafka-python-ng >= 2.0, < 3.0", + "instrumentation": "opentelemetry-instrumentation-kafka-python==0.61b0.dev", + }, + { + "library": "mysql-connector-python >= 8.0, < 10.0", + "instrumentation": "opentelemetry-instrumentation-mysql==0.61b0.dev", + }, + { + "library": "mysqlclient < 3", + "instrumentation": "opentelemetry-instrumentation-mysqlclient==0.61b0.dev", + }, + { + "library": "pika >= 0.12.0", + "instrumentation": "opentelemetry-instrumentation-pika==0.61b0.dev", + }, + { + "library": "psycopg >= 3.1.0", + "instrumentation": "opentelemetry-instrumentation-psycopg==0.61b0.dev", + }, + { + "library": "psycopg2 >= 2.7.3.1", + "instrumentation": "opentelemetry-instrumentation-psycopg2==0.61b0.dev", + }, + { + "library": "psycopg2-binary >= 2.7.3.1", + "instrumentation": "opentelemetry-instrumentation-psycopg2==0.61b0.dev", + }, + { + "library": "pymemcache >= 1.3.5, < 5", + "instrumentation": "opentelemetry-instrumentation-pymemcache==0.61b0.dev", + }, + { + "library": "pymongo >= 3.1, < 5.0", + "instrumentation": "opentelemetry-instrumentation-pymongo==0.61b0.dev", + }, + { + "library": "pymssql >= 2.1.5, < 3", + "instrumentation": "opentelemetry-instrumentation-pymssql==0.61b0.dev", + }, + { + "library": "PyMySQL < 2", + "instrumentation": "opentelemetry-instrumentation-pymysql==0.61b0.dev", + }, + { + "library": "pyramid >= 1.7", + "instrumentation": "opentelemetry-instrumentation-pyramid==0.61b0.dev", + }, + { + "library": "redis >= 2.6", + "instrumentation": "opentelemetry-instrumentation-redis==0.61b0.dev", + }, + { + "library": "remoulade >= 0.50", + "instrumentation": "opentelemetry-instrumentation-remoulade==0.61b0.dev", + }, + { + "library": "requests ~= 2.0", + "instrumentation": "opentelemetry-instrumentation-requests==0.61b0.dev", + }, + { + "library": "sqlalchemy >= 1.0.0, < 2.1.0", + "instrumentation": "opentelemetry-instrumentation-sqlalchemy==0.61b0.dev", + }, + { + "library": "starlette >= 0.13", + "instrumentation": "opentelemetry-instrumentation-starlette==0.61b0.dev", + }, + { + "library": "psutil >= 5", + "instrumentation": "opentelemetry-instrumentation-system-metrics==0.61b0.dev", + }, + { + "library": "tornado >= 5.1.1", + "instrumentation": "opentelemetry-instrumentation-tornado==0.61b0.dev", + }, + { + "library": "tortoise-orm >= 0.17.0", + "instrumentation": "opentelemetry-instrumentation-tortoiseorm==0.61b0.dev", + }, + { + "library": "pydantic >= 1.10.2", + "instrumentation": "opentelemetry-instrumentation-tortoiseorm==0.61b0.dev", + }, + { + "library": "urllib3 >= 1.0.0, < 3.0.0", + "instrumentation": "opentelemetry-instrumentation-urllib3==0.61b0.dev", + }, + { + "library": "agentscope >= 1.0.0", + "instrumentation": "loongsuite-instrumentation-agentscope==1.0.0", + }, + { + "library": "agno", + "instrumentation": "loongsuite-instrumentation-agno==0.1b0.dev", + }, + { + "library": "claude-agent-sdk >= 0.1.0", + "instrumentation": "loongsuite-instrumentation-claude-agent-sdk==0.1.0.dev0", + }, + { + "library": "dashscope >= 1.0.0", + "instrumentation": "loongsuite-instrumentation-dashscope==0.1.0.dev0", + }, + { + "library": "google-adk >= 0.1.0", + "instrumentation": "loongsuite-instrumentation-google-adk==0.1.0", + }, + { + "library": "langchain_core >= 0.1.0", + "instrumentation": "loongsuite-instrumentation-langchain==1.0.0", + }, + { + "library": "mcp >= 1.3.0, <= 1.25.0", + "instrumentation": "loongsuite-instrumentation-mcp==0.1.0", + }, + { + "library": "mem0ai >= 1.0.0", + "instrumentation": "loongsuite-instrumentation-mem0==0.1.0", + }, +] + +default_instrumentations = [ + "opentelemetry-instrumentation-asyncio==0.61b0.dev", + "opentelemetry-instrumentation-dbapi==0.61b0.dev", + "opentelemetry-instrumentation-logging==0.61b0.dev", + "opentelemetry-instrumentation-sqlite3==0.61b0.dev", + "opentelemetry-instrumentation-threading==0.61b0.dev", + "opentelemetry-instrumentation-urllib==0.61b0.dev", + "opentelemetry-instrumentation-wsgi==0.61b0.dev", + "loongsuite-instrumentation-dify==1.1.0", +] diff --git a/loongsuite-distro/src/loongsuite/distro/py.typed b/loongsuite-distro/src/loongsuite/distro/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/loongsuite-distro/src/loongsuite/distro/version.py b/loongsuite-distro/src/loongsuite/distro/version.py new file mode 100644 index 000000000..4effd145c --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.1.0.dev" diff --git a/pkg-requirements.txt b/pkg-requirements.txt new file mode 100644 index 000000000..460f1ac63 --- /dev/null +++ b/pkg-requirements.txt @@ -0,0 +1,11 @@ +# LoongSuite build dependencies +build>=1.0.0 +setuptools>=65.0.0 +wheel>=0.40.0 + +# LoongSuite release dependencies +tomli>=2.0.0 +tomlkit>=0.12.0 +hatch>=1.7.0 +astor>=0.8.0 +packaging>=21.0 diff --git a/scripts/build_loongsuite_package.py b/scripts/build_loongsuite_package.py new file mode 100755 index 000000000..924938288 --- /dev/null +++ b/scripts/build_loongsuite_package.py @@ -0,0 +1,675 @@ +#!/usr/bin/env python3 +""" +LoongSuite Release Build Script + +This script supports the following release modes: + +1. --build-pypi: Build packages for PyPI publishing + - loongsuite-util-genai (renamed from opentelemetry-util-genai) + - loongsuite-distro + +2. --build-github-release: Build packages for GitHub Release (tar.gz) + - instrumentation-genai/ packages (renamed to loongsuite-*, depends on loongsuite-util-genai) + - instrumentation-loongsuite/ packages (depends on loongsuite-util-genai) + - processor/loongsuite-processor-baggage/ + +Version replacement: +- --version: Sets version for all packages being built +- --upstream-version: Sets version for upstream opentelemetry-instrumentation-* packages + (used in bootstrap_gen.py) + +Dependency replacement: +- opentelemetry-util-genai -> loongsuite-util-genai (with ~= version spec) + +Package name replacement (for instrumentation-genai/): +- opentelemetry-instrumentation-* -> loongsuite-instrumentation-* +""" + +import argparse +import json +import logging +import re +import subprocess +import sys +import tarfile +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Dict, List, Optional, Set + +import tomlkit + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + + +def load_skip_config(config_path: Path) -> Set[str]: + """Load package names to skip from config file""" + if not config_path.exists(): + return set() + + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + + return set(config.get("skip_packages", [])) + + +def depends_on_util_genai(pyproject_path: Path) -> bool: + """Check if a package depends on opentelemetry-util-genai by reading pyproject.toml.""" + if not pyproject_path.exists(): + return False + + content = pyproject_path.read_text(encoding="utf-8") + return "opentelemetry-util-genai" in content + + +def should_rename_package(package_dir: Path) -> bool: + """ + Determine if a package should be renamed from opentelemetry-* to loongsuite-*. + + Rule: All packages under instrumentation-genai/ with opentelemetry-* prefix + should be renamed to loongsuite-* prefix. + """ + # Check if under instrumentation-genai directory + if "instrumentation-genai" not in str(package_dir): + return False + + # Check if package name starts with opentelemetry- + return package_dir.name.startswith("opentelemetry-") + + +def get_package_name_from_whl(whl_path: Path) -> str: + """Extract package name from whl filename. + + Wheel filename format: {package}-{version}-{python}-{abi}-{platform}.whl + Example: loongsuite_instrumentation_openai_v2-0.1.0-py3-none-any.whl + + Note: Package names may contain version-like parts (e.g., 'v2', 'agents-v2') + that should NOT be treated as version numbers. + """ + name = whl_path.stem + parts = name.split("-") + if len(parts) >= 2: + package_parts = [] + for part in parts: + # Check if this looks like a version number: + # - Starts with digit and contains dot (e.g., "0.1.0", "1.2.3") + # - Or is a known build tag + is_version = ( + (part and part[0].isdigit() and "." in part) # e.g., "0.1.0" + or part in ("dev", "b0", "b1", "rc0", "rc1") + or part.startswith("py") # Python tag: py3, py2 + or part in ("none", "any") # ABI/platform tags + ) + if is_version: + break + package_parts.append(part) + return "-".join(package_parts).replace("_", "-") + return name.replace("_", "-") + + +@contextmanager +def _patch_pyproject(pyproject_path: Path, modifications: Dict[str, Any]): + """ + Temporarily patch pyproject.toml using TOML parsing, restore on exit. + + Args: + pyproject_path: Path to pyproject.toml + modifications: Dict with optional keys: + - "name": New package name (str) + - "replace_dependency": Dict with "old_pattern" and "new_value" + e.g., {"old_pattern": "opentelemetry-util-genai", "new_value": "loongsuite-util-genai ~= 0.1.0"} + """ + original_content = pyproject_path.read_text(encoding="utf-8") + try: + doc = tomlkit.parse(original_content) + + # Modify package name if specified + if "name" in modifications: + doc["project"]["name"] = modifications["name"] + + # Replace dependency if specified + if "replace_dependency" in modifications: + old_pattern = modifications["replace_dependency"]["old_pattern"] + new_value = modifications["replace_dependency"]["new_value"] + + if "dependencies" in doc["project"]: + deps = doc["project"]["dependencies"] + new_deps = [] + for dep in deps: + # Check if this dependency matches the pattern (package name prefix) + # e.g., "opentelemetry-util-genai >= 0.2b0" matches "opentelemetry-util-genai" + dep_str = str(dep).strip() + dep_name = re.split(r"[<>=~!\s\[]", dep_str)[0].strip() + if dep_name == old_pattern: + new_deps.append(new_value) + else: + new_deps.append(dep) + doc["project"]["dependencies"] = new_deps + + pyproject_path.write_text(tomlkit.dumps(doc), encoding="utf-8") + yield + finally: + pyproject_path.write_text(original_content, encoding="utf-8") + + +@contextmanager +def _patch_version_py(version_py_path: Path, new_version: str): + """Temporarily patch version.py, restore on exit.""" + if not version_py_path.exists(): + yield + return + + content = version_py_path.read_text(encoding="utf-8") + try: + patched = re.sub( + r'__version__\s*=\s*["\'][^"\']*["\']', + f'__version__ = "{new_version}"', + content, + ) + version_py_path.write_text(patched, encoding="utf-8") + yield + finally: + version_py_path.write_text(content, encoding="utf-8") + + +def find_version_py(package_dir: Path) -> Optional[Path]: + """Find version.py file in package directory""" + for version_py in package_dir.rglob("version.py"): + if "site-packages" not in str(version_py): + return version_py + return None + + +def build_package( + package_dir: Path, + dist_dir: Path, + existing_whl_files: Set[Path], +) -> List[Path]: + """Build whl file for a single package""" + pyproject_toml = package_dir / "pyproject.toml" + if not pyproject_toml.exists(): + logger.debug(f"Skipping {package_dir}, no pyproject.toml") + return [] + + logger.info(f"Building package: {package_dir}") + try: + before_whl_files = set(dist_dir.glob("*.whl")) + + result = subprocess.run( + [ + sys.executable, + "-m", + "build", + "--wheel", + "--outdir", + str(dist_dir), + ], + cwd=package_dir, + check=True, + capture_output=True, + text=True, + ) + + after_whl_files = set(dist_dir.glob("*.whl")) + new_whl_files = [ + f for f in after_whl_files - before_whl_files if f.suffix == ".whl" + ] + + if not new_whl_files: + logger.warning( + f"No new whl files found after building {package_dir}" + ) + if result.stdout: + logger.debug(f"stdout: {result.stdout}") + if result.stderr: + logger.debug(f"stderr: {result.stderr}") + + return sorted(new_whl_files) + except subprocess.CalledProcessError as e: + logger.error(f"Failed to build {package_dir}: {e}") + if e.stdout: + logger.error(f"stdout: {e.stdout}") + if e.stderr: + logger.error(f"stderr: {e.stderr}") + return [] + + +def build_pypi_packages( + base_dir: Path, + dist_dir: Path, + version: str, + util_genai_version: Optional[str] = None, +) -> List[Path]: + """ + Build packages for PyPI: + - loongsuite-util-genai (renamed from opentelemetry-util-genai) + - loongsuite-distro + """ + all_whl_files = [] + existing_whl_files = set(dist_dir.glob("*.whl")) + + util_ver = util_genai_version or version + + # 1. Build util/opentelemetry-util-genai as loongsuite-util-genai + util_genai_dir = base_dir / "util" / "opentelemetry-util-genai" + if ( + util_genai_dir.exists() + and (util_genai_dir / "pyproject.toml").exists() + ): + logger.info(f"Building loongsuite-util-genai (version {util_ver})...") + version_py = find_version_py(util_genai_dir) + + modifications = { + "name": "loongsuite-util-genai", + } + + with _patch_pyproject( + util_genai_dir / "pyproject.toml", modifications + ): + with ( + _patch_version_py(version_py, util_ver) + if version_py + else nullcontext() + ): + whl_files = build_package( + util_genai_dir, dist_dir, existing_whl_files + ) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + # 2. Build loongsuite-distro + distro_dir = base_dir / "loongsuite-distro" + if distro_dir.exists() and (distro_dir / "pyproject.toml").exists(): + logger.info(f"Building loongsuite-distro (version {version})...") + version_py = find_version_py(distro_dir) + + with ( + _patch_version_py(version_py, version) + if version_py + else nullcontext() + ): + whl_files = build_package(distro_dir, dist_dir, existing_whl_files) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + return all_whl_files + + +def build_github_release_packages( + base_dir: Path, + dist_dir: Path, + version: str, + util_genai_version: Optional[str] = None, + skip_packages: Optional[Set[str]] = None, +) -> List[Path]: + """ + Build packages for GitHub Release (tar.gz): + - instrumentation-genai/ (renamed to loongsuite-*, depends on loongsuite-util-genai) + - instrumentation-loongsuite/ (depends on loongsuite-util-genai) + - processor/loongsuite-processor-baggage/ + """ + all_whl_files = [] + existing_whl_files = set(dist_dir.glob("*.whl")) + skip_packages = skip_packages or set() + + util_ver = util_genai_version or version + util_dep_spec = f"loongsuite-util-genai ~= {util_ver}" + + def _get_modifications(package_dir: Path) -> Dict[str, Any]: + """ + Get pyproject.toml modifications for a package based on rules: + + Rules: + 1. Dependency replacement: If package depends on opentelemetry-util-genai, + replace it with loongsuite-util-genai (detected by reading pyproject.toml) + 2. Name replacement: If package is under instrumentation-genai/ and has + opentelemetry-* prefix, rename to loongsuite-* prefix + + Returns: + Dict with modifications to apply, e.g.: + { + "name": "loongsuite-instrumentation-foo", + "replace_dependency": { + "old_pattern": "opentelemetry-util-genai", + "new_value": "loongsuite-util-genai ~= 0.1.0" + } + } + """ + modifications: Dict[str, Any] = {} + pyproject_path = package_dir / "pyproject.toml" + + # Rule 1: Dependency replacement (dynamic detection) + # Replace any version of opentelemetry-util-genai with loongsuite-util-genai + if depends_on_util_genai(pyproject_path): + modifications["replace_dependency"] = { + "old_pattern": "opentelemetry-util-genai", + "new_value": util_dep_spec, + } + + # Rule 2: Name replacement (instrumentation-genai/ packages with opentelemetry-* prefix) + if should_rename_package(package_dir): + pkg_name = package_dir.name + new_name = pkg_name.replace("opentelemetry-", "loongsuite-") + modifications["name"] = new_name + + return modifications + + # 1. Build instrumentation-genai/ packages + instrumentation_genai_dir = base_dir / "instrumentation-genai" + if instrumentation_genai_dir.exists(): + logger.info("Building packages under instrumentation-genai/...") + for package_dir in sorted(instrumentation_genai_dir.iterdir()): + if ( + not package_dir.is_dir() + or not (package_dir / "pyproject.toml").exists() + ): + continue + + pkg_name = package_dir.name + if pkg_name in skip_packages: + logger.info(f"Skipping {pkg_name} (in skip list)") + continue + + modifications = _get_modifications(package_dir) + version_py = find_version_py(package_dir) + + logger.info(f"Building {pkg_name} (version {version})...") + with ( + _patch_pyproject(package_dir / "pyproject.toml", modifications) + if modifications + else nullcontext() + ): + with ( + _patch_version_py(version_py, version) + if version_py + else nullcontext() + ): + whl_files = build_package( + package_dir, dist_dir, existing_whl_files + ) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + # 2. Build instrumentation-loongsuite/ packages + instrumentation_loongsuite_dir = base_dir / "instrumentation-loongsuite" + if instrumentation_loongsuite_dir.exists(): + logger.info("Building packages under instrumentation-loongsuite/...") + for package_dir in sorted(instrumentation_loongsuite_dir.iterdir()): + if ( + not package_dir.is_dir() + or not (package_dir / "pyproject.toml").exists() + ): + continue + + pkg_name = package_dir.name + if pkg_name in skip_packages: + logger.info(f"Skipping {pkg_name} (in skip list)") + continue + + modifications = _get_modifications(package_dir) + version_py = find_version_py(package_dir) + + logger.info(f"Building {pkg_name} (version {version})...") + with ( + _patch_pyproject(package_dir / "pyproject.toml", modifications) + if modifications + else nullcontext() + ): + with ( + _patch_version_py(version_py, version) + if version_py + else nullcontext() + ): + whl_files = build_package( + package_dir, dist_dir, existing_whl_files + ) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + # 3. Build processor/loongsuite-processor-baggage/ + processor_baggage_dir = ( + base_dir / "processor" / "loongsuite-processor-baggage" + ) + if ( + processor_baggage_dir.exists() + and (processor_baggage_dir / "pyproject.toml").exists() + ): + pkg_name = processor_baggage_dir.name + if pkg_name not in skip_packages: + version_py = find_version_py(processor_baggage_dir) + logger.info(f"Building {pkg_name} (version {version})...") + with ( + _patch_version_py(version_py, version) + if version_py + else nullcontext() + ): + whl_files = build_package( + processor_baggage_dir, dist_dir, existing_whl_files + ) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + return all_whl_files + + +def _filter_and_dedupe_whl_files( + all_whl_files: List[Path], + skip_packages: Set[str], +) -> List[Path]: + """Filter skip list and deduplicate whl files.""" + filtered_whl_files = [] + skipped_count = 0 + seen_packages = {} + + for whl_file in all_whl_files: + package_name = get_package_name_from_whl(whl_file) + + if package_name in skip_packages: + logger.info(f"Skipping package: {package_name} (in skip list)") + skipped_count += 1 + whl_file.unlink() + continue + + if package_name in seen_packages: + existing_file = seen_packages[package_name] + if whl_file.stat().st_mtime > existing_file.stat().st_mtime: + logger.debug(f"Replacing duplicate {package_name}") + existing_file.unlink() + seen_packages[package_name] = whl_file + filtered_whl_files.remove(existing_file) + filtered_whl_files.append(whl_file) + else: + whl_file.unlink() + else: + seen_packages[package_name] = whl_file + filtered_whl_files.append(whl_file) + + logger.info(f"Built {len(all_whl_files)} whl files") + logger.info(f"Skipped {skipped_count} packages") + logger.info(f"Final: {len(filtered_whl_files)} whl files") + + return filtered_whl_files + + +def create_tar_archive(whl_files: List[Path], output_path: Path): + """Package all whl files into tar.gz""" + logger.info(f"Creating tar archive: {output_path}") + + with tarfile.open(output_path, "w:gz") as tar: + for whl_file in sorted(whl_files): + tar.add(whl_file, arcname=whl_file.name) + logger.debug(f"Added: {whl_file.name}") + + size_mb = output_path.stat().st_size / 1024 / 1024 + logger.info(f"Created: {output_path} ({size_mb:.2f} MB)") + + +# Python 3.9 compatibility: nullcontext +try: + from contextlib import nullcontext +except ImportError: + from contextlib import contextmanager + + @contextmanager + def nullcontext(): + yield + + +def main(): + parser = argparse.ArgumentParser( + description="LoongSuite Release Build Script", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Build for PyPI (loongsuite-util-genai + loongsuite-distro) + python build_loongsuite_package.py --build-pypi --version 0.1.0 + + # Build for GitHub Release (instrumentation packages) + python build_loongsuite_package.py --build-github-release --version 0.1.0 + + # Build both + python build_loongsuite_package.py --build-pypi --build-github-release --version 0.1.0 + """, + ) + + parser.add_argument( + "--base-dir", + type=Path, + default=Path(__file__).parent.parent, + help="Project root directory", + ) + parser.add_argument( + "--dist-dir", + type=Path, + default=None, + help="Build output directory (default: base-dir/dist)", + ) + parser.add_argument( + "--config", + type=Path, + default=Path(__file__).parent / "loongsuite-build-config.json", + help="Config file for skip packages", + ) + + # Build mode + parser.add_argument( + "--build-pypi", + action="store_true", + help="Build packages for PyPI (loongsuite-util-genai, loongsuite-distro)", + ) + parser.add_argument( + "--build-github-release", + action="store_true", + help="Build packages for GitHub Release (instrumentation-genai, instrumentation-loongsuite)", + ) + + # Legacy mode (for backward compatibility) + parser.add_argument( + "--loongsuite-release", + action="store_true", + help="(Legacy) Same as --build-github-release", + ) + + # Version settings + parser.add_argument( + "--version", + type=str, + required=True, + help="Version for all packages", + ) + parser.add_argument( + "--util-genai-version", + type=str, + default=None, + help="Version for loongsuite-util-genai (default: same as --version)", + ) + + # Output + parser.add_argument( + "--output", + type=Path, + default=None, + help="Output tar.gz path (for GitHub Release)", + ) + + args = parser.parse_args() + + base_dir = args.base_dir.resolve() + dist_dir = args.dist_dir or (base_dir / "dist") + dist_dir.mkdir(parents=True, exist_ok=True) + + # Clean old whl files + logger.info(f"Cleaning old build files: {dist_dir}") + for old_file in dist_dir.glob("*.whl"): + old_file.unlink() + + skip_packages = load_skip_config(args.config) + + # Handle legacy mode + if args.loongsuite_release: + args.build_github_release = True + + if not args.build_pypi and not args.build_github_release: + parser.error( + "Must specify at least one of --build-pypi or --build-github-release" + ) + + pypi_whl_files = [] + github_whl_files = [] + + # Build PyPI packages + if args.build_pypi: + logger.info("=" * 50) + logger.info("Building PyPI packages...") + logger.info("=" * 50) + pypi_whl_files = build_pypi_packages( + base_dir, + dist_dir, + args.version, + args.util_genai_version, + ) + logger.info(f"PyPI packages built: {len(pypi_whl_files)}") + for whl in pypi_whl_files: + logger.info(f" - {whl.name}") + + # Build GitHub Release packages + if args.build_github_release: + logger.info("=" * 50) + logger.info("Building GitHub Release packages...") + logger.info("=" * 50) + github_whl_files = build_github_release_packages( + base_dir, + dist_dir, + args.version, + args.util_genai_version, + skip_packages, + ) + + github_whl_files = _filter_and_dedupe_whl_files( + github_whl_files, skip_packages + ) + + if github_whl_files: + output_path = args.output or ( + dist_dir / f"loongsuite-python-agent-{args.version}.tar.gz" + ) + create_tar_archive(github_whl_files, output_path) + logger.info(f"GitHub Release tar: {output_path}") + + logger.info("=" * 50) + logger.info("Build completed!") + logger.info("=" * 50) + + if pypi_whl_files: + logger.info("PyPI packages (upload with twine):") + for whl in pypi_whl_files: + logger.info(f" {whl}") + + if github_whl_files: + logger.info( + f"GitHub Release tar ready: {dist_dir}/loongsuite-python-agent-{args.version}.tar.gz" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/dry_run_loongsuite_release.sh b/scripts/dry_run_loongsuite_release.sh new file mode 100755 index 000000000..5e200b439 --- /dev/null +++ b/scripts/dry_run_loongsuite_release.sh @@ -0,0 +1,348 @@ +#!/usr/bin/env bash +# +# LoongSuite Release Dry Run Script +# +# Simulates the GitHub Actions loongsuite-release workflow locally to verify: +# 1. bootstrap_gen.py generation with version overrides +# 2. PyPI package build (loongsuite-util-genai, loongsuite-distro) +# 3. GitHub Release package build (instrumentation-genai, instrumentation-loongsuite) +# 4. Package content verification +# 5. Optional: Installation test in temporary venv +# +# Usage: +# ./scripts/dry_run_loongsuite_release.sh --loongsuite-version 0.1.0 --upstream-version 0.60b1 +# ./scripts/dry_run_loongsuite_release.sh -l 0.1.0 -u 0.60b1 --skip-install +# ./scripts/dry_run_loongsuite_release.sh -l 0.1.0 -u 0.60b1 --skip-pypi +# +set -e + +# Default values +LOONGSUITE_VERSION="" +UPSTREAM_VERSION="" +SKIP_INSTALL=false +SKIP_PYPI=false + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + -l|--loongsuite-version) + LOONGSUITE_VERSION="$2" + shift 2 + ;; + -u|--upstream-version) + UPSTREAM_VERSION="$2" + shift 2 + ;; + --skip-install) + SKIP_INSTALL=true + shift + ;; + --skip-pypi) + SKIP_PYPI=true + shift + ;; + -h|--help) + echo "Usage: $0 --loongsuite-version --upstream-version [options]" + echo "" + echo "Required:" + echo " -l, --loongsuite-version Version for loongsuite-* packages (e.g., 0.1.0)" + echo " -u, --upstream-version Version for opentelemetry-* packages (e.g., 0.60b1)" + echo "" + echo "Options:" + echo " --skip-install Skip installation verification" + echo " --skip-pypi Skip PyPI package build" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Validate required arguments +if [[ -z "$LOONGSUITE_VERSION" ]]; then + echo "ERROR: --loongsuite-version is required" + exit 1 +fi +if [[ -z "$UPSTREAM_VERSION" ]]; then + echo "ERROR: --upstream-version is required" + exit 1 +fi + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +TAR_NAME="loongsuite-python-agent-${LOONGSUITE_VERSION}.tar.gz" +TAR_PATH="${REPO_ROOT}/dist/${TAR_NAME}" +RELEASE_NOTES_FILE="${REPO_ROOT}/dist/release-notes-dryrun.txt" +DRYRUN_VENV="${REPO_ROOT}/.venv_loongsuite_dryrun" + +echo "==========================================" +echo "LoongSuite Release Dry Run" +echo "==========================================" +echo "LoongSuite version: $LOONGSUITE_VERSION" +echo "Upstream version: $UPSTREAM_VERSION" +echo "Repo root: $REPO_ROOT" +echo "" + +# Step 1: Install build dependencies +echo ">>> Step 1: Installing build dependencies..." +python -m pip install -q -r pkg-requirements.txt 2>/dev/null || { + echo " Installing dependencies from pkg-requirements.txt..." + python -m pip install -r pkg-requirements.txt +} +echo " OK" +echo "" + +# Step 2: Generate bootstrap_gen.py +echo ">>> Step 2: Generating bootstrap_gen.py..." +python scripts/generate_loongsuite_bootstrap.py \ + --upstream-version "$UPSTREAM_VERSION" \ + --loongsuite-version "$LOONGSUITE_VERSION" + +echo " OK: Generated bootstrap_gen.py" +echo " Preview (first 20 lines):" +head -20 loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py | sed 's/^/ /' +echo "" + +# Step 3: Build PyPI packages +PYPI_DIST_DIR="${REPO_ROOT}/dist-pypi" +rm -rf "$PYPI_DIST_DIR" +mkdir -p "$PYPI_DIST_DIR" + +if [[ "$SKIP_PYPI" != "true" ]]; then + echo ">>> Step 3: Building PyPI packages..." + python scripts/build_loongsuite_package.py \ + --build-pypi \ + --version "$LOONGSUITE_VERSION" + + # Save PyPI packages to separate directory (before step 4 cleans dist/) + cp dist/*.whl "$PYPI_DIST_DIR/" 2>/dev/null || true + + echo " OK: PyPI packages built" + echo " Packages:" + ls "$PYPI_DIST_DIR"/*.whl 2>/dev/null | while read f; do echo " - $(basename "$f")"; done + echo "" +else + echo ">>> Step 3: Skipped (--skip-pypi)" + echo "" +fi + +# Step 4: Build GitHub Release packages +echo ">>> Step 4: Building GitHub Release packages..." +python scripts/build_loongsuite_package.py \ + --build-github-release \ + --version "$LOONGSUITE_VERSION" + +if [[ ! -f "$TAR_PATH" ]]; then + echo " ERROR: Build failed, $TAR_PATH not found" + exit 1 +fi +echo " OK: $TAR_PATH ($(du -h "$TAR_PATH" | cut -f1))" +echo "" + +# Step 5: Verify tar contents +echo ">>> Step 5: Verifying tar contents..." + +# Check for loongsuite-util-genai (should NOT be in tar, it's on PyPI) +if tar -tzf "$TAR_PATH" | grep -q "loongsuite_util_genai"; then + echo " WARN: loongsuite-util-genai in tar (should be on PyPI only)" +else + echo " OK: loongsuite-util-genai not in tar (correct, on PyPI)" +fi + +# Check for opentelemetry-util-genai (should NOT be in tar) +if tar -tzf "$TAR_PATH" | grep -q "opentelemetry_util_genai"; then + echo " ERROR: opentelemetry-util-genai should NOT be in tar" + exit 1 +else + echo " OK: opentelemetry-util-genai not in tar" +fi + +# Check for loongsuite-instrumentation-* packages +if tar -tzf "$TAR_PATH" | grep -q "loongsuite_instrumentation"; then + echo " OK: loongsuite-instrumentation-* packages in tar" +else + echo " WARN: No loongsuite-instrumentation-* packages found in tar" +fi + +# Check that opentelemetry-instrumentation-flask is NOT in tar +if tar -tzf "$TAR_PATH" | grep -q "opentelemetry_instrumentation_flask"; then + echo " WARN: opentelemetry-instrumentation-flask in tar (should be from PyPI)" +else + echo " OK: opentelemetry-instrumentation-flask not in tar (from PyPI)" +fi + +echo " Package count: $(tar -tzf "$TAR_PATH" | wc -l | tr -d ' ')" +echo " Contents:" +tar -tzf "$TAR_PATH" | head -20 | sed 's/^/ /' +echo "" + +# Step 6: Generate release notes +echo ">>> Step 6: Generating release notes..." + +# Start with header +cat > "$RELEASE_NOTES_FILE" << EOF +# LoongSuite Python Agent v$LOONGSUITE_VERSION + +## Installation + +\`\`\`bash +pip install loongsuite-distro==$LOONGSUITE_VERSION +loongsuite-bootstrap -a install --version $LOONGSUITE_VERSION +\`\`\` + +## Package Versions + +- loongsuite-* packages: $LOONGSUITE_VERSION +- opentelemetry-* packages: $UPSTREAM_VERSION + +--- + +EOF + +# Collect from root CHANGELOG-loongsuite.md +if [[ -f CHANGELOG-loongsuite.md ]]; then + echo "## loongsuite-distro" >> "$RELEASE_NOTES_FILE" + echo "" >> "$RELEASE_NOTES_FILE" + # Extract Unreleased section (handle both "## Unreleased" and "## [Unreleased]") + sed -n '/^## \[*Unreleased\]*$/,/^## /p' CHANGELOG-loongsuite.md | sed '/^## /d' >> "$RELEASE_NOTES_FILE" + echo "" >> "$RELEASE_NOTES_FILE" +fi + +# Collect from instrumentation-loongsuite/*/CHANGELOG.md +for changelog in instrumentation-loongsuite/*/CHANGELOG.md; do + if [[ -f "$changelog" ]]; then + # Extract package name from path + pkg_dir=$(dirname "$changelog") + pkg_name=$(basename "$pkg_dir") + + # Extract Unreleased section + unreleased_content=$(sed -n '/^## \[*Unreleased\]*$/,/^## /p' "$changelog" | sed '/^## /d') + + if [[ -n "$unreleased_content" && "$unreleased_content" =~ [^[:space:]] ]]; then + echo "## $pkg_name" >> "$RELEASE_NOTES_FILE" + echo "" >> "$RELEASE_NOTES_FILE" + echo "$unreleased_content" >> "$RELEASE_NOTES_FILE" + echo "" >> "$RELEASE_NOTES_FILE" + fi + fi +done + +echo " OK: $RELEASE_NOTES_FILE" +echo " Preview:" +head -30 "$RELEASE_NOTES_FILE" | sed 's/^/ /' +echo "" + +# Step 7: Install verification (optional) +if [[ "$SKIP_INSTALL" == "true" ]]; then + echo ">>> Step 7: Skipped (--skip-install)" +else + echo ">>> Step 7: Install verification (temp venv)..." + rm -rf "$DRYRUN_VENV" + python -m venv "$DRYRUN_VENV" + source "$DRYRUN_VENV/bin/activate" + + # Install loongsuite-distro from local (has loongsuite-bootstrap) + echo " Installing loongsuite-distro from local..." + pip install -q -e ./loongsuite-distro + + # Pre-install loongsuite-util-genai from local build (simulating PyPI) + # In production, this is installed as a transitive dependency of instrumentation packages + # In dry run, we need to install it first because it's not yet on PyPI + if [[ "$SKIP_PYPI" != "true" ]]; then + UTIL_WHL=$(ls "$PYPI_DIST_DIR"/loongsuite_util_genai-*.whl 2>/dev/null | head -1) + if [[ -n "$UTIL_WHL" ]]; then + echo " Pre-installing loongsuite-util-genai from local build (simulating PyPI)..." + pip install -q "$UTIL_WHL" + else + echo " ERROR: loongsuite-util-genai wheel not found in $PYPI_DIST_DIR" + echo " Cannot proceed - instrumentation packages depend on it" + deactivate + rm -rf "$DRYRUN_VENV" + exit 1 + fi + else + echo " ERROR: PyPI build skipped, loongsuite-util-genai not available" + echo " Cannot proceed - instrumentation packages depend on it" + deactivate + rm -rf "$DRYRUN_VENV" + exit 1 + fi + + # Create whitelist for minimal test + WHITELIST_FILE=$(mktemp) + cat > "$WHITELIST_FILE" << 'WL' +loongsuite-instrumentation-dashscope +WL + + echo " Running: loongsuite-bootstrap -a install --tar $TAR_PATH --whitelist $WHITELIST_FILE" + if loongsuite-bootstrap -a install --tar "$TAR_PATH" --whitelist "$WHITELIST_FILE" 2>&1; then + echo "" + echo " Verifying installed packages..." + + # Check loongsuite-util-genai (from PyPI/local build) + if pip show loongsuite-util-genai &>/dev/null; then + echo " OK: loongsuite-util-genai installed ($(pip show loongsuite-util-genai | grep Version:))" + else + echo " WARN: loongsuite-util-genai not installed" + fi + + # Check opentelemetry-util-genai should NOT be installed + if pip show opentelemetry-util-genai &>/dev/null; then + echo " WARN: opentelemetry-util-genai installed (may conflict)" + else + echo " OK: opentelemetry-util-genai not installed (correct)" + fi + + # Check loongsuite-instrumentation-dashscope + if pip show loongsuite-instrumentation-dashscope &>/dev/null; then + echo " OK: loongsuite-instrumentation-dashscope installed" + else + echo " WARN: loongsuite-instrumentation-dashscope not installed" + fi + + rm -f "$WHITELIST_FILE" + deactivate + rm -rf "$DRYRUN_VENV" + echo " OK: Install verification passed" + else + echo " ERROR: loongsuite-bootstrap install failed" + rm -f "$WHITELIST_FILE" + deactivate + rm -rf "$DRYRUN_VENV" + exit 1 + fi +fi +echo "" + +echo "==========================================" +echo "Dry Run Complete" +echo "==========================================" +echo "" +echo "Artifacts:" +if [[ "$SKIP_PYPI" != "true" ]]; then + echo " PyPI packages (in $PYPI_DIST_DIR):" + ls "$PYPI_DIST_DIR"/*.whl 2>/dev/null | while read f; do echo " - $f"; done +fi +echo " GitHub Release:" +echo " - $TAR_PATH" +echo " Release notes:" +echo " - $RELEASE_NOTES_FILE" +echo "" +echo "Simulated GitHub release commands:" +echo "" +echo " # PyPI publish (with twine):" +if [[ "$SKIP_PYPI" != "true" ]]; then + echo " twine upload $PYPI_DIST_DIR/loongsuite_util_genai-${LOONGSUITE_VERSION}-*.whl" + echo " twine upload $PYPI_DIST_DIR/loongsuite_distro-${LOONGSUITE_VERSION}-*.whl" +fi +echo "" +echo " # GitHub Release:" +echo " gh release create v$LOONGSUITE_VERSION \\" +echo " --title \"LoongSuite Python Agent v$LOONGSUITE_VERSION\" \\" +echo " --notes-file $RELEASE_NOTES_FILE \\" +echo " $TAR_PATH" +echo "" diff --git a/scripts/generate_loongsuite_bootstrap.py b/scripts/generate_loongsuite_bootstrap.py new file mode 100755 index 000000000..ea655d90d --- /dev/null +++ b/scripts/generate_loongsuite_bootstrap.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 + +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Generate bootstrap_gen.py for loongsuite-distro. + +This script generates the libraries and default_instrumentations lists +used by loongsuite-bootstrap to install instrumentations. + +Package naming and version strategy: +- instrumentation-genai/* packages: renamed to loongsuite-* prefix +- instrumentation-loongsuite/* packages: keep loongsuite-* prefix +- instrumentation/* packages: keep opentelemetry-* prefix (from upstream PyPI) + +Version strategy: +- --upstream-version: Version for upstream opentelemetry-instrumentation-* packages +- --loongsuite-version: Version for loongsuite-* packages + +Usage: + # Generate with default versions from source + python scripts/generate_loongsuite_bootstrap.py + + # Generate with specific versions for release + python scripts/generate_loongsuite_bootstrap.py \\ + --upstream-version 0.60b1 \\ + --loongsuite-version 0.1.0 +""" + +import argparse +import ast +import logging +import subprocess +from pathlib import Path +from typing import Optional + +import tomli +from generate_instrumentation_bootstrap import ( + independent_packages, + packages_to_exclude, +) +from otel_packaging import ( + get_instrumentation_packages as get_upstream_packages, +) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("loongsuite_bootstrap_generator") + +# Get root path +scripts_path = Path(__file__).parent +root_path = scripts_path.parent + +_template = """ +{header} + +# DO NOT EDIT. THIS FILE WAS AUTOGENERATED FROM INSTRUMENTATION PACKAGES. +# RUN `python scripts/generate_loongsuite_bootstrap.py` TO REGENERATE. +# +# Generated with options: +# --upstream-version: {upstream_version} +# --loongsuite-version: {loongsuite_version} + +{source} +""" + +gen_path = ( + root_path + / "loongsuite-distro" + / "src" + / "loongsuite" + / "distro" + / "bootstrap_gen.py" +) + + +def get_genai_packages_to_rename() -> dict[str, str]: + """ + Dynamically discover instrumentation-genai packages that need renaming. + + Rule: All packages under instrumentation-genai/ with opentelemetry-* prefix + should be renamed to loongsuite-* prefix, except those in packages_to_exclude. + + Returns: + Dict mapping original name to new name, e.g.: + {"opentelemetry-instrumentation-vertexai": "loongsuite-instrumentation-vertexai"} + """ + result = {} + genai_dir = root_path / "instrumentation-genai" + + if not genai_dir.exists(): + return result + + for pkg_dir in genai_dir.iterdir(): + if not pkg_dir.is_dir(): + continue + + pkg_name = pkg_dir.name + + # Rule: opentelemetry-* prefix packages should be renamed to loongsuite-* + if pkg_name.startswith("opentelemetry-"): + # Skip packages in upstream exclude list + if pkg_name in packages_to_exclude: + continue + + new_name = pkg_name.replace("opentelemetry-", "loongsuite-") + result[pkg_name] = new_name + + return result + + +def get_instrumentation_packages( + upstream_version: Optional[str] = None, + loongsuite_version: Optional[str] = None, +): + """ + Get all instrumentation packages from various directories. + + Args: + upstream_version: Override version for upstream opentelemetry-* packages + loongsuite_version: Override version for loongsuite-* packages + + Note: + When the same package name exists in both instrumentation-genai and + instrumentation-loongsuite, the version from instrumentation-loongsuite + takes precedence (overwrites the earlier one). + """ + packages = [] + seen_packages: dict[str, int] = {} # name -> index in packages list + + logger.info("Scanning instrumentation packages...") + + # Dynamically get packages that need renaming + genai_packages_to_rename = get_genai_packages_to_rename() + + # Get packages from upstream directories (instrumentation, instrumentation-genai) + logger.info( + "Processing upstream packages (instrumentation, instrumentation-genai)..." + ) + for pkg in get_upstream_packages( + independent_packages=independent_packages + ): + pkg_name = pkg["name"] + + # Skip packages in exclude list (follow upstream behavior) + if pkg_name in packages_to_exclude: + continue + + # Check if this package should be renamed (instrumentation-genai with opentelemetry-* prefix) + if pkg_name in genai_packages_to_rename: + new_name = genai_packages_to_rename[pkg_name] + logger.info(f"Renaming {pkg_name} -> {new_name}") + pkg["name"] = new_name + + # Use loongsuite version for renamed packages + if loongsuite_version: + pkg["requirement"] = f"{new_name}=={loongsuite_version}" + else: + # Keep original version but update name + pkg["requirement"] = pkg["requirement"].replace( + pkg_name, new_name + ) + else: + # Use upstream version for opentelemetry-* packages + if upstream_version and pkg_name.startswith("opentelemetry-"): + pkg["requirement"] = f"{pkg_name}=={upstream_version}" + + # Track package by name (for deduplication) + final_name = pkg["name"] + if final_name in seen_packages: + # Replace existing package (instrumentation-loongsuite should win) + idx = seen_packages[final_name] + packages[idx] = pkg + else: + seen_packages[final_name] = len(packages) + packages.append(pkg) + + # Scan instrumentation-loongsuite directory + loongsuite_dir = root_path / "instrumentation-loongsuite" + if loongsuite_dir.exists(): + logger.info( + "Processing loongsuite packages (instrumentation-loongsuite)..." + ) + pkg_dirs = sorted([d for d in loongsuite_dir.iterdir() if d.is_dir()]) + for pkg_dir in pkg_dirs: + pyproject_toml = pkg_dir / "pyproject.toml" + if not pyproject_toml.exists(): + continue + + try: + # Get version using hatch command + version = subprocess.check_output( + "hatch version", + shell=True, + cwd=pkg_dir, + universal_newlines=True, + stderr=subprocess.DEVNULL, + ).strip() + + # Read pyproject.toml + with open(pyproject_toml, "rb") as f: + pyproject = tomli.load(f) + + pkg_name = pyproject["project"]["name"] + + if pkg_name in packages_to_exclude: + continue + + # Get optional dependencies + optional_deps = pyproject["project"].get( + "optional-dependencies", {} + ) + instruments = optional_deps.get("instruments", []) + instruments_any = optional_deps.get("instruments-any", []) + + # Use loongsuite version if specified + if loongsuite_version: + requirement = f"{pkg_name}=={loongsuite_version}" + elif pkg_name in independent_packages: + specifier = independent_packages[pkg_name] + requirement = ( + f"{pkg_name}{specifier}" + if specifier + else f"{pkg_name}=={version}" + ) + else: + requirement = f"{pkg_name}=={version}" + + new_pkg = { + "name": pkg_name, + "version": version, + "instruments": instruments, + "instruments-any": instruments_any, + "requirement": requirement, + } + + # Deduplication: instrumentation-loongsuite takes precedence + if pkg_name in seen_packages: + idx = seen_packages[pkg_name] + logger.info( + f"Overwriting {pkg_name} with instrumentation-loongsuite version" + ) + packages[idx] = new_pkg + else: + seen_packages[pkg_name] = len(packages) + packages.append(new_pkg) + except subprocess.CalledProcessError as e: + logger.warning( + f"Could not get hatch version from {pkg_dir.name}: {e}" + ) + continue + except Exception as e: + logger.warning(f"Failed to process {pkg_dir.name}: {e}") + continue + + return packages + + +def main(): + parser = argparse.ArgumentParser( + description="Generate bootstrap_gen.py for loongsuite-distro" + ) + parser.add_argument( + "--upstream-version", + type=str, + default=None, + help="Override version for upstream opentelemetry-instrumentation-* packages (e.g., 0.60b1)", + ) + parser.add_argument( + "--loongsuite-version", + type=str, + default=None, + help="Override version for loongsuite-* packages (e.g., 0.1.0)", + ) + args = parser.parse_args() + + # Read license header + header_path = scripts_path / "license_header.txt" + if header_path.exists(): + with open(header_path, "r", encoding="utf-8") as f: + header = f.read() + else: + header = "# Copyright The OpenTelemetry Authors\n# Licensed under the Apache License, Version 2.0\n" + + # Get all packages + packages = get_instrumentation_packages( + upstream_version=args.upstream_version, + loongsuite_version=args.loongsuite_version, + ) + logger.info(f"Found {len(packages)} instrumentation packages") + + # Build AST nodes + default_instrumentations = ast.List(elts=[]) + libraries = ast.List(elts=[]) + + logger.info("Building bootstrap configuration...") + for pkg in packages: + # If no instruments and no instruments-any, it's a default instrumentation + if not pkg.get("instruments") and not pkg.get("instruments-any"): + default_instrumentations.elts.append( + ast.Constant(value=pkg["requirement"]) + ) + else: + # Add instruments (all must be installed) + for target_pkg in pkg.get("instruments", []): + libraries.elts.append( + ast.Dict( + keys=[ + ast.Constant(value="library"), + ast.Constant(value="instrumentation"), + ], + values=[ + ast.Constant(value=target_pkg), + ast.Constant(value=pkg["requirement"]), + ], + ) + ) + + # Add instruments-any (at least one must be installed) + for target_pkg in pkg.get("instruments-any", []): + libraries.elts.append( + ast.Dict( + keys=[ + ast.Constant(value="library"), + ast.Constant(value="instrumentation"), + ], + values=[ + ast.Constant(value=target_pkg), + ast.Constant(value=pkg["requirement"]), + ], + ) + ) + + # Generate source code + logger.info("Generating source code...") + # Build libraries list string + libraries_lines = ["libraries = ["] + for lib_mapping in libraries.elts: + if isinstance(lib_mapping, ast.Dict): + lib_key = lib_mapping.keys[0] + instr_key = lib_mapping.keys[1] + lib_val_node = lib_mapping.values[0] + instr_val_node = lib_mapping.values[1] + + lib_key_str = ( + lib_key.value + if isinstance(lib_key, ast.Constant) + else (lib_key.s if hasattr(lib_key, "s") else "library") + ) + instr_key_str = ( + instr_key.value + if isinstance(instr_key, ast.Constant) + else ( + instr_key.s + if hasattr(instr_key, "s") + else "instrumentation" + ) + ) + lib_val_str = ( + lib_val_node.value + if isinstance(lib_val_node, ast.Constant) + else (lib_val_node.s if hasattr(lib_val_node, "s") else "") + ) + instr_val_str = ( + instr_val_node.value + if isinstance(instr_val_node, ast.Constant) + else (instr_val_node.s if hasattr(instr_val_node, "s") else "") + ) + + lib_val_str = lib_val_str.replace('"', '\\"') + instr_val_str = instr_val_str.replace('"', '\\"') + + libraries_lines.append( + f' {{"{lib_key_str}": "{lib_val_str}", "{instr_key_str}": "{instr_val_str}"}},' + ) + libraries_lines.append("]") + + # Build default_instrumentations list string + default_lines = ["default_instrumentations = ["] + for default_instr in default_instrumentations.elts: + if isinstance(default_instr, ast.Constant): + instr_val = default_instr.value.replace('"', '\\"') + default_lines.append(f' "{instr_val}",') + default_lines.append("]") + + # Combine source + source = "\n".join(libraries_lines) + "\n\n" + "\n".join(default_lines) + + # Format with header and version info + formatted_source = _template.format( + header=header, + source=source, + upstream_version=args.upstream_version or "(from source)", + loongsuite_version=args.loongsuite_version or "(from source)", + ) + + # Write to file + gen_path.parent.mkdir(parents=True, exist_ok=True) + with open(gen_path, "w", encoding="utf-8") as f: + f.write(formatted_source) + + logger.info("generated %s", gen_path) + logger.info( + " - %d default instrumentations", len(default_instrumentations.elts) + ) + logger.info(" - %d library mappings", len(libraries.elts)) + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_loongsuite_readme.py b/scripts/generate_loongsuite_readme.py new file mode 100755 index 000000000..62cf70549 --- /dev/null +++ b/scripts/generate_loongsuite_readme.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import logging +import os +from pathlib import Path + +import tomli + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("loongsuite_readme_generator") + +_prefix = "loongsuite-instrumentation-" + +header = """ +| Instrumentation | Supported Packages | Metrics support | Semconv status | +| --------------- | ------------------ | --------------- | -------------- |""" + + +def main(base_instrumentation_path): + table = [header] + for instrumentation in sorted(os.listdir(base_instrumentation_path)): + instrumentation_path = os.path.join( + base_instrumentation_path, instrumentation + ) + if not os.path.isdir( + instrumentation_path + ) or not instrumentation.startswith(_prefix): + continue + + pyproject_toml = Path(instrumentation_path) / "pyproject.toml" + if not pyproject_toml.exists(): + continue + + try: + with open(pyproject_toml, "rb") as f: + pyproject = tomli.load(f) + + project = pyproject.get("project", {}) + optional_deps = project.get("optional-dependencies", {}) + instruments = optional_deps.get("instruments", []) + instruments_any = optional_deps.get("instruments-any", []) + + # Extract package name from instrumentation directory name + # e.g., "loongsuite-instrumentation-agentscope" -> "agentscope" + name = instrumentation.replace(_prefix, "") + + instruments_all = () + if not instruments and not instruments_any: + instruments_all = (name,) + else: + instruments_all = tuple(instruments + instruments_any) + + # Try to get metrics support and semconv status from pyproject.toml + # These might not be present, so use defaults + supports_metrics = project.get("supports_metrics", False) + semconv_status = project.get("semconv_status", "development") + + metric_column = "Yes" if supports_metrics else "No" + + table.append( + f"| [{instrumentation}](./{instrumentation}) | {','.join(instruments_all)} | {metric_column} | {semconv_status}" + ) + except Exception as e: + logger.warning(f"Failed to process {instrumentation}: {e}") + continue + + readme_path = os.path.join(base_instrumentation_path, "README.md") + with open(readme_path, "w", encoding="utf-8") as fh: + fh.write("\n".join(table)) + logger.info(f"Generated {readme_path}") + + +if __name__ == "__main__": + root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + instrumentation_path = os.path.join( + root_path, "instrumentation-loongsuite" + ) + if os.path.exists(instrumentation_path): + main(instrumentation_path) + else: + logger.warning( + f"Instrumentation path does not exist: {instrumentation_path}" + ) diff --git a/scripts/loongsuite-build-config.json b/scripts/loongsuite-build-config.json new file mode 100644 index 000000000..2eb533711 --- /dev/null +++ b/scripts/loongsuite-build-config.json @@ -0,0 +1,11 @@ +{ + "skip_packages": [], + "description": "Build configuration file, defines packages to skip during build", + "notes": [ + "Packages in upstream packages_to_exclude are already excluded automatically.", + "This config is for additional LoongSuite-specific skip rules if needed.", + "When there are packages with the same name in instrumentation-genai and instrumentation-loongsuite,", + "prefer the version in instrumentation-loongsuite and skip the version in instrumentation-genai." + ] +} + diff --git a/tox-loongsuite.ini b/tox-loongsuite.ini index e2c78c608..e66fa243c 100644 --- a/tox-loongsuite.ini +++ b/tox-loongsuite.ini @@ -48,6 +48,13 @@ envlist = py3{10,11,12,13}-test-loongsuite-instrumentation-mem0-{oldest,latest} lint-loongsuite-instrumentation-mem0 + ; loongsuite-processor-baggage + py3{9,10,11,12,13}-test-loongsuite-processor-baggage + lint-loongsuite-processor-baggage + + ; generate tasks + generate-loongsuite + [testenv] test_deps = opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api @@ -95,6 +102,9 @@ deps = loongsuite-mem0-latest: {[testenv]test_deps} loongsuite-mem0-latest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0/test-requirements-latest.txt + + loongsuite-processor-baggage: {[testenv]test_deps} + loongsuite-processor-baggage: -r {toxinidir}/processor/loongsuite-processor-baggage/test-requirements.txt ; FIXME: add coverage testing allowlist_externals = @@ -141,9 +151,25 @@ commands = test-loongsuite-instrumentation-mem0: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0/tests {posargs} lint-loongsuite-instrumentation-mem0: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0 + test-loongsuite-processor-baggage: pytest {toxinidir}/processor/loongsuite-processor-baggage/tests {posargs} + lint-loongsuite-processor-baggage: python -m ruff check {toxinidir}/processor/loongsuite-processor-baggage + ; TODO: add coverage commands ; coverage: {toxinidir}/scripts/coverage.sh +[testenv:generate-loongsuite] +deps = + -r {toxinidir}/gen-requirements.txt + +allowlist_externals = + {toxinidir}/scripts/generate_loongsuite_bootstrap.py + {toxinidir}/scripts/generate_loongsuite_readme.py + pytest + +commands = + {toxinidir}/scripts/generate_loongsuite_bootstrap.py + {toxinidir}/scripts/generate_loongsuite_readme.py + [testenv:docs] deps = -c {toxinidir}/dev-requirements.txt From 7f3cd75d3a0f4dce1facc9e75ccb5d4e666a7872 Mon Sep 17 00:00:00 2001 From: cirilla-zmh Date: Fri, 27 Feb 2026 15:37:29 +0800 Subject: [PATCH 3/4] refactor: remove loongsuite baggage processor from release branch Move all baggage processor related code to feat/loongsuite-baggage-processor branch. - Remove baggage processor integration from loongsuite-distro - Remove processor from build script, tox, workflows, docs Change-Id: I40e5c6f812c2e19b6c268e2aedacd6550fac4435 Co-developed-by: Cursor Made-with: Cursor --- .github/workflows/loongsuite_lint_0.yml | 20 --- .github/workflows/loongsuite_test_0.yml | 100 ------------- docs/loongsuite-release.md | 2 - loongsuite-distro/README.rst | 21 --- loongsuite-distro/pyproject.toml | 3 - .../src/loongsuite/distro/__init__.py | 138 +----------------- scripts/build_loongsuite_package.py | 25 ---- tox-loongsuite.ini | 10 -- 8 files changed, 2 insertions(+), 317 deletions(-) diff --git a/.github/workflows/loongsuite_lint_0.yml b/.github/workflows/loongsuite_lint_0.yml index cc2e3c931..ec19cde2a 100644 --- a/.github/workflows/loongsuite_lint_0.yml +++ b/.github/workflows/loongsuite_lint_0.yml @@ -127,23 +127,3 @@ jobs: - name: Run tests run: tox -c tox-loongsuite.ini -e lint-loongsuite-instrumentation-mem0 - lint-loongsuite-processor-baggage: - name: LoongSuite loongsuite-processor-baggage - runs-on: ubuntu-latest - if: hashFiles('processor/loongsuite-processor-baggage/**') != '' - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.13 - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -c tox-loongsuite.ini -e lint-loongsuite-processor-baggage - diff --git a/.github/workflows/loongsuite_test_0.yml b/.github/workflows/loongsuite_test_0.yml index 5011b7737..2ef853c67 100644 --- a/.github/workflows/loongsuite_test_0.yml +++ b/.github/workflows/loongsuite_test_0.yml @@ -868,103 +868,3 @@ jobs: - name: Run tests run: tox -c tox-loongsuite.ini -e py313-test-loongsuite-instrumentation-mem0-latest -- -ra - py39-test-loongsuite-processor-baggage_ubuntu-latest: - name: LoongSuite loongsuite-processor-baggage 3.9 Ubuntu - runs-on: ubuntu-latest - if: hashFiles('processor/loongsuite-processor-baggage/**') != '' - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: "3.9" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -c tox-loongsuite.ini -e py39-test-loongsuite-processor-baggage -- -ra - - py310-test-loongsuite-processor-baggage_ubuntu-latest: - name: LoongSuite loongsuite-processor-baggage 3.10 Ubuntu - runs-on: ubuntu-latest - if: hashFiles('processor/loongsuite-processor-baggage/**') != '' - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -c tox-loongsuite.ini -e py310-test-loongsuite-processor-baggage -- -ra - - py311-test-loongsuite-processor-baggage_ubuntu-latest: - name: LoongSuite loongsuite-processor-baggage 3.11 Ubuntu - runs-on: ubuntu-latest - if: hashFiles('processor/loongsuite-processor-baggage/**') != '' - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -c tox-loongsuite.ini -e py311-test-loongsuite-processor-baggage -- -ra - - py312-test-loongsuite-processor-baggage_ubuntu-latest: - name: LoongSuite loongsuite-processor-baggage 3.12 Ubuntu - runs-on: ubuntu-latest - if: hashFiles('processor/loongsuite-processor-baggage/**') != '' - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -c tox-loongsuite.ini -e py312-test-loongsuite-processor-baggage -- -ra - - py313-test-loongsuite-processor-baggage_ubuntu-latest: - name: LoongSuite loongsuite-processor-baggage 3.13 Ubuntu - runs-on: ubuntu-latest - if: hashFiles('processor/loongsuite-processor-baggage/**') != '' - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.13 - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -c tox-loongsuite.ini -e py313-test-loongsuite-processor-baggage -- -ra - diff --git a/docs/loongsuite-release.md b/docs/loongsuite-release.md index 57a0e8775..81354ffd0 100644 --- a/docs/loongsuite-release.md +++ b/docs/loongsuite-release.md @@ -54,7 +54,6 @@ LoongSuite Python Agent 是基于 [OpenTelemetry Python Contrib](https://github. | **GenAI instrumentations** | `instrumentation-genai/*` | 来自上游的 GenAI 插桩，如 OpenAI、VertexAI 等 | | **LoongSuite instrumentations** | `instrumentation-loongsuite/*` | LoongSuite 自研插桩，如 DashScope、AgentScope、MCP 等 | | **标准 instrumentations** | `instrumentation/*` | 标准微服务插桩（Flask、Django、Redis 等），由上游发布 | -| **processor** | `processor/loongsuite-processor-baggage` | Baggage 处理器 | ### 1.3 发布渠道 @@ -170,7 +169,6 @@ python scripts/build_loongsuite_package.py --build-github-release \ - **规则 2**：动态检测依赖，将 `opentelemetry-util-genai` 替换为 `loongsuite-util-genai` - 遍历 `instrumentation-loongsuite/` 目录： - 仅应用依赖替换规则 -- 遍历 `processor/loongsuite-processor-baggage/` - 所有 `.whl` 打包为 `loongsuite-python-agent-{version}.tar.gz` **规则匹配（无需硬编码包名）：** diff --git a/loongsuite-distro/README.rst b/loongsuite-distro/README.rst index 142d2d6fb..65568067a 100644 --- a/loongsuite-distro/README.rst +++ b/loongsuite-distro/README.rst @@ -14,21 +14,14 @@ Optional dependencies: :: - # Install with baggage processor support - pip install loongsuite-distro[baggage] - # Install with OTLP exporter support pip install loongsuite-distro[otlp] - - # Install with both - pip install loongsuite-distro[baggage,otlp] Features -------- 1. **LoongSuite Distro**: Provides LoongSuite-specific OpenTelemetry configuration 2. **LoongSuite Bootstrap**: Install all LoongSuite components from tar package -3. **Baggage Processor**: Optional baggage span processor with prefix matching and stripping support Usage ----- @@ -53,20 +46,6 @@ Install latest version:: loongsuite-bootstrap --latest -### Configure Baggage Processor - -The baggage processor is automatically loaded if configured via environment variables. -First, install the optional dependency:: - - pip install loongsuite-distro[baggage] - -Then configure via environment variables:: - - export LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES="traffic.,app." - export LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES="traffic." - -The processor will only be loaded if ``LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES`` is set. - For more usage, please refer to `LOONGSUITE_BOOTSTRAP_README.md`. References diff --git a/loongsuite-distro/pyproject.toml b/loongsuite-distro/pyproject.toml index f7cba84f8..956dd2edb 100644 --- a/loongsuite-distro/pyproject.toml +++ b/loongsuite-distro/pyproject.toml @@ -36,9 +36,6 @@ dependencies = [ otlp = [ "opentelemetry-exporter-otlp ~= 1.40", ] -baggage = [ - "loongsuite-processor-baggage", -] [project.entry-points.opentelemetry_configurator] loongsuite = "loongsuite.distro:LoongSuiteConfigurator" diff --git a/loongsuite-distro/src/loongsuite/distro/__init__.py b/loongsuite-distro/src/loongsuite/distro/__init__.py index 375f38909..86d73b0e2 100644 --- a/loongsuite-distro/src/loongsuite/distro/__init__.py +++ b/loongsuite-distro/src/loongsuite/distro/__init__.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os -from typing import TYPE_CHECKING, Any, Optional, Set, cast +from typing import Any -from opentelemetry import trace from opentelemetry.environment_variables import ( OTEL_LOGS_EXPORTER, OTEL_METRICS_EXPORTER, @@ -25,145 +23,13 @@ from opentelemetry.instrumentation.distro import BaseDistro from opentelemetry.sdk._configuration import _OTelSDKConfigurator from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_PROTOCOL -from opentelemetry.sdk.trace import TracerProvider - -if TYPE_CHECKING: - from opentelemetry.sdk.trace import SpanProcessor - -logger = logging.getLogger(__name__) - -# Environment variable names for baggage processor configuration -_LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES = ( - "LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES" -) -_LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES = ( - "LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES" -) class LoongSuiteConfigurator(_OTelSDKConfigurator): """ - LoongSuite configurator, inherits from OpenTelemetry SDK configurator - - Automatically adds LoongSuiteBaggageSpanProcessor if configured via environment variables. - Only loads the processor if LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES is set. + LoongSuite configurator, inherits from OpenTelemetry SDK configurator. """ - def _configure(self, **kwargs: Any) -> None: - # Call parent method to complete base initialization - super()._configure(**kwargs) # type: ignore[misc] - - # Get tracer provider - tracer_provider = trace.get_tracer_provider() - - if isinstance(tracer_provider, TracerProvider): - # Get additional processors - additional_processors = self._get_additional_span_processors( - **kwargs - ) - - # Add additional processors - for processor in additional_processors: - tracer_provider.add_span_processor(processor) - - def _get_additional_span_processors( - self, **kwargs: Any - ) -> list["SpanProcessor"]: - """ - Return additional span processors to add to trace provider - - Subclasses can override this method to provide custom processors. - - Supports configuration via environment variables for baggage processor: - - LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES: Comma-separated list of prefixes for matching baggage keys - - LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES: Comma-separated list of prefixes to strip from baggage keys - - The baggage processor is only loaded if LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES is set. - - Args: - **kwargs: Arguments passed to _configure - - Returns: - List of span processors to add - """ - processors: list["SpanProcessor"] = [] - - # Check if baggage allowed prefixes is configured - allowed_prefixes_str = os.getenv( - _LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES - ) - - if allowed_prefixes_str: - # Try to load loongsuite-processor-baggage - try: - # Dynamic import to avoid type checker errors - from loongsuite.processor.baggage import ( # noqa: PLC0415 - LoongSuiteBaggageSpanProcessor, - ) - - # Parse allowed prefixes - allowed_prefixes = self._parse_prefixes(allowed_prefixes_str) - - # Parse strip prefixes - strip_prefixes_str = os.getenv( - _LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES - ) - strip_prefixes = ( - self._parse_prefixes(strip_prefixes_str) - if strip_prefixes_str - else None - ) - - # Create processor - # LoongSuiteBaggageSpanProcessor inherits from SpanProcessor - processor_instance = LoongSuiteBaggageSpanProcessor( # type: ignore[misc] - allowed_prefixes=allowed_prefixes - if allowed_prefixes - else None, - strip_prefixes=strip_prefixes if strip_prefixes else None, - ) - # Type cast since LoongSuiteBaggageSpanProcessor inherits from SpanProcessor - processor = cast("SpanProcessor", processor_instance) - processors.append(processor) - - logger.info( - "Loaded LoongSuiteBaggageSpanProcessor with allowed_prefixes=%s, strip_prefixes=%s", - allowed_prefixes, - strip_prefixes, - ) - except ImportError as e: - logger.warning( - "Failed to import loongsuite.processor.baggage: %s. " - "Baggage processor will not be loaded. " - "Please install loongsuite-processor-baggage package.", - e, - ) - - return processors - - @staticmethod - def _parse_prefixes(prefixes_str: str) -> Optional[Set[str]]: - """ - Parse comma-separated prefix string - - Args: - prefixes_str: Comma-separated prefix string, e.g., "traffic.,app." - - Returns: - Set of prefixes, or None if input is empty - """ - if not prefixes_str or not prefixes_str.strip(): - return None - - # Split and strip whitespace - prefixes = { - prefix.strip() - for prefix in prefixes_str.split(",") - if prefix.strip() - } - - return prefixes if prefixes else None - class LoongSuiteDistro(BaseDistro): """ diff --git a/scripts/build_loongsuite_package.py b/scripts/build_loongsuite_package.py index 924938288..2aa189dec 100755 --- a/scripts/build_loongsuite_package.py +++ b/scripts/build_loongsuite_package.py @@ -11,7 +11,6 @@ 2. --build-github-release: Build packages for GitHub Release (tar.gz) - instrumentation-genai/ packages (renamed to loongsuite-*, depends on loongsuite-util-genai) - instrumentation-loongsuite/ packages (depends on loongsuite-util-genai) - - processor/loongsuite-processor-baggage/ Version replacement: - --version: Sets version for all packages being built @@ -306,7 +305,6 @@ def build_github_release_packages( Build packages for GitHub Release (tar.gz): - instrumentation-genai/ (renamed to loongsuite-*, depends on loongsuite-util-genai) - instrumentation-loongsuite/ (depends on loongsuite-util-genai) - - processor/loongsuite-processor-baggage/ """ all_whl_files = [] existing_whl_files = set(dist_dir.glob("*.whl")) @@ -426,29 +424,6 @@ def _get_modifications(package_dir: Path) -> Dict[str, Any]: all_whl_files.extend(whl_files) existing_whl_files.update(whl_files) - # 3. Build processor/loongsuite-processor-baggage/ - processor_baggage_dir = ( - base_dir / "processor" / "loongsuite-processor-baggage" - ) - if ( - processor_baggage_dir.exists() - and (processor_baggage_dir / "pyproject.toml").exists() - ): - pkg_name = processor_baggage_dir.name - if pkg_name not in skip_packages: - version_py = find_version_py(processor_baggage_dir) - logger.info(f"Building {pkg_name} (version {version})...") - with ( - _patch_version_py(version_py, version) - if version_py - else nullcontext() - ): - whl_files = build_package( - processor_baggage_dir, dist_dir, existing_whl_files - ) - all_whl_files.extend(whl_files) - existing_whl_files.update(whl_files) - return all_whl_files diff --git a/tox-loongsuite.ini b/tox-loongsuite.ini index e66fa243c..924c47765 100644 --- a/tox-loongsuite.ini +++ b/tox-loongsuite.ini @@ -48,10 +48,6 @@ envlist = py3{10,11,12,13}-test-loongsuite-instrumentation-mem0-{oldest,latest} lint-loongsuite-instrumentation-mem0 - ; loongsuite-processor-baggage - py3{9,10,11,12,13}-test-loongsuite-processor-baggage - lint-loongsuite-processor-baggage - ; generate tasks generate-loongsuite @@ -103,9 +99,6 @@ deps = loongsuite-mem0-latest: {[testenv]test_deps} loongsuite-mem0-latest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0/test-requirements-latest.txt - loongsuite-processor-baggage: {[testenv]test_deps} - loongsuite-processor-baggage: -r {toxinidir}/processor/loongsuite-processor-baggage/test-requirements.txt - ; FIXME: add coverage testing allowlist_externals = sh @@ -151,9 +144,6 @@ commands = test-loongsuite-instrumentation-mem0: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0/tests {posargs} lint-loongsuite-instrumentation-mem0: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0 - test-loongsuite-processor-baggage: pytest {toxinidir}/processor/loongsuite-processor-baggage/tests {posargs} - lint-loongsuite-processor-baggage: python -m ruff check {toxinidir}/processor/loongsuite-processor-baggage - ; TODO: add coverage commands ; coverage: {toxinidir}/scripts/coverage.sh From 0f823a521120aa52c3ae4ba281bc68073c73a90d Mon Sep 17 00:00:00 2001 From: cirilla-zmh Date: Fri, 27 Feb 2026 15:38:57 +0800 Subject: [PATCH 4/4] feat: add baggage processor integration to distro, build, tox, workflows and docs - LoongSuiteConfigurator: load LoongSuiteBaggageSpanProcessor when env configured - loongsuite-distro: add baggage optional dependency and README docs - build_loongsuite_package: build processor/loongsuite-processor-baggage for GitHub Release - tox-loongsuite.ini: add processor test and lint envs - loongsuite_test_0/lint_0: add CI jobs for processor Change-Id: Ia3c29c1e71f480e1d862af815f52c263f177da02 Co-developed-by: Cursor Made-with: Cursor --- .github/workflows/loongsuite_lint_0.yml | 20 +++ .github/workflows/loongsuite_test_0.yml | 100 +++++++++++++ docs/loongsuite-release.md | 2 + loongsuite-distro/README.rst | 21 +++ loongsuite-distro/pyproject.toml | 3 + .../src/loongsuite/distro/__init__.py | 138 +++++++++++++++++- scripts/build_loongsuite_package.py | 25 ++++ tox-loongsuite.ini | 10 ++ 8 files changed, 317 insertions(+), 2 deletions(-) diff --git a/.github/workflows/loongsuite_lint_0.yml b/.github/workflows/loongsuite_lint_0.yml index ec19cde2a..cc2e3c931 100644 --- a/.github/workflows/loongsuite_lint_0.yml +++ b/.github/workflows/loongsuite_lint_0.yml @@ -127,3 +127,23 @@ jobs: - name: Run tests run: tox -c tox-loongsuite.ini -e lint-loongsuite-instrumentation-mem0 + lint-loongsuite-processor-baggage: + name: LoongSuite loongsuite-processor-baggage + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e lint-loongsuite-processor-baggage + diff --git a/.github/workflows/loongsuite_test_0.yml b/.github/workflows/loongsuite_test_0.yml index 2ef853c67..5011b7737 100644 --- a/.github/workflows/loongsuite_test_0.yml +++ b/.github/workflows/loongsuite_test_0.yml @@ -868,3 +868,103 @@ jobs: - name: Run tests run: tox -c tox-loongsuite.ini -e py313-test-loongsuite-instrumentation-mem0-latest -- -ra + py39-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.9 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py39-test-loongsuite-processor-baggage -- -ra + + py310-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.10 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py310-test-loongsuite-processor-baggage -- -ra + + py311-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.11 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py311-test-loongsuite-processor-baggage -- -ra + + py312-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.12 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py312-test-loongsuite-processor-baggage -- -ra + + py313-test-loongsuite-processor-baggage_ubuntu-latest: + name: LoongSuite loongsuite-processor-baggage 3.13 Ubuntu + runs-on: ubuntu-latest + if: hashFiles('processor/loongsuite-processor-baggage/**') != '' + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -c tox-loongsuite.ini -e py313-test-loongsuite-processor-baggage -- -ra + diff --git a/docs/loongsuite-release.md b/docs/loongsuite-release.md index 81354ffd0..57a0e8775 100644 --- a/docs/loongsuite-release.md +++ b/docs/loongsuite-release.md @@ -54,6 +54,7 @@ LoongSuite Python Agent 是基于 [OpenTelemetry Python Contrib](https://github. | **GenAI instrumentations** | `instrumentation-genai/*` | 来自上游的 GenAI 插桩，如 OpenAI、VertexAI 等 | | **LoongSuite instrumentations** | `instrumentation-loongsuite/*` | LoongSuite 自研插桩，如 DashScope、AgentScope、MCP 等 | | **标准 instrumentations** | `instrumentation/*` | 标准微服务插桩（Flask、Django、Redis 等），由上游发布 | +| **processor** | `processor/loongsuite-processor-baggage` | Baggage 处理器 | ### 1.3 发布渠道 @@ -169,6 +170,7 @@ python scripts/build_loongsuite_package.py --build-github-release \ - **规则 2**：动态检测依赖，将 `opentelemetry-util-genai` 替换为 `loongsuite-util-genai` - 遍历 `instrumentation-loongsuite/` 目录： - 仅应用依赖替换规则 +- 遍历 `processor/loongsuite-processor-baggage/` - 所有 `.whl` 打包为 `loongsuite-python-agent-{version}.tar.gz` **规则匹配（无需硬编码包名）：** diff --git a/loongsuite-distro/README.rst b/loongsuite-distro/README.rst index 65568067a..142d2d6fb 100644 --- a/loongsuite-distro/README.rst +++ b/loongsuite-distro/README.rst @@ -14,14 +14,21 @@ Optional dependencies: :: + # Install with baggage processor support + pip install loongsuite-distro[baggage] + # Install with OTLP exporter support pip install loongsuite-distro[otlp] + + # Install with both + pip install loongsuite-distro[baggage,otlp] Features -------- 1. **LoongSuite Distro**: Provides LoongSuite-specific OpenTelemetry configuration 2. **LoongSuite Bootstrap**: Install all LoongSuite components from tar package +3. **Baggage Processor**: Optional baggage span processor with prefix matching and stripping support Usage ----- @@ -46,6 +53,20 @@ Install latest version:: loongsuite-bootstrap --latest +### Configure Baggage Processor + +The baggage processor is automatically loaded if configured via environment variables. +First, install the optional dependency:: + + pip install loongsuite-distro[baggage] + +Then configure via environment variables:: + + export LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES="traffic.,app." + export LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES="traffic." + +The processor will only be loaded if ``LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES`` is set. + For more usage, please refer to `LOONGSUITE_BOOTSTRAP_README.md`. References diff --git a/loongsuite-distro/pyproject.toml b/loongsuite-distro/pyproject.toml index 956dd2edb..f7cba84f8 100644 --- a/loongsuite-distro/pyproject.toml +++ b/loongsuite-distro/pyproject.toml @@ -36,6 +36,9 @@ dependencies = [ otlp = [ "opentelemetry-exporter-otlp ~= 1.40", ] +baggage = [ + "loongsuite-processor-baggage", +] [project.entry-points.opentelemetry_configurator] loongsuite = "loongsuite.distro:LoongSuiteConfigurator" diff --git a/loongsuite-distro/src/loongsuite/distro/__init__.py b/loongsuite-distro/src/loongsuite/distro/__init__.py index 86d73b0e2..375f38909 100644 --- a/loongsuite-distro/src/loongsuite/distro/__init__.py +++ b/loongsuite-distro/src/loongsuite/distro/__init__.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os -from typing import Any +from typing import TYPE_CHECKING, Any, Optional, Set, cast +from opentelemetry import trace from opentelemetry.environment_variables import ( OTEL_LOGS_EXPORTER, OTEL_METRICS_EXPORTER, @@ -23,13 +25,145 @@ from opentelemetry.instrumentation.distro import BaseDistro from opentelemetry.sdk._configuration import _OTelSDKConfigurator from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_PROTOCOL +from opentelemetry.sdk.trace import TracerProvider + +if TYPE_CHECKING: + from opentelemetry.sdk.trace import SpanProcessor + +logger = logging.getLogger(__name__) + +# Environment variable names for baggage processor configuration +_LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES = ( + "LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES" +) +_LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES = ( + "LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES" +) class LoongSuiteConfigurator(_OTelSDKConfigurator): """ - LoongSuite configurator, inherits from OpenTelemetry SDK configurator. + LoongSuite configurator, inherits from OpenTelemetry SDK configurator + + Automatically adds LoongSuiteBaggageSpanProcessor if configured via environment variables. + Only loads the processor if LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES is set. """ + def _configure(self, **kwargs: Any) -> None: + # Call parent method to complete base initialization + super()._configure(**kwargs) # type: ignore[misc] + + # Get tracer provider + tracer_provider = trace.get_tracer_provider() + + if isinstance(tracer_provider, TracerProvider): + # Get additional processors + additional_processors = self._get_additional_span_processors( + **kwargs + ) + + # Add additional processors + for processor in additional_processors: + tracer_provider.add_span_processor(processor) + + def _get_additional_span_processors( + self, **kwargs: Any + ) -> list["SpanProcessor"]: + """ + Return additional span processors to add to trace provider + + Subclasses can override this method to provide custom processors. + + Supports configuration via environment variables for baggage processor: + - LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES: Comma-separated list of prefixes for matching baggage keys + - LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES: Comma-separated list of prefixes to strip from baggage keys + + The baggage processor is only loaded if LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES is set. + + Args: + **kwargs: Arguments passed to _configure + + Returns: + List of span processors to add + """ + processors: list["SpanProcessor"] = [] + + # Check if baggage allowed prefixes is configured + allowed_prefixes_str = os.getenv( + _LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES + ) + + if allowed_prefixes_str: + # Try to load loongsuite-processor-baggage + try: + # Dynamic import to avoid type checker errors + from loongsuite.processor.baggage import ( # noqa: PLC0415 + LoongSuiteBaggageSpanProcessor, + ) + + # Parse allowed prefixes + allowed_prefixes = self._parse_prefixes(allowed_prefixes_str) + + # Parse strip prefixes + strip_prefixes_str = os.getenv( + _LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES + ) + strip_prefixes = ( + self._parse_prefixes(strip_prefixes_str) + if strip_prefixes_str + else None + ) + + # Create processor + # LoongSuiteBaggageSpanProcessor inherits from SpanProcessor + processor_instance = LoongSuiteBaggageSpanProcessor( # type: ignore[misc] + allowed_prefixes=allowed_prefixes + if allowed_prefixes + else None, + strip_prefixes=strip_prefixes if strip_prefixes else None, + ) + # Type cast since LoongSuiteBaggageSpanProcessor inherits from SpanProcessor + processor = cast("SpanProcessor", processor_instance) + processors.append(processor) + + logger.info( + "Loaded LoongSuiteBaggageSpanProcessor with allowed_prefixes=%s, strip_prefixes=%s", + allowed_prefixes, + strip_prefixes, + ) + except ImportError as e: + logger.warning( + "Failed to import loongsuite.processor.baggage: %s. " + "Baggage processor will not be loaded. " + "Please install loongsuite-processor-baggage package.", + e, + ) + + return processors + + @staticmethod + def _parse_prefixes(prefixes_str: str) -> Optional[Set[str]]: + """ + Parse comma-separated prefix string + + Args: + prefixes_str: Comma-separated prefix string, e.g., "traffic.,app." + + Returns: + Set of prefixes, or None if input is empty + """ + if not prefixes_str or not prefixes_str.strip(): + return None + + # Split and strip whitespace + prefixes = { + prefix.strip() + for prefix in prefixes_str.split(",") + if prefix.strip() + } + + return prefixes if prefixes else None + class LoongSuiteDistro(BaseDistro): """ diff --git a/scripts/build_loongsuite_package.py b/scripts/build_loongsuite_package.py index 2aa189dec..924938288 100755 --- a/scripts/build_loongsuite_package.py +++ b/scripts/build_loongsuite_package.py @@ -11,6 +11,7 @@ 2. --build-github-release: Build packages for GitHub Release (tar.gz) - instrumentation-genai/ packages (renamed to loongsuite-*, depends on loongsuite-util-genai) - instrumentation-loongsuite/ packages (depends on loongsuite-util-genai) + - processor/loongsuite-processor-baggage/ Version replacement: - --version: Sets version for all packages being built @@ -305,6 +306,7 @@ def build_github_release_packages( Build packages for GitHub Release (tar.gz): - instrumentation-genai/ (renamed to loongsuite-*, depends on loongsuite-util-genai) - instrumentation-loongsuite/ (depends on loongsuite-util-genai) + - processor/loongsuite-processor-baggage/ """ all_whl_files = [] existing_whl_files = set(dist_dir.glob("*.whl")) @@ -424,6 +426,29 @@ def _get_modifications(package_dir: Path) -> Dict[str, Any]: all_whl_files.extend(whl_files) existing_whl_files.update(whl_files) + # 3. Build processor/loongsuite-processor-baggage/ + processor_baggage_dir = ( + base_dir / "processor" / "loongsuite-processor-baggage" + ) + if ( + processor_baggage_dir.exists() + and (processor_baggage_dir / "pyproject.toml").exists() + ): + pkg_name = processor_baggage_dir.name + if pkg_name not in skip_packages: + version_py = find_version_py(processor_baggage_dir) + logger.info(f"Building {pkg_name} (version {version})...") + with ( + _patch_version_py(version_py, version) + if version_py + else nullcontext() + ): + whl_files = build_package( + processor_baggage_dir, dist_dir, existing_whl_files + ) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + return all_whl_files diff --git a/tox-loongsuite.ini b/tox-loongsuite.ini index 924c47765..d356a196c 100644 --- a/tox-loongsuite.ini +++ b/tox-loongsuite.ini @@ -48,6 +48,10 @@ envlist = py3{10,11,12,13}-test-loongsuite-instrumentation-mem0-{oldest,latest} lint-loongsuite-instrumentation-mem0 + ; loongsuite-processor-baggage + py3{9,10,11,12,13}-test-loongsuite-processor-baggage + lint-loongsuite-processor-baggage + ; generate tasks generate-loongsuite @@ -99,6 +103,9 @@ deps = loongsuite-mem0-latest: {[testenv]test_deps} loongsuite-mem0-latest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0/test-requirements-latest.txt + loongsuite-processor-baggage: {[testenv]test_deps} + loongsuite-processor-baggage: -r {toxinidir}/processor/loongsuite-processor-baggage/test-requirements.txt + ; FIXME: add coverage testing allowlist_externals = sh @@ -144,6 +151,9 @@ commands = test-loongsuite-instrumentation-mem0: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0/tests {posargs} lint-loongsuite-instrumentation-mem0: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0 + test-loongsuite-processor-baggage: pytest {toxinidir}/processor/loongsuite-processor-baggage/tests {posargs} + lint-loongsuite-processor-baggage: python -m ruff check {toxinidir}/processor/loongsuite-processor-baggage + ; TODO: add coverage commands ; coverage: {toxinidir}/scripts/coverage.sh