From 315102de633dc5543de727af026564e1b9b115d2 Mon Sep 17 00:00:00 2001 From: Harsh Modi Date: Mon, 11 Sep 2023 11:27:19 -0400 Subject: [PATCH] add `openetelemetry_info` to `user_params` Signed-off-by: Harsh Modi --- .github/workflows/unittests.yaml | 4 +-- osbs/api.py | 22 ++++++++++++++++ osbs/build/user_params.py | 15 +++++++++++ osbs/cli/main.py | 43 +++++++++++++++++++++++++----- osbs/conf.py | 10 +++++++ osbs/constants.py | 2 ++ osbs/utils/otel.py | 45 ++++++++++++++++++++++++++++++++ requirements.txt | 5 ++++ test.sh | 4 +-- 9 files changed, 139 insertions(+), 11 deletions(-) create mode 100644 osbs/utils/otel.py diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index 020e2c477..8e34dcb22 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -18,12 +18,12 @@ jobs: matrix: os: - name: fedora - version: 32 + version: 36 python: 3 engine: docker - name: fedora - version: 33 + version: 37 python: 3 engine: docker diff --git a/osbs/api.py b/osbs/api.py index 186bf534b..b421f493a 100644 --- a/osbs/api.py +++ b/osbs/api.py @@ -16,6 +16,8 @@ from typing import Any, Dict from string import Template +from otel_extensions import instrumented + from osbs.build.user_params import ( BuildUserParams, SourceContainerUserParams @@ -27,6 +29,7 @@ from osbs.utils.labels import Labels # import utils in this way, so that we can mock standalone functions with flexmock from osbs import utils +from osbs.utils.otel import get_current_traceparent, init_otel def _load_pipeline_from_template(pipeline_run_path, substitutions): @@ -299,6 +302,15 @@ def create_binary_container_pipeline_run(self, req_labels = self._check_labels(repo_info) + if 'traceparent' in kwargs: + # we are effectively making current span as parent here + # if the traceparent is not updated then child call will + # be linked to the parent of current call + traceparent = get_current_traceparent() + kwargs.update({ + 'traceparent': traceparent + }) + user_params = self.get_user_params( base_image=repo_info.base_image, component=component, @@ -309,6 +321,7 @@ def create_binary_container_pipeline_run(self, req_labels=req_labels, repo_info=repo_info, operator_csv_modifications_url=operator_csv_modifications_url, + otel_url=self.os_conf.get_otel_url(), **kwargs) self._checks_for_isolated(user_params) @@ -371,11 +384,20 @@ def create_source_container_pipeline_run(self, if error_messages: raise OsbsValidationException(", ".join(error_messages)) + if 'traceparent' in kwargs: + # we are effectively making current span as parent here + # if the traceparent is not updated then child call will + # be linked to the parent of current call + traceparent = get_current_traceparent() + kwargs.update({ + 'traceparent': traceparent + }) user_params = SourceContainerUserParams.make_params( build_conf=self.os_conf, component=component, koji_target=target, koji_task_id=koji_task_id, + otel_url=self.os_conf.get_otel_url(), **kwargs ) diff --git a/osbs/build/user_params.py b/osbs/build/user_params.py index b276c3a35..31b4e2435 100644 --- a/osbs/build/user_params.py +++ b/osbs/build/user_params.py @@ -86,6 +86,7 @@ class BuildCommon(BuildParamsBase): koji_target = BuildParam("koji_target") koji_task_id = BuildParam("koji_task_id") platform = BuildParam("platform") + opentelemetry_info = BuildParam("opentelemetry_info") reactor_config_map = BuildParam("reactor_config_map") scratch = BuildParam("scratch") signing_intent = BuildParam("signing_intent") @@ -102,6 +103,7 @@ def make_params(cls, component=None, koji_target=None, koji_task_id=None, + otel_url=None, platform=None, scratch=None, signing_intent=None, @@ -129,6 +131,7 @@ def make_params(cls, :param koji_parent_build: str, :param koji_target: str, koji tag with packages used to build the image :param koji_task_id: int, koji *task* ID + :param otel_url: str, opentelemetry collector URL :param platform: str, platform :param scratch: bool, build as a scratch build (if not specified in build_conf) :param signing_intent: bool, True to sign the resulting image @@ -148,6 +151,13 @@ def make_params(cls, reactor_config = build_conf.get_reactor_config_map_scratch() else: reactor_config = build_conf.get_reactor_config_map() + traceparent = kwargs.get("traceparent", None) + opentelemetry_info = None + if traceparent or otel_url: + opentelemetry_info = { + "traceparent": traceparent, + "otel_url": otel_url, + } # Update kwargs with arguments explicitly accepted by this method kwargs.update({ "component": component, @@ -162,6 +172,11 @@ def make_params(cls, "scratch": build_conf.get_scratch(scratch), }) + if opentelemetry_info: + kwargs.update({ + "opentelemetry_info": opentelemetry_info + }) + # Drop arguments that are: # - unknown; some callers may pass deprecated params # - not set (set to None, either explicitly or implicitly) diff --git a/osbs/cli/main.py b/osbs/cli/main.py index 03cd7b0c8..68c642d9f 100644 --- a/osbs/cli/main.py +++ b/osbs/cli/main.py @@ -14,14 +14,18 @@ import sys import argparse + +from otel_extensions import get_tracer + from osbs import set_logging from osbs.api import OSBS from osbs.conf import Configuration from osbs.constants import (DEFAULT_CONFIGURATION_FILE, DEFAULT_CONF_BINARY_SECTION, - DEFAULT_CONF_SOURCE_SECTION) + DEFAULT_CONF_SOURCE_SECTION, OTEL_SERVICE_NAME) from osbs.exceptions import (OsbsNetworkException, OsbsException, OsbsAuthException, OsbsResponseException) from osbs.utils import UserWarningsStore +from osbs.utils.otel import init_otel logger = logging.getLogger('osbs') @@ -134,12 +138,21 @@ def cmd_build(args): } if args.userdata: build_kwargs['userdata'] = json.loads(args.userdata) + if args.traceparent: + build_kwargs['traceparent'] = args.traceparent if osbs.os_conf.get_flatpak(): build_kwargs['flatpak'] = True - pipeline_run = osbs.create_binary_container_pipeline_run(**build_kwargs) + init_otel(otel_url=osbs.os_conf.get_otel_url(), + traceparent=args.traceparent) + + span_name = "binary_build_pipeline" + tracer = get_tracer(module_name=span_name, service_name=OTEL_SERVICE_NAME) + + with tracer.start_as_current_span(span_name): + pipeline_run = osbs.create_binary_container_build(**build_kwargs) - print_output(pipeline_run, export_metadata_file=args.export_metadata_file) + print_output(pipeline_run, export_metadata_file=args.export_metadata_file) return_val = -1 @@ -148,7 +161,8 @@ def cmd_build(args): cleanup_used_resources = osbs.os_conf.get_cleanup_used_resources() if cleanup_used_resources and pipeline_run.data is not None: try: - logger.info("pipeline run removed: %s", pipeline_run.remove_pipeline_run()) + with tracer.start_as_current_span("remove_binary_pipeline"): + logger.info("pipeline run removed: %s", pipeline_run.remove_pipeline_run()) except OsbsResponseException: logger.error("failed to remove pipeline run %s", pipeline_run.pipeline_run_name) raise @@ -176,10 +190,19 @@ def cmd_build_source_container(args): } if args.userdata: build_kwargs['userdata'] = json.loads(args.userdata) + if args.traceparent: + build_kwargs['traceparent'] = args.traceparent - pipeline_run = osbs.create_source_container_pipeline_run(**build_kwargs) + init_otel(otel_url=osbs.os_conf.get_otel_url(), + traceparent=args.traceparent) - print_output(pipeline_run, export_metadata_file=args.export_metadata_file) + span_name = "source_build_pipeline" + tracer = get_tracer(module_name=span_name, service_name=OTEL_SERVICE_NAME) + + with tracer.start_as_current_span(span_name): + pipeline_run = osbs.create_source_container_build(**build_kwargs) + + print_output(pipeline_run, export_metadata_file=args.export_metadata_file) return_val = -1 @@ -188,7 +211,8 @@ def cmd_build_source_container(args): cleanup_used_resources = osbs.os_conf.get_cleanup_used_resources() if cleanup_used_resources and pipeline_run.data is not None: try: - logger.info("pipeline run removed: %s", pipeline_run.remove_pipeline_run()) + with tracer.start_as_current_span("remove_source_pipeline"): + logger.info("pipeline run removed: %s", pipeline_run.remove_pipeline_run()) except OsbsResponseException: logger.error("failed to remove pipeline run %s", pipeline_run.pipeline_run_name) raise @@ -289,6 +313,8 @@ def cli(): help='name of each platform to use (deprecated)') build_parser.add_argument('--source-registry-uri', action='store', required=False, help="set source registry for pulling parent image") + build_parser.add_argument("--traceparent", required=False, + help="TRACEPARENT for opentelemetry tracing") build_parser.add_argument("--userdata", required=False, help="JSON dictionary of user defined custom metadata") build_parser.set_defaults(func=cmd_build) @@ -329,6 +355,9 @@ def cli(): build_source_container_parser.add_argument( '--signing-intent', action='store', required=False, help='override signing intent') + build_source_container_parser.add_argument( + '--traceparent', required=False, + help='TRACEPARENT for opentelemetry tracing') build_source_container_parser.add_argument( '--userdata', required=False, help='JSON dictionary of user defined custom metadata') diff --git a/osbs/conf.py b/osbs/conf.py index 181cc7696..b429dfbfa 100644 --- a/osbs/conf.py +++ b/osbs/conf.py @@ -116,6 +116,16 @@ def get_openshift_base_uri(self): val = self._get_value(key, self.conf_section, key) return val + def get_otel_url(self): + """ + https://[:]/ + + :return: str + """ + key = "otel_url" + val = self._get_value(key, self.conf_section, key) + return val + @staticmethod def get_k8s_api_version(): # This is not configurable. diff --git a/osbs/constants.py b/osbs/constants.py index 18e003f48..caec35d4d 100644 --- a/osbs/constants.py +++ b/osbs/constants.py @@ -74,6 +74,8 @@ # number of seconds to wait, before retrying on openshift not found OS_NOT_FOUND_MAX_WAIT = 1 +OTEL_SERVICE_NAME = "osbs" + ISOLATED_RELEASE_FORMAT = re.compile(r'^\d+\.\d+(\..+)?$') RELEASE_LABEL_FORMAT = re.compile(r"""^\d+ # First character must be a digit ([._]? # allow separators between groups diff --git a/osbs/utils/otel.py b/osbs/utils/otel.py new file mode 100644 index 000000000..17915e6a2 --- /dev/null +++ b/osbs/utils/otel.py @@ -0,0 +1,45 @@ +""" +Copyright (c) 2023 Red Hat, Inc +All rights reserved. + +This software may be modified and distributed under the terms +of the BSD license. See the LICENSE file for details. +""" +import logging +import os +from typing import Optional + +from opentelemetry import trace +from opentelemetry.instrumentation.requests import RequestsInstrumentor +from opentelemetry.trace import format_trace_id, format_span_id +from otel_extensions import TelemetryOptions, init_telemetry_provider + +from osbs.constants import OTEL_SERVICE_NAME + +logger = logging.getLogger(__name__) + + +def init_otel(otel_url: Optional[str], traceparent: Optional[str]): + span_exporter = '' + otel_protocol = 'http/protobuf' + if not otel_url: + otel_protocol = 'custom' + span_exporter = '"opentelemetry.sdk.trace.export.ConsoleSpanExporter"' + + if traceparent: + os.environ['TRACEPARENT'] = traceparent + otel_options = TelemetryOptions( + OTEL_SERVICE_NAME=OTEL_SERVICE_NAME, + OTEL_EXPORTER_CUSTOM_SPAN_EXPORTER_TYPE=span_exporter, + OTEL_EXPORTER_OTLP_ENDPOINT=otel_url, + OTEL_EXPORTER_OTLP_PROTOCOL=otel_protocol, + ) + init_telemetry_provider(otel_options) + RequestsInstrumentor().instrument() + + +def get_current_traceparent(**kwargs): + tracecontext = trace.get_current_span().get_span_context() + traceparent = (f'00-{format_trace_id(tracecontext.trace_id)}-' + f'{format_span_id(tracecontext.span_id)}-01') + return traceparent diff --git a/requirements.txt b/requirements.txt index 69d285777..b557c5040 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,8 @@ requests requests-kerberos six PyYAML +opentelemetry-api==1.19.0 +opentelemetry-exporter-otlp==1.19.0 +opentelemetry-instrumentation-requests==0.40b0 +opentelemetry-sdk==1.19.0 +otel-extensions diff --git a/test.sh b/test.sh index 4db042730..291a585d0 100755 --- a/test.sh +++ b/test.sh @@ -31,14 +31,14 @@ function setup_osbs() { PIP_PKG="python-pip" PIP="pip" PKG="yum" - PKG_EXTRA=(yum-utils epel-release) + PKG_EXTRA=(yum-utils epel-release gcc-c++) BUILDDEP="yum-builddep" else PYTHON="python$PYTHON_VERSION" PIP_PKG="$PYTHON-pip" PIP="pip$PYTHON_VERSION" PKG="dnf" - PKG_EXTRA=(dnf-plugins-core "$PYTHON"-pylint) + PKG_EXTRA=(dnf-plugins-core "$PYTHON"-pylint gcc-c++) BUILDDEP=(dnf builddep) fi