diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b880a515f..cf31464010 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +**Breaking Changes**: + +- Remove the AWS lambda extension. ([#3568](https://github.com/getsentry/relay/pull/3568)) + **Bug fixes:** - Properly handle AI metrics from the Python SDK's `@ai_track` decorator ([#3539](https://github.com/getsentry/relay/pull/3539)) diff --git a/Cargo.lock b/Cargo.lock index 0c82cb82c7..af781f5049 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3618,18 +3618,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "relay-aws-extension" -version = "24.4.2" -dependencies = [ - "relay-log", - "relay-system", - "reqwest", - "serde", - "serde_json", - "tokio", -] - [[package]] name = "relay-base-schema" version = "24.4.2" @@ -4142,7 +4130,6 @@ dependencies = [ "rand", "regex", "relay-auth", - "relay-aws-extension", "relay-base-schema", "relay-cardinality", "relay-cogs", diff --git a/Cargo.toml b/Cargo.toml index 12c2127c64..ce16f1bbb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,6 @@ dbg_macro = "warn" [workspace.dependencies] relay-auth = { path = "relay-auth" } -relay-aws-extension = { path = "relay-aws-extension" } relay-base-schema = { path = "relay-base-schema" } relay-cardinality = { path = "relay-cardinality" } relay-cogs = { path = "relay-cogs" } diff --git a/relay-aws-extension/Cargo.toml b/relay-aws-extension/Cargo.toml deleted file mode 100644 index af7ddb06cc..0000000000 --- a/relay-aws-extension/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "relay-aws-extension" -authors = ["Sentry "] -description = "AWS extension implementation for Sentry's AWS Lambda layer" -homepage = "https://getsentry.github.io/relay/" -repository = "https://github.com/getsentry/relay" -version = "24.4.2" -edition = "2021" -license-file = "../LICENSE.md" -publish = false - -[lints] -workspace = true - -[dependencies] -relay-log = { workspace = true } -relay-system = { workspace = true } -reqwest = { workspace = true, features = ["json"] } -serde = { workspace = true } -serde_json = { workspace = true } -tokio = { workspace = true } diff --git a/relay-aws-extension/README.md b/relay-aws-extension/README.md deleted file mode 100644 index 4b66e10aff..0000000000 --- a/relay-aws-extension/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Sentry AWS Extension - -## Compilation on Apple M1 - -**NOTE:** This applies only if you have an **ARM based Apple M1**! - -Follow these steps below on your Apple M1 machine to compile relay to be -run in the AWS Lambda execution environment. - -(See https://github.com/messense/homebrew-macos-cross-toolchains for details) - -```bash -# Install cross compiling tool chain -brew tap messense/macos-cross-toolchains -brew install x86_64-unknown-linux-gnu - -export CC_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-gcc -export CXX_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-g++ -export AR_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-ar -export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-unknown-linux-gnu-gcc - -rustup target add x86_64-unknown-linux-gnu - -# Install objcopy (used in `Makefile`) -brew install binutils -ln -s "$(brew --prefix binutils)/bin/gobjcopy" /usr/local/bin/objcopy - -# Build the relay binary -export TARGET=x86_64-unknown-linux-gnu -make build-linux-release -``` - -The `relay` binary can be found at `target/x86_64-unknown-linux-gnu/release/relay` -and can be used as an AWS Lambda extension. diff --git a/relay-aws-extension/src/aws_extension.rs b/relay-aws-extension/src/aws_extension.rs deleted file mode 100644 index e8610845dd..0000000000 --- a/relay-aws-extension/src/aws_extension.rs +++ /dev/null @@ -1,246 +0,0 @@ -use std::collections::HashMap; -use std::fmt; -use std::ops::ControlFlow; - -use relay_system::{Controller, Service, ShutdownMode}; -use reqwest::{Client, ClientBuilder, StatusCode, Url}; -use serde::Deserialize; - -const EXTENSION_NAME: &str = "sentry-lambda-extension"; -const EXTENSION_NAME_HEADER: &str = "Lambda-Extension-Name"; -const EXTENSION_ID_HEADER: &str = "Lambda-Extension-Identifier"; - -/// Response received from the register API. -/// -/// # Example -/// -/// ```json -/// { -/// "functionName": "helloWorld", -/// "functionVersion": "$LATEST", -/// "handler": "lambda_function.lambda_handler" -/// } -/// ``` -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RegisterResponse { - /// The name of the lambda function. - pub function_name: String, - /// The version of the lambda function. - pub function_version: String, - /// The handler that the labmda function invokes. - pub handler: String, -} - -/// Tracing headers from an [`InvokeResponse`]. -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Tracing { - /// type of tracing header - #[serde(rename = "type")] - pub ty: String, - /// tracing header value - pub value: String, -} - -/// Response received from the next event API on an `INVOKE` event. -/// -/// See [Invoke Phase] for more information. -/// -/// # Example -/// -/// ```json -/// { -/// "eventType": "INVOKE", -/// "deadlineMs": 676051, -/// "requestId": "3da1f2dc-3222-475e-9205-e2e6c6318895", -/// "invokedFunctionArn": "arn:aws:lambda:us-east-1:123456789012:function:ExtensionTest", -/// "tracing": { -/// "type": "X-Amzn-Trace-Id", -/// "value": "Root=1-5f35ae12-0c0fec141ab77a00bc047aa2;Parent=2be948a625588e32;Sampled=1" -/// } -/// } -/// ``` -/// -/// [invoke phase]: https://docs.aws.amazon.com/lambda/latest/dg/runtimes-extensions-api.html#runtimes-lifecycle-invoke -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InvokeResponse { - /// The time and date when the lambda function times out in Unix time milliseconds. - pub deadline_ms: u64, - /// Unique request identifier. - pub request_id: String, - /// The invoked lambda function's ARN (Amazon Resource Name). - pub invoked_function_arn: String, - /// Tracing headers. - pub tracing: Tracing, -} - -/// Response received from the next event API on a `SHUTDOWN` event. -/// -/// See [Shutdown Phase] for more information. -/// -/// # Example -/// -/// ```json -/// { -/// "eventType": "SHUTDOWN", -/// "shutdownReason": "TIMEOUT", -/// "deadlineMs": 42069 -/// } -/// ``` -/// -/// [shutdown phase]: https://docs.aws.amazon.com/lambda/latest/dg/runtimes-extensions-api.html#runtimes-lifecycle-shutdown -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ShutdownResponse { - /// The reason for the shutdown. - pub shutdown_reason: String, - /// The time and date when the lambda function times out in Unix time milliseconds. - pub deadline_ms: u64, -} - -/// All possible next event responses. -#[derive(Debug, Deserialize)] -#[serde(rename_all = "UPPERCASE", tag = "eventType")] -pub enum NextEventResponse { - /// `INVOKE` response. - Invoke(InvokeResponse), - /// `SHUTDOWN` response. - Shutdown(ShutdownResponse), -} - -/// Generic error in an AWS extension context. -#[derive(Debug)] -pub struct AwsExtensionError(()); - -impl fmt::Display for AwsExtensionError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "aws extension error") - } -} - -impl std::error::Error for AwsExtensionError {} - -/// Service implementing an AWS extension. -/// -/// Spawns a service that: -/// -/// * registers with the AWS extensions API -/// * sends blocking (0 timeout) NextEvent calls to the extensions API to get the next invocation -/// -/// Each finished invocation immediately polls for the next event. Note that AWS might freeze the -/// container indefinitely for unused lambdas and this request will also wait until things are -/// active again. -/// -/// The actual requests are done in a separate tokio runtime with one worker thread with a oneshot -/// channel being used for communicating the necessary responses. -#[derive(Debug)] -pub struct AwsExtension { - /// The base url for the AWS Extensions API. - base_url: Url, - /// The reqwest client to make Extensions API requests with. - /// - /// Note that all timeouts need to be 0 because the container might get frozen if lambda is - /// idle. - reqwest_client: Client, -} - -impl AwsExtension { - /// Creates a new `AwsExtension` instance. - pub fn new(aws_runtime_api: &str) -> Result { - let base_url = format!("http://{aws_runtime_api}/2020-01-01/extension") - .parse() - .map_err(|_| AwsExtensionError(()))?; - - let reqwest_client = ClientBuilder::new() - .pool_idle_timeout(None) - .build() - .map_err(|_| AwsExtensionError(()))?; - - Ok(AwsExtension { - base_url, - reqwest_client, - }) - } - - async fn register(&mut self) -> Result { - relay_log::info!("Registering AWS extension on {}", self.base_url); - let url = format!("{}/register", self.base_url); - let body = HashMap::from([("events", ["INVOKE", "SHUTDOWN"])]); - - let request = self - .reqwest_client - .post(&url) - .header(EXTENSION_NAME_HEADER, EXTENSION_NAME) - .json(&body); - - let response = request.send().await.map_err(|_| AwsExtensionError(()))?; - - if response.status() != StatusCode::OK { - return Err(AwsExtensionError(())); - } - - let extension_id = response - .headers() - .get(EXTENSION_ID_HEADER) - .and_then(|h| h.to_str().ok()) - .map(|h| h.to_string()) - .ok_or(AwsExtensionError(()))?; - - Ok(extension_id) - } - - async fn next_event(&self, extension_id: &str) -> Result, AwsExtensionError> { - let url = format!("{}/event/next", self.base_url); - - let request = self - .reqwest_client - .get(&url) - .header(EXTENSION_ID_HEADER, extension_id); - - let res = request.send().await.map_err(|_| AwsExtensionError(()))?; - let next_event = res.json().await.map_err(|_| AwsExtensionError(()))?; - - match next_event { - NextEventResponse::Invoke(response) => { - relay_log::debug!("Received INVOKE: request_id {}", response.request_id); - Ok(ControlFlow::Continue(())) - } - NextEventResponse::Shutdown(response) => { - relay_log::debug!("Received SHUTDOWN: reason {}", response.shutdown_reason); - Controller::shutdown(ShutdownMode::Graceful); - Ok(ControlFlow::Break(())) - } - } - } -} - -impl Service for AwsExtension { - type Interface = (); - - fn spawn_handler(mut self, _rx: relay_system::Receiver) { - tokio::spawn(async move { - relay_log::info!("AWS extension started"); - - let extension_id = match self.register().await { - Ok(extension_id) => { - relay_log::info!("AWS extension successfully registered"); - extension_id - } - Err(_) => { - relay_log::info!("AWS extension registration failed"); - return; - } - }; - - while let Ok(control_flow) = self.next_event(&extension_id).await { - if control_flow.is_break() { - break; - } - } - - relay_log::info!("AWS extension stopped"); - }); - } -} diff --git a/relay-aws-extension/src/lib.rs b/relay-aws-extension/src/lib.rs deleted file mode 100644 index 73e2a0f17c..0000000000 --- a/relay-aws-extension/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! AWS extension implementation for Sentry's AWS Lambda layer. -//! -//! An AWS extension is a binary that is bundled along with the SDK and its dependencies in an AWS -//! Layer that is added to a user's Lambda function. The Lambda Runtime will start the extension -//! binary as a concurrent process that runs independently of the actual function invocation and -//! continues running across multiple function invocations. -//! -//! Sentry's extension is basically existing Relay running in proxy mode with an additional -//! [`AwsExtension`] service. The service takes care of the extension lifecycle, namely registering -//! the extension and continuously polling for next events. Note that the interval between two next -//! event calls adds to the billing time of the lambda invocation. -//! -//! The main advantage we get currently from the extension is the lambda function not having to wait -//! for the event being sent to Sentry by the SDK. The actual sending happens in a concurrent Relay -//! process so the main function can return sooner and reduce response time of the user's function. -//! In the future, we might use some of [`RegisterResponse`] and [`NextEventResponse`]'s fields in -//! the event payload but that requires intercepting the envelope and modifying it before sending to -//! upstream. -//! -//! See the official [Lambda Extensions API -//! documentation](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-extensions-api.html) for -//! further details. -#![warn(missing_docs)] -#![doc( - html_logo_url = "https://raw.githubusercontent.com/getsentry/relay/master/artwork/relay-icon.png", - html_favicon_url = "https://raw.githubusercontent.com/getsentry/relay/master/artwork/relay-icon.png" -)] -#![allow(clippy::derive_partial_eq_without_eq)] - -mod aws_extension; -pub use aws_extension::*; diff --git a/relay-config/src/config.rs b/relay-config/src/config.rs index 5e9827f842..cc3948cd7b 100644 --- a/relay-config/src/config.rs +++ b/relay-config/src/config.rs @@ -247,8 +247,6 @@ pub struct OverridableConfig { pub outcome_source: Option, /// shutdown timeout pub shutdown_timeout: Option, - /// AWS Extensions API URL - pub aws_runtime_api: Option, } /// The relay credentials @@ -1297,16 +1295,6 @@ pub struct AuthConfig { pub static_relays: HashMap, } -/// AWS extension config. -#[derive(Serialize, Deserialize, Debug, Default)] -pub struct AwsConfig { - /// The host and port of the AWS lambda extensions API. - /// - /// This value can be found in the `AWS_LAMBDA_RUNTIME_API` environment variable in a Lambda - /// Runtime and contains a socket address, usually `"127.0.0.1:9001"`. - pub runtime_api: Option, -} - /// GeoIp database configuration options. #[derive(Serialize, Deserialize, Debug, Default)] pub struct GeoIpConfig { @@ -1450,8 +1438,6 @@ struct ConfigValues { #[serde(default)] auth: AuthConfig, #[serde(default)] - aws: AwsConfig, - #[serde(default)] geoip: GeoIpConfig, #[serde(default)] normalization: Normalization, @@ -1662,11 +1648,6 @@ impl Config { } } - let aws = &mut self.values.aws; - if let Some(aws_runtime_api) = overrides.aws_runtime_api { - aws.runtime_api = Some(aws_runtime_api); - } - Ok(self) } @@ -2419,11 +2400,6 @@ impl Config { let forward = self.values.routing.accept_unknown_items; forward.unwrap_or_else(|| !self.processing_enabled()) } - - /// Returns the host and port of the AWS lambda runtime API. - pub fn aws_runtime_api(&self) -> Option<&str> { - self.values.aws.runtime_api.as_deref() - } } impl Default for Config { diff --git a/relay-server/Cargo.toml b/relay-server/Cargo.toml index eabcc7c8fc..6e05ab6f21 100644 --- a/relay-server/Cargo.toml +++ b/relay-server/Cargo.toml @@ -68,7 +68,6 @@ pin-project-lite = { workspace = true } rand = { workspace = true } regex = { workspace = true } relay-auth = { workspace = true } -relay-aws-extension = { workspace = true } relay-base-schema = { workspace = true } relay-cardinality = { workspace = true } relay-cogs = { workspace = true } diff --git a/relay-server/src/service.rs b/relay-server/src/service.rs index 5c82a30566..9971939299 100644 --- a/relay-server/src/service.rs +++ b/relay-server/src/service.rs @@ -6,7 +6,6 @@ use crate::metric_stats::MetricStats; use anyhow::{Context, Result}; use axum::extract::FromRequestParts; use axum::http::request::Parts; -use relay_aws_extension::AwsExtension; use relay_cogs::Cogs; use relay_config::Config; use relay_metrics::Aggregator; @@ -214,12 +213,6 @@ impl ServiceState { .start(); let relay_cache = RelayCacheService::new(config.clone(), upstream_relay.clone()).start(); - if let Some(aws_api) = config.aws_runtime_api() { - if let Ok(aws_extension) = AwsExtension::new(aws_api) { - aws_extension.start(); - } - } - let registry = Registry { aggregator, processor, diff --git a/relay/src/cli.rs b/relay/src/cli.rs index eeb7d416b8..f26485773f 100644 --- a/relay/src/cli.rs +++ b/relay/src/cli.rs @@ -97,7 +97,6 @@ pub fn extract_config_args(matches: &ArgMatches) -> OverridableConfig { secret_key: matches.get_one("secret_key").cloned(), outcome_source: matches.get_one("source_id").cloned(), shutdown_timeout: matches.get_one("shutdown_timeout").cloned(), - aws_runtime_api: matches.get_one("aws_runtime_api").cloned(), } } @@ -118,7 +117,6 @@ pub fn extract_config_env_vars() -> OverridableConfig { secret_key: env::var("RELAY_SECRET_KEY").ok(), outcome_source: None, //already extracted in params shutdown_timeout: env::var("SHUTDOWN_TIMEOUT").ok(), - aws_runtime_api: None, } } diff --git a/relay/src/cliapp.rs b/relay/src/cliapp.rs index cfc2bd9814..187d184281 100644 --- a/relay/src/cliapp.rs +++ b/relay/src/cliapp.rs @@ -141,16 +141,6 @@ pub fn make_app() -> Command { "Maximum number of seconds to wait for pending envelopes on shutdown.", ), ) - .arg( - Arg::new("aws_runtime_api") - .value_name("address") - .long("aws-runtime-api") - .help( - "Host and port of the AWS lambda extensions API, usually provided in \ - the AWS_LAMBDA_RUNTIME_API environment variable. This integrates Relay \ - with the lambda execution environment lifecycle.", - ), - ), ) .subcommand( Command::new("credentials") diff --git a/relay/src/main.rs b/relay/src/main.rs index 8289974651..a36e151d4c 100644 --- a/relay/src/main.rs +++ b/relay/src/main.rs @@ -32,7 +32,6 @@ //! //! - `relay`: Main entry point and command line interface. //! - [`relay-auth`]: Authentication and crypto. -//! - [`relay-aws-extension`]: AWS extension implementation for Sentry's AWS Lambda layer. //! - [`relay-base-schema`]: Basic types for Relay's API schema used across multiple services. //! - [`relay-cabi`]: C-bindings for exposing functionality to Python. //! - [`relay-cardinality`]: Metrics cardinality limiter. @@ -78,7 +77,6 @@ //! //! [relay documentation]: https://docs.sentry.io/product/relay/ //! [`relay-auth`]: ../relay_auth/index.html -//! [`relay-aws-extension`]: ../relay_aws_extension/index.html //! [`relay-base-schema`]: ../relay_base_schema/index.html //! [`relay-cabi`]: ../relay_cabi/index.html //! [`relay-cardinality`]: ../relay_cardinality/index.html diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3521cd9092..308ba0d08d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -12,7 +12,6 @@ from .fixtures.gobetween import gobetween # noqa from .fixtures.haproxy import haproxy # noqa from .fixtures.mini_sentry import mini_sentry # noqa -from .fixtures.aws_lambda_runtime import aws_lambda_runtime # noqa from .fixtures.relay import ( # noqa relay, relay_credentials, diff --git a/tests/integration/fixtures/aws_lambda_runtime.py b/tests/integration/fixtures/aws_lambda_runtime.py deleted file mode 100644 index 3230db0723..0000000000 --- a/tests/integration/fixtures/aws_lambda_runtime.py +++ /dev/null @@ -1,84 +0,0 @@ -import pytest -from urllib.parse import urlparse -from flask import Flask, request as flask_request -from pytest_localserver.http import WSGIServer - - -BASE_URL = "/2020-01-01/extension" -LAMBDA_EXTENSION_ID = "mockedExtensionID" -HEADERS = {"Lambda-Extension-Identifier": LAMBDA_EXTENSION_ID} -SHUTDOWN_MAX = 5 - - -class AwsLambdaRuntime: - def __init__(self, server, app): - self.server = server - self.app = app - self.requests = [] - - def api_netloc(self): - return urlparse(self.server.url).netloc - - def register_url(self): - return f"http://{self.api_netloc()}{BASE_URL}/register" - - def next_event_url(self): - return f"http://{self.api_netloc()}{BASE_URL}/event/next" - - def capture_request(self, request): - self.requests.append(request) - - -@pytest.fixture -def aws_lambda_runtime(request): - app = Flask(__name__) - runtime = None - - @app.before_request - def capture_request(): - runtime.capture_request(flask_request._get_current_object()) - - @app.route(f"{BASE_URL}/register", methods=["POST"]) - def register(): - request_headers = flask_request.headers - request_body = flask_request.get_json() - print(f"Register request headers: {request_headers}") - print(f"Register request body: {request_body}") - - data = { - "functionName": "helloWorld", - "functionVersion": "X.X.X", - "handler": "lambda_function.lambda_handler", - } - - return (data, HEADERS) - - @app.route(f"{BASE_URL}/event/next") - def next_event(): - request_headers = flask_request.headers - print(f"Next request headers: {request_headers}") - - if len(runtime.requests) > SHUTDOWN_MAX: - data = { - "eventType": "SHUTDOWN", - "deadlineMs": 1581512138111, - "shutdownReason": "OOPSIES", - } - else: - data = { - "eventType": "INVOKE", - "deadlineMs": 1581512138111, - "requestId": "aws-request-ID", - "invokedFunctionArn": "invoked-function-arn", - "tracing": { - "type": "X-Amzn-Trace-Id", - "value": "Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=1", - }, - } - return (data, HEADERS) - - server = WSGIServer(application=app, threaded=True) - server.start() - request.addfinalizer(server.stop) - runtime = AwsLambdaRuntime(server, app) - return runtime diff --git a/tests/integration/test_aws_extension.py b/tests/integration/test_aws_extension.py deleted file mode 100644 index 327df28369..0000000000 --- a/tests/integration/test_aws_extension.py +++ /dev/null @@ -1,26 +0,0 @@ -from time import sleep - - -def test_register(aws_lambda_runtime, relay, mini_sentry): - netloc = aws_lambda_runtime.api_netloc() - - relay_options = {"aws": {"runtime_api": netloc}, "limits": {"shutdown_timeout": 2}} - - relay(mini_sentry, options=relay_options, wait_health_check=False) - sleep(2) # wait for clean shutdown, trigerred by 5th next_event request - - requests = aws_lambda_runtime.requests - assert len(requests) == 6 - - register = requests[0] - assert register.url == aws_lambda_runtime.register_url() - assert register.headers["Lambda-Extension-Name"] == "sentry-lambda-extension" - assert register.get_json() == {"events": ["INVOKE", "SHUTDOWN"]} - - next_event = requests[1] - assert next_event.url == aws_lambda_runtime.next_event_url() - assert next_event.headers["Lambda-Extension-Identifier"] == "mockedExtensionID" - - shutdown = requests[-1] - assert shutdown.url == aws_lambda_runtime.next_event_url() - assert shutdown.headers["Lambda-Extension-Identifier"] == "mockedExtensionID"