diff --git a/CHANGELOG.md b/CHANGELOG.md index 81484b7735..e00e63348e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Filter outliers (>180s) for mobile measurements. ([#2649](https://github.com/getsentry/relay/pull/2649)) - Allow access to more context fields in dynamic sampling and metric extraction. ([#2607](https://github.com/getsentry/relay/pull/2607), [#2640](https://github.com/getsentry/relay/pull/2640), [#2675](https://github.com/getsentry/relay/pull/2675)) - Allow advanced scrubbing expressions for datascrubbing safe fields. ([#2670](https://github.com/getsentry/relay/pull/2670)) +- Add context for NEL (Network Error Logging) reports to the event schema. ([#2421](https://github.com/getsentry/relay/pull/2421)) **Bug Fixes**: diff --git a/py/CHANGELOG.md b/py/CHANGELOG.md index 603953f975..bbb91e0f62 100644 --- a/py/CHANGELOG.md +++ b/py/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- Add context for NEL (Network Error Logging) reports to the event schema. ([#2421](https://github.com/getsentry/relay/pull/2421)) + ## 0.8.33 - Drop events starting or ending before January 1, 1970 UTC. ([#2613](https://github.com/getsentry/relay/pull/2613)) @@ -12,6 +16,7 @@ ## 0.8.32 - Add `scraping_attempts` field to the event schema. ([#2575](https://github.com/getsentry/relay/pull/2575)) +- Drop events starting or ending before January 1, 1970 UTC. ([#2613](https://github.com/getsentry/relay/pull/2613)) ## 0.8.31 diff --git a/relay-base-schema/src/data_category.rs b/relay-base-schema/src/data_category.rs index 2268418641..ed29d684dd 100644 --- a/relay-base-schema/src/data_category.rs +++ b/relay-base-schema/src/data_category.rs @@ -160,7 +160,7 @@ impl FromStr for DataCategory { impl From for DataCategory { fn from(ty: EventType) -> Self { match ty { - EventType::Default | EventType::Error => Self::Error, + EventType::Default | EventType::Error | EventType::Nel => Self::Error, EventType::Transaction => Self::Transaction, EventType::Csp | EventType::Hpkp | EventType::ExpectCt | EventType::ExpectStaple => { Self::Security diff --git a/relay-base-schema/src/events.rs b/relay-base-schema/src/events.rs index 0849d6acaa..4283b6889e 100644 --- a/relay-base-schema/src/events.rs +++ b/relay-base-schema/src/events.rs @@ -39,6 +39,8 @@ pub enum EventType { ExpectCt, /// An ExpectStaple violation payload. ExpectStaple, + /// Network Error Logging report. + Nel, /// Performance monitoring transactions carrying spans. Transaction, /// User feedback payload. @@ -74,6 +76,7 @@ impl FromStr for EventType { "hpkp" => EventType::Hpkp, "expectct" => EventType::ExpectCt, "expectstaple" => EventType::ExpectStaple, + "nel" => EventType::Nel, "transaction" => EventType::Transaction, "feedback" => EventType::UserReportV2, _ => return Err(ParseEventTypeError), @@ -90,6 +93,7 @@ impl fmt::Display for EventType { EventType::Hpkp => write!(f, "hpkp"), EventType::ExpectCt => write!(f, "expectct"), EventType::ExpectStaple => write!(f, "expectstaple"), + EventType::Nel => write!(f, "nel"), EventType::Transaction => write!(f, "transaction"), EventType::UserReportV2 => write!(f, "feedback"), } diff --git a/relay-event-normalization/src/normalize/mod.rs b/relay-event-normalization/src/normalize/mod.rs index 4f4728dcc0..f8b262c7cf 100644 --- a/relay-event-normalization/src/normalize/mod.rs +++ b/relay-event-normalization/src/normalize/mod.rs @@ -17,7 +17,7 @@ use relay_event_schema::processor::{ use relay_event_schema::protocol::{ AsPair, Breadcrumb, ClientSdkInfo, Context, ContextInner, Contexts, DebugImage, DeviceClass, Event, EventId, EventType, Exception, Frame, Headers, IpAddr, Level, LogEntry, Measurement, - Measurements, ReplayContext, Request, SpanAttribute, SpanStatus, Stacktrace, Tags, + Measurements, NelContext, ReplayContext, Request, SpanAttribute, SpanStatus, Stacktrace, Tags, TraceContext, User, VALID_PLATFORMS, }; use relay_protocol::{ @@ -36,6 +36,7 @@ use crate::{ }; pub mod breakdowns; +pub mod nel; pub mod span; pub mod user_agent; pub mod utils; @@ -270,6 +271,8 @@ impl<'a> NormalizeProcessor<'a> { EventType::ExpectCt } else if event.expectstaple.value().is_some() { EventType::ExpectStaple + } else if event.context::().is_some() { + EventType::Nel } else { EventType::Default } @@ -774,6 +777,18 @@ fn is_security_report(event: &Event) -> bool { || event.hpkp.value().is_some() } +/// Backfills the client IP address on for the NEL reports. +fn normalize_nel_report(event: &mut Event, client_ip: Option<&IpAddr>) { + if event.context::().is_none() { + return; + } + + if let Some(client_ip) = client_ip { + let user = event.user.value_mut().get_or_insert_with(User::default); + user.ip_address = Annotated::new(client_ip.to_owned()); + } +} + /// Backfills common security report attributes. fn normalize_security_report( event: &mut Event, @@ -1172,6 +1187,9 @@ pub fn light_normalize_event( // Process security reports first to ensure all props. normalize_security_report(event, config.client_ip, &config.user_agent); + // Process NEL reports to ensure all props. + normalize_nel_report(event, config.client_ip); + // Insert IP addrs before recursing, since geo lookup depends on it. normalize_ip_addresses( &mut event.request, diff --git a/relay-event-normalization/src/normalize/nel.rs b/relay-event-normalization/src/normalize/nel.rs new file mode 100644 index 0000000000..8401188ca6 --- /dev/null +++ b/relay-event-normalization/src/normalize/nel.rs @@ -0,0 +1,90 @@ +//! Contains helper function for NEL reports. + +use chrono::{Duration, Utc}; +use relay_event_schema::protocol::{ + Contexts, Event, HeaderName, HeaderValue, Headers, LogEntry, NelContext, NetworkReportRaw, + Request, ResponseContext, Timestamp, +}; +use relay_protocol::Annotated; + +/// Enriches the event with new values using the provided [`NetworkReportRaw`]. +pub fn enrich_event(event: &mut Event, nel: Annotated) { + // If the incoming NEL report is empty or it contains an empty body, just exit. + let Some(nel) = nel.into_value() else { + return; + }; + let Some(body) = nel.body.into_value() else { + return; + }; + + event.logger = Annotated::from("nel".to_string()); + + event.logentry = Annotated::new(LogEntry::from({ + if nel.ty.value().map_or("", |v| v.as_str()) == "http.error" { + format!( + "{} / {} ({})", + body.phase.as_str().unwrap_or(""), + body.ty.as_str().unwrap_or(""), + body.status_code.value().unwrap_or(&0) + ) + } else { + format!( + "{} / {}", + body.phase.as_str().unwrap_or(""), + body.ty.as_str().unwrap_or(""), + ) + } + })); + + let request = event.request.get_or_insert_with(Request::default); + request.url = nel.url; + request.method = body.method; + request.protocol = body.protocol; + + let headers = request.headers.get_or_insert_with(Headers::default); + + if let Some(ref user_agent) = nel.user_agent.value() { + if !user_agent.is_empty() { + headers.insert( + HeaderName::new("user-agent"), + HeaderValue::new(user_agent).into(), + ); + } + } + + if let Some(referrer) = body.referrer.value() { + headers.insert( + HeaderName::new("referer"), + HeaderValue::new(referrer).into(), + ); + } + + let contexts = event.contexts.get_or_insert_with(Contexts::new); + + let nel_context = contexts.get_or_default::(); + nel_context.server_ip = body.server_ip; + nel_context.elapsed_time = body.elapsed_time; + nel_context.error_type = body.ty; + nel_context.phase = body.phase; + nel_context.sampling_fraction = body.sampling_fraction; + + // Set response status code only if it's bigger than zero. + let status_code = body + .status_code + .map_value(|v| u64::try_from(v).unwrap_or(0)); + if status_code.value().unwrap_or(&0) > &0 { + let response_context = contexts.get_or_default::(); + response_context.status_code = status_code; + } + + // Set the timestamp on the event when it actually occurred. + let event_time = event + .timestamp + .value_mut() + .map_or(Utc::now(), |timestamp| timestamp.into_inner()); + if let Some(event_time) = + event_time.checked_sub_signed(Duration::milliseconds(*nel.age.value().unwrap_or(&0))) + { + event.timestamp = Annotated::new(Timestamp::from(event_time)) + } +} diff --git a/relay-event-schema/src/protocol/contexts/mod.rs b/relay-event-schema/src/protocol/contexts/mod.rs index 919050efd2..5c2b8222be 100644 --- a/relay-event-schema/src/protocol/contexts/mod.rs +++ b/relay-event-schema/src/protocol/contexts/mod.rs @@ -4,6 +4,7 @@ mod cloud_resource; mod device; mod gpu; mod monitor; +mod nel; mod os; mod otel; mod profile; @@ -19,6 +20,7 @@ pub use cloud_resource::*; pub use device::*; pub use gpu::*; pub use monitor::*; +pub use nel::*; pub use os::*; pub use otel::*; pub use profile::*; @@ -82,6 +84,8 @@ pub enum Context { Otel(Box), /// Cloud resource information. CloudResource(Box), + /// Nel information. + Nel(Box), /// Additional arbitrary fields for forwards compatibility. #[metastructure(fallback_variant)] Other(#[metastructure(pii = "true")] Object), diff --git a/relay-event-schema/src/protocol/contexts/nel.rs b/relay-event-schema/src/protocol/contexts/nel.rs new file mode 100644 index 0000000000..753361b78a --- /dev/null +++ b/relay-event-schema/src/protocol/contexts/nel.rs @@ -0,0 +1,63 @@ +#[cfg(feature = "jsonschema")] +use relay_jsonschema_derive::JsonSchema; +use relay_protocol::{Annotated, Empty, FromValue, IntoValue, Object, Value}; + +use crate::processor::ProcessValue; +use crate::protocol::{IpAddr, NetworkReportPhases}; + +/// Contains NEL report information. +/// +/// Network Error Logging (NEL) is a browser feature that allows reporting of failed network +/// requests from the client side. See the following resources for more information: +/// +/// - [W3C Editor's Draft](https://w3c.github.io/network-error-logging/) +/// - [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Network_Error_Logging) +#[derive(Clone, Debug, Default, PartialEq, Empty, FromValue, IntoValue, ProcessValue)] +#[cfg_attr(feature = "jsonschema", derive(JsonSchema))] +pub struct NelContext { + /// If request failed, the type of its network error. If request succeeded, "ok". + pub error_type: Annotated, + /// Server IP where the requests was sent to. + #[metastructure(pii = "maybe")] + pub server_ip: Annotated, + /// The number of milliseconds between the start of the resource fetch and when it was aborted by the user agent. + pub elapsed_time: Annotated, + /// If request failed, the phase of its network error. If request succeeded, "application". + pub phase: Annotated, + /// The sampling rate. + pub sampling_fraction: Annotated, + /// For forward compatibility. + #[metastructure(additional_properties, pii = "maybe")] + pub other: Object, +} + +impl super::DefaultContext for NelContext { + fn default_key() -> &'static str { + "nel" + } + + fn from_context(context: super::Context) -> Option { + match context { + super::Context::Nel(c) => Some(*c), + _ => None, + } + } + + fn cast(context: &super::Context) -> Option<&Self> { + match context { + super::Context::Nel(c) => Some(c), + _ => None, + } + } + + fn cast_mut(context: &mut super::Context) -> Option<&mut Self> { + match context { + super::Context::Nel(c) => Some(c), + _ => None, + } + } + + fn into_context(self) -> super::Context { + super::Context::Nel(Box::new(self)) + } +} diff --git a/relay-event-schema/src/protocol/mod.rs b/relay-event-schema/src/protocol/mod.rs index 1a6dee166c..83a55aca8a 100644 --- a/relay-event-schema/src/protocol/mod.rs +++ b/relay-event-schema/src/protocol/mod.rs @@ -16,6 +16,7 @@ mod logentry; mod measurements; mod mechanism; mod metrics; +mod nel; mod relay_info; mod replay; mod request; @@ -52,6 +53,7 @@ pub use self::logentry::*; pub use self::measurements::*; pub use self::mechanism::*; pub use self::metrics::*; +pub use self::nel::*; pub use self::relay_info::*; pub use self::replay::*; pub use self::request::*; diff --git a/relay-event-schema/src/protocol/nel.rs b/relay-event-schema/src/protocol/nel.rs new file mode 100644 index 0000000000..9758f6acb9 --- /dev/null +++ b/relay-event-schema/src/protocol/nel.rs @@ -0,0 +1,233 @@ +//! Contains definitions for the Network Error Logging (NEL) interface. +//! +//! See: [`crate::protocol::contexts::NelContext`]. + +use std::fmt; +use std::str::FromStr; + +#[cfg(feature = "jsonschema")] +use relay_jsonschema_derive::JsonSchema; +use relay_protocol::{Annotated, Empty, FromValue, IntoValue, Object, Value}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::processor::ProcessValue; +use crate::protocol::IpAddr; + +/// Describes which phase the error occurred in. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ProcessValue)] +#[cfg_attr(feature = "jsonschema", derive(JsonSchema))] +#[serde(rename_all = "lowercase")] +pub enum NetworkReportPhases { + /// The error occurred during DNS resolution. + DNS, + /// The error occurred during secure connection establishment. + Connections, + /// The error occurred during the transmission of request and response . + Application, + /// For forward-compatibility. + Other(String), +} + +impl NetworkReportPhases { + /// Creates the string representation of the current enum value. + pub fn as_str(&self) -> &str { + match *self { + NetworkReportPhases::DNS => "dns", + NetworkReportPhases::Connections => "connection", + NetworkReportPhases::Application => "application", + NetworkReportPhases::Other(ref unknown) => unknown, + } + } +} + +impl fmt::Display for NetworkReportPhases { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl AsRef for NetworkReportPhases { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl Empty for NetworkReportPhases { + #[inline] + fn is_empty(&self) -> bool { + false + } +} + +/// Error parsing a [`NetworkReportPhases`]. +#[derive(Clone, Copy, Debug)] +pub struct ParseNetworkReportPhaseError; + +impl fmt::Display for ParseNetworkReportPhaseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "invalid network report phase") + } +} + +impl FromStr for NetworkReportPhases { + type Err = ParseNetworkReportPhaseError; + + fn from_str(s: &str) -> Result { + Ok(match s.to_lowercase().as_str() { + "dns" => NetworkReportPhases::DNS, + "connection" => NetworkReportPhases::Connections, + "application" => NetworkReportPhases::Application, + unknown => NetworkReportPhases::Other(unknown.to_string()), + }) + } +} + +impl FromValue for NetworkReportPhases { + fn from_value(value: Annotated) -> Annotated { + match value { + Annotated(Some(Value::String(value)), mut meta) => match value.parse() { + Ok(phase) => Annotated(Some(phase), meta), + Err(_) => { + meta.add_error(relay_protocol::Error::expected("a string")); + meta.set_original_value(Some(value)); + Annotated(None, meta) + } + }, + Annotated(None, meta) => Annotated(None, meta), + Annotated(Some(value), mut meta) => { + meta.add_error(relay_protocol::Error::expected("a string")); + meta.set_original_value(Some(value)); + Annotated(None, meta) + } + } + } +} + +impl IntoValue for NetworkReportPhases { + fn into_value(self) -> Value { + Value::String(self.to_string()) + } + + fn serialize_payload( + &self, + s: S, + _behavior: relay_protocol::SkipSerialization, + ) -> Result + where + Self: Sized, + S: serde::Serializer, + { + Serialize::serialize(&self.to_string(), s) + } +} + +/// The NEL parsing errors. +#[derive(Debug, Error)] +pub enum NetworkReportError { + /// Incoming Json is unparsable. + #[error("incoming json is unparsable")] + InvalidJson(#[from] serde_json::Error), +} + +/// Generated network error report (NEL). +#[derive(Debug, Default, Clone, PartialEq, FromValue, IntoValue, Empty)] +#[cfg_attr(feature = "jsonschema", derive(JsonSchema))] +pub struct BodyRaw { + /// The time between the start of the resource fetch and when it was completed or aborted. + pub elapsed_time: Annotated, + /// HTTP method. + pub method: Annotated, + /// If request failed, the phase of its network error. If request succeeded, "application". + pub phase: Annotated, + /// The HTTP protocol and version. + pub protocol: Annotated, + /// Request's referrer, as determined by the referrer policy associated with its client. + pub referrer: Annotated, + /// The sampling rate. + pub sampling_fraction: Annotated, + /// The IP address of the server where the site is hosted. + pub server_ip: Annotated, + /// HTTP status code. + pub status_code: Annotated, + /// If request failed, the type of its network error. If request succeeded, "ok". + #[metastructure(field = "type")] + pub ty: Annotated, + /// For forward compatibility. + #[metastructure(additional_properties, pii = "maybe")] + pub other: Object, +} + +/// Models the content of a NEL report. +/// +/// See +#[derive(Debug, Default, Clone, PartialEq, FromValue, IntoValue, Empty)] +#[cfg_attr(feature = "jsonschema", derive(JsonSchema))] +pub struct NetworkReportRaw { + /// The age of the report since it got collected and before it got sent. + pub age: Annotated, + /// The type of the report. + #[metastructure(field = "type")] + pub ty: Annotated, + /// The URL of the document in which the error occurred. + #[metastructure(pii = "true")] + pub url: Annotated, + /// The User-Agent HTTP header. + pub user_agent: Annotated, + /// The body of the NEL report. + pub body: Annotated, + /// For forward compatibility. + #[metastructure(additional_properties, pii = "maybe")] + pub other: Object, +} + +#[cfg(test)] +mod tests { + use relay_protocol::{assert_annotated_snapshot, Annotated}; + + use crate::protocol::NetworkReportRaw; + + #[test] + fn test_nel_raw_basic() { + let json = r#"{ + "age": 31042, + "body": { + "elapsed_time": 0, + "method": "GET", + "phase": "connection", + "protocol": "http/1.1", + "referrer": "", + "sampling_fraction": 1.0, + "server_ip": "127.0.0.1", + "status_code": 0, + "type": "tcp.refused" + }, + "type": "network-error", + "url": "http://example.com/", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" + }"#; + + let report: Annotated = + Annotated::from_json_bytes(json.as_bytes()).unwrap(); + + assert_annotated_snapshot!(report, @r###" + { + "age": 31042, + "type": "network-error", + "url": "http://example.com/", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", + "body": { + "elapsed_time": 0, + "method": "GET", + "phase": "connection", + "protocol": "http/1.1", + "referrer": "", + "sampling_fraction": 1.0, + "server_ip": "127.0.0.1", + "status_code": 0, + "type": "tcp.refused" + } + } + "###); + } +} diff --git a/relay-event-schema/src/protocol/request.rs b/relay-event-schema/src/protocol/request.rs index 88a26c1275..2114188249 100644 --- a/relay-event-schema/src/protocol/request.rs +++ b/relay-event-schema/src/protocol/request.rs @@ -444,6 +444,9 @@ pub struct Request { /// HTTP request method. pub method: Annotated, + /// HTTP protocol. + pub protocol: Annotated, + /// Request data in any format that makes sense. /// /// SDKs should discard large and binary bodies by default. Can be given as a string or @@ -684,6 +687,7 @@ mod tests { let request = Annotated::new(Request { url: Annotated::new("https://google.com/search".to_string()), method: Annotated::new("GET".to_string()), + protocol: Annotated::empty(), data: { let mut map = Object::new(); map.insert("some".to_string(), Annotated::new(Value::I64(1))); diff --git a/relay-pii/src/snapshots/relay_pii__processor__tests__does_not_scrub_if_no_graphql.snap b/relay-pii/src/snapshots/relay_pii__processor__tests__does_not_scrub_if_no_graphql.snap index 1445f33a32..e50de50dd9 100644 --- a/relay-pii/src/snapshots/relay_pii__processor__tests__does_not_scrub_if_no_graphql.snap +++ b/relay-pii/src/snapshots/relay_pii__processor__tests__does_not_scrub_if_no_graphql.snap @@ -28,6 +28,7 @@ Event { request: Request { url: ~, method: ~, + protocol: ~, data: Object( { "query": String( diff --git a/relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_with_variables.snap b/relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_with_variables.snap index 7172c39d51..4db6570675 100644 --- a/relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_with_variables.snap +++ b/relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_with_variables.snap @@ -28,6 +28,7 @@ Event { request: Request { url: ~, method: ~, + protocol: ~, data: Object( { "query": String( diff --git a/relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_without_variables.snap b/relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_without_variables.snap index 21e17a8153..4076a0cf9f 100644 --- a/relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_without_variables.snap +++ b/relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_without_variables.snap @@ -28,6 +28,7 @@ Event { request: Request { url: ~, method: ~, + protocol: ~, data: Object( { "query": String( diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index 81736c331e..645ad7769a 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -24,16 +24,16 @@ use relay_dynamic_config::{ }; use relay_event_normalization::replay::{self, ReplayError}; use relay_event_normalization::{ - ClockDriftProcessor, DynamicMeasurementsConfig, LightNormalizationConfig, MeasurementsConfig, - TransactionNameConfig, + nel, ClockDriftProcessor, DynamicMeasurementsConfig, LightNormalizationConfig, + MeasurementsConfig, TransactionNameConfig, }; use relay_event_normalization::{GeoIpLookup, RawUserAgentInfo}; use relay_event_schema::processor::{self, ProcessingAction, ProcessingState}; use relay_event_schema::protocol::{ Breadcrumb, ClientReport, Contexts, Csp, Event, EventType, ExpectCt, ExpectStaple, Hpkp, - IpAddr, LenientString, Metrics, OtelContext, RelayInfo, Replay, SecurityReportType, - SessionAggregates, SessionAttributes, SessionStatus, SessionUpdate, Timestamp, TraceContext, - UserReport, Values, + IpAddr, LenientString, Metrics, NetworkReportError, OtelContext, RelayInfo, Replay, + SecurityReportType, SessionAggregates, SessionAttributes, SessionStatus, SessionUpdate, + Timestamp, TraceContext, UserReport, Values, }; use relay_filter::FilterStatKey; use relay_metrics::{Bucket, MergeBuckets, MetricNamespace}; @@ -120,6 +120,9 @@ pub enum ProcessingError { #[error("invalid security report")] InvalidSecurityReport(#[source] serde_json::Error), + #[error("invalid nel report")] + InvalidNelReport(#[source] NetworkReportError), + #[error("event filtered with reason: {0:?}")] EventFiltered(FilterStatKey), @@ -151,6 +154,7 @@ impl ProcessingError { Some(Outcome::Invalid(DiscardReason::SecurityReportType)) } Self::InvalidSecurityReport(_) => Some(Outcome::Invalid(DiscardReason::SecurityReport)), + Self::InvalidNelReport(_) => Some(Outcome::Invalid(DiscardReason::InvalidJson)), Self::InvalidTransaction => Some(Outcome::Invalid(DiscardReason::InvalidTransaction)), Self::InvalidTimestamp => Some(Outcome::Invalid(DiscardReason::Timestamp)), Self::DuplicateItem(_) => Some(Outcome::Invalid(DiscardReason::DuplicateItem)), @@ -1495,6 +1499,39 @@ impl EnvelopeProcessorService { Ok((Annotated::new(event), len)) } + fn event_from_nel_item( + &self, + item: Item, + _meta: &RequestMeta, + ) -> Result { + let len = item.len(); + let mut event = Event { + ty: Annotated::new(EventType::Nel), + ..Default::default() + }; + let data: &[u8] = &item.payload(); + + // Try to get the raw network report. + let report = Annotated::from_json_bytes(data).map_err(NetworkReportError::InvalidJson); + + match report { + // If the incoming payload could be converted into the raw network error, try + // to use it to normalize the event. + Ok(report) => { + nel::enrich_event(&mut event, report); + } + Err(err) => { + // logged in extract_event + relay_log::configure_scope(|scope| { + scope.set_extra("payload", String::from_utf8_lossy(data).into()); + }); + return Err(ProcessingError::InvalidNelReport(err)); + } + } + + Ok((Annotated::new(event), len)) + } + fn merge_formdata(&self, target: &mut SerdeValue, item: Item) { let payload = item.payload(); let mut aggregator = ChunkedFormDataAggregator::new(); @@ -1652,6 +1689,7 @@ impl EnvelopeProcessorService { // These may be forwarded to upstream / store: ItemType::Attachment => false, + ItemType::Nel => false, ItemType::UserReport => false, // Aggregate data is never considered as part of deduplication @@ -1689,9 +1727,9 @@ impl EnvelopeProcessorService { let transaction_item = envelope.take_item_by(|item| item.ty() == &ItemType::Transaction); let security_item = envelope.take_item_by(|item| item.ty() == &ItemType::Security); let raw_security_item = envelope.take_item_by(|item| item.ty() == &ItemType::RawSecurity); + let nel_item = envelope.take_item_by(|item| item.ty() == &ItemType::Nel); let user_report_v2_item = envelope.take_item_by(|item| item.ty() == &ItemType::UserReportV2); - let form_item = envelope.take_item_by(|item| item.ty() == &ItemType::FormData); let attachment_item = envelope .take_item_by(|item| item.attachment_type() == Some(&AttachmentType::EventPayload)); @@ -1742,6 +1780,13 @@ impl EnvelopeProcessorService { ); error })? + } else if let Some(item) = nel_item { + relay_log::trace!("processing nel report"); + self.event_from_nel_item(item, envelope.meta()) + .map_err(|error| { + relay_log::error!(error = &error as &dyn Error, "failed to extract NEL report"); + error + })? } else if attachment_item.is_some() || breadcrumbs1.is_some() || breadcrumbs2.is_some() { relay_log::trace!("extracting attached event data"); Self::event_from_attachments( diff --git a/relay-server/src/endpoints/common.rs b/relay-server/src/endpoints/common.rs index 20039a16a2..26455c76d8 100644 --- a/relay-server/src/endpoints/common.rs +++ b/relay-server/src/endpoints/common.rs @@ -278,6 +278,23 @@ fn queue_envelope( }); } + // Take all NEL reports and split them up into the separate envelopes with 1 item per + // envelope. + for nel_envelope in envelope.split_all_by(|item| matches!(item.ty(), ItemType::Nel)) { + relay_log::trace!("queueing separate envelopes for NEL report"); + let buffer_guard = state.buffer_guard(); + let nel_envelope = buffer_guard + .enter( + nel_envelope, + state.outcome_aggregator().clone(), + state.test_store().clone(), + ) + .map_err(BadStoreRequest::QueueFailed)?; + state + .project_cache() + .send(ValidateEnvelope::new(nel_envelope)); + } + // Split the envelope into event-related items and other items. This allows to fast-track: // 1. Envelopes with only session items. They only require rate limiting. // 2. Event envelope processing can bail out if the event is filtered or rate limited, @@ -291,14 +308,14 @@ fn queue_envelope( state.outcome_aggregator().clone(), state.test_store().clone(), )?; - - // Update the old context after successful forking. - managed_envelope.update(); state .project_cache() .send(ValidateEnvelope::new(event_context)); } + // Update the old context before continuing with the source envelope. + managed_envelope.update(); + if managed_envelope.envelope().is_empty() { // The envelope can be empty here if it contained only metrics items which were removed // above. In this case, the envelope was accepted and needs no further queueing. diff --git a/relay-server/src/endpoints/mod.rs b/relay-server/src/endpoints/mod.rs index e887ae10f9..6f9cdc17b6 100644 --- a/relay-server/src/endpoints/mod.rs +++ b/relay-server/src/endpoints/mod.rs @@ -15,6 +15,7 @@ mod health_check; mod logs; mod minidump; mod monitor; +mod nel; mod outcomes; mod project_configs; mod public_keys; @@ -78,6 +79,7 @@ where .route("/api/:project_id/envelope/", envelope::route(config)) .route("/api/:project_id/security/", security_report::route(config)) .route("/api/:project_id/csp-report/", security_report::route(config)) + .route("/api/:project_id/nel/", nel::route(config)) // No mandatory trailing slash here because people already use it like this. .route("/api/:project_id/minidump", minidump::route(config)) .route("/api/:project_id/minidump/", minidump::route(config)) diff --git a/relay-server/src/endpoints/nel.rs b/relay-server/src/endpoints/nel.rs new file mode 100644 index 0000000000..28aabb4e9c --- /dev/null +++ b/relay-server/src/endpoints/nel.rs @@ -0,0 +1,69 @@ +//! Endpoint for Network Error Logging (NEL) reports. +//! +//! It split list of incoming events from the envelope into separate envelope with 1 item inside. +//! Which later get failed by the service infrastructure. + +use axum::extract::{DefaultBodyLimit, FromRequest}; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use axum::routing::{post, MethodRouter}; +use bytes::Bytes; +use relay_config::Config; +use relay_event_schema::protocol::EventId; +use serde_json::value::RawValue; + +use crate::endpoints::common::{self, BadStoreRequest}; +use crate::envelope::{ContentType, Envelope, Item, ItemType}; +use crate::extractors::{Mime, RequestMeta}; +use crate::service::ServiceState; + +#[derive(Debug, FromRequest)] +#[from_request(state(ServiceState))] +struct NelReportParams { + meta: RequestMeta, + body: Bytes, +} + +fn is_nel_mime(mime: Mime) -> bool { + let ty = mime.type_().as_str(); + let subty = mime.subtype().as_str(); + let suffix = mime.suffix().map(|suffix| suffix.as_str()); + + matches!( + (ty, subty, suffix), + ("application", "json", None) | ("application", "reports", Some("json")) + ) +} + +/// Handles all messages coming on the NEL endpoint. +async fn handle( + state: ServiceState, + mime: Mime, + params: NelReportParams, +) -> Result { + if !is_nel_mime(mime) { + return Ok(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()); + } + + let items: Vec<&RawValue> = + serde_json::from_slice(¶ms.body).map_err(BadStoreRequest::InvalidJson)?; + + let mut envelope = Envelope::from_request(Some(EventId::new()), params.meta.clone()); + for item in items { + let mut report_item = Item::new(ItemType::Nel); + report_item.set_payload(ContentType::Json, item.to_owned().to_string()); + envelope.add_item(report_item); + } + + common::handle_envelope(&state, envelope).await?; + Ok(().into_response()) +} + +pub fn route(config: &Config) -> MethodRouter +where + B: axum::body::HttpBody + Send + 'static, + B::Data: Send, + B::Error: Into, +{ + post(handle).route_layer(DefaultBodyLimit::max(config.max_event_size())) +} diff --git a/relay-server/src/envelope.rs b/relay-server/src/envelope.rs index 8132ca066e..a0679c531a 100644 --- a/relay-server/src/envelope.rs +++ b/relay-server/src/envelope.rs @@ -46,7 +46,7 @@ use relay_quotas::DataCategory; use relay_sampling::DynamicSamplingContext; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use smallvec::SmallVec; +use smallvec::{smallvec, SmallVec}; use crate::constants::DEFAULT_EVENT_RETENTION; use crate::extractors::{PartialMeta, RequestMeta}; @@ -88,6 +88,8 @@ pub enum ItemType { FormData, /// Security report as sent by the browser in JSON. RawSecurity, + /// NEL report as sent by the browser. + Nel, /// Raw compressed UE4 crash report. UnrealReport, /// User feedback encoded as JSON. @@ -127,7 +129,7 @@ impl ItemType { /// Returns the event item type corresponding to the given `EventType`. pub fn from_event_type(event_type: EventType) -> Self { match event_type { - EventType::Default | EventType::Error => ItemType::Event, + EventType::Default | EventType::Error | EventType::Nel => ItemType::Event, EventType::Transaction => ItemType::Transaction, EventType::UserReportV2 => ItemType::UserReportV2, EventType::Csp | EventType::Hpkp | EventType::ExpectCt | EventType::ExpectStaple => { @@ -146,6 +148,7 @@ impl fmt::Display for ItemType { Self::Attachment => write!(f, "attachment"), Self::FormData => write!(f, "form_data"), Self::RawSecurity => write!(f, "raw_security"), + Self::Nel => write!(f, "nel"), Self::UnrealReport => write!(f, "unreal_report"), Self::UserReport => write!(f, "user_report"), Self::UserReportV2 => write!(f, "feedback"), @@ -175,6 +178,7 @@ impl std::str::FromStr for ItemType { "attachment" => Self::Attachment, "form_data" => Self::FormData, "raw_security" => Self::RawSecurity, + "nel" => Self::Nel, "unreal_report" => Self::UnrealReport, "user_report" => Self::UserReport, "feedback" => Self::UserReportV2, @@ -555,6 +559,7 @@ impl Item { DataCategory::Transaction }), ItemType::Security | ItemType::RawSecurity => Some(DataCategory::Security), + ItemType::Nel => None, ItemType::UnrealReport => Some(DataCategory::Error), ItemType::Attachment => Some(DataCategory::Attachment), ItemType::Session | ItemType::Sessions => None, @@ -706,6 +711,7 @@ impl Item { | ItemType::Transaction | ItemType::Security | ItemType::RawSecurity + | ItemType::Nel | ItemType::UnrealReport | ItemType::UserReportV2 => true, @@ -763,6 +769,7 @@ impl Item { ItemType::Attachment => true, ItemType::FormData => true, ItemType::RawSecurity => true, + ItemType::Nel => false, ItemType::UnrealReport => true, ItemType::UserReport => true, ItemType::UserReportV2 => true, @@ -1078,19 +1085,17 @@ impl Envelope { self.items.push(item) } - /// Splits the envelope by the given predicate. - /// - /// The predicate passed to `split_by()` can return `true`, or `false`. If it returns `true` or - /// `false` for all items, then this returns `None`. Otherwise, a new envelope is constructed - /// with all items that return `true`. Items that return `false` remain in this envelope. + /// Splits off the items from the envelope using provided predicates. /// - /// The returned envelope assumes the same headers. - pub fn split_by(&mut self, mut f: F) -> Option> + /// First predicate is the the additional condition on the count of found items by second + /// predicate. + fn split_off_items(&mut self, cond: C, mut f: F) -> Option> where + C: Fn(usize) -> bool, F: FnMut(&Item) -> bool, { let split_count = self.items().filter(|item| f(item)).count(); - if split_count == self.len() || split_count == 0 { + if cond(split_count) { return None; } @@ -1098,12 +1103,59 @@ impl Envelope { let (split_items, own_items) = old_items.into_iter().partition(f); self.items = own_items; + Some(split_items) + } + + /// Splits the envelope by the given predicate. + /// + /// The predicate passed to `split_by()` can return `true`, or `false`. If it returns `true` or + /// `false` for all items, then this returns `None`. Otherwise, a new envelope is constructed + /// with all items that return `true`. Items that return `false` remain in this envelope. + /// + /// The returned envelope assumes the same headers. + pub fn split_by(&mut self, f: F) -> Option> + where + F: FnMut(&Item) -> bool, + { + let items_count = self.len(); + let split_items = self.split_off_items(|count| count == 0 || count == items_count, f)?; Some(Box::new(Envelope { headers: self.headers.clone(), items: split_items, })) } + /// Splits the envelope by the given predicate. + /// + /// The main differents from `split_by()` is this function returns the list of the newly + /// constracted envelopes with all the items where the predicate returns `true`. Otherwise it + /// returns an empty list. + /// + /// The returned envelopes assume the same headers. + pub fn split_all_by(&mut self, f: F) -> SmallVec<[Box; 3]> + where + F: FnMut(&Item) -> bool, + { + let mut envelopes = smallvec![]; + let Some(split_items) = self.split_off_items(|count| count == 0, f) else { + return envelopes; + }; + + let headers = &mut self.headers; + + for item in split_items { + // Each item should get an envelope with the new event id. + headers.event_id = Some(EventId::new()); + + envelopes.push(Box::new(Envelope { + items: smallvec![item], + headers: headers.clone(), + })) + } + + envelopes + } + /// Retains only the items specified by the predicate. /// /// In other words, remove all elements where `f(&item)` returns `false`. This method operates diff --git a/relay-server/src/utils/rate_limits.rs b/relay-server/src/utils/rate_limits.rs index a1aa37c726..80c868b7da 100644 --- a/relay-server/src/utils/rate_limits.rs +++ b/relay-server/src/utils/rate_limits.rs @@ -93,6 +93,7 @@ fn infer_event_category(item: &Item) -> Option { ItemType::Event => Some(DataCategory::Error), ItemType::Transaction => Some(DataCategory::Transaction), ItemType::Security | ItemType::RawSecurity => Some(DataCategory::Security), + ItemType::Nel => Some(DataCategory::Error), ItemType::UnrealReport => Some(DataCategory::Error), ItemType::UserReportV2 => Some(DataCategory::UserReportV2), ItemType::Attachment if item.creates_event() => Some(DataCategory::Error), diff --git a/relay-server/src/utils/sizes.rs b/relay-server/src/utils/sizes.rs index 11dd5157c6..9b5d610443 100644 --- a/relay-server/src/utils/sizes.rs +++ b/relay-server/src/utils/sizes.rs @@ -29,6 +29,7 @@ pub fn check_envelope_size_limits(config: &Config, envelope: &Envelope) -> bool | ItemType::Security | ItemType::ReplayEvent | ItemType::RawSecurity + | ItemType::Nel | ItemType::UserReportV2 | ItemType::FormData => { event_size += item.len(); diff --git a/relay-server/tests/snapshots/test_fixtures__event_schema.snap b/relay-server/tests/snapshots/test_fixtures__event_schema.snap index c8af3400e3..780b234760 100644 --- a/relay-server/tests/snapshots/test_fixtures__event_schema.snap +++ b/relay-server/tests/snapshots/test_fixtures__event_schema.snap @@ -928,6 +928,9 @@ expression: "relay_event_schema::protocol::event_json_schema()" { "$ref": "#/definitions/CloudResourceContext" }, + { + "$ref": "#/definitions/NelContext" + }, { "type": "object", "additionalProperties": true @@ -1494,6 +1497,7 @@ expression: "relay_event_schema::protocol::event_json_schema()" "hpkp", "expectct", "expectstaple", + "nel", "transaction", "userreportv2", "default" @@ -2590,6 +2594,88 @@ expression: "relay_event_schema::protocol::event_json_schema()" } ] }, + "NelContext": { + "description": " Contains NEL report information.\n\n Network Error Logging (NEL) is a browser feature that allows reporting of failed network\n requests from the client side. See the following resources for more information:\n\n - [W3C Editor's Draft](https://w3c.github.io/network-error-logging/)\n - [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Network_Error_Logging)", + "anyOf": [ + { + "type": "object", + "properties": { + "elapsed_time": { + "description": " The number of milliseconds between the start of the resource fetch and when it was aborted by the user agent.", + "default": null, + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "error_type": { + "description": " If request failed, the type of its network error. If request succeeded, \"ok\".", + "default": null, + "type": [ + "string", + "null" + ] + }, + "phase": { + "description": " If request failed, the phase of its network error. If request succeeded, \"application\".", + "default": null, + "anyOf": [ + { + "$ref": "#/definitions/NetworkReportPhases" + }, + { + "type": "null" + } + ] + }, + "sampling_fraction": { + "description": " The sampling rate.", + "default": null, + "type": [ + "number", + "null" + ], + "format": "double" + }, + "server_ip": { + "description": " Server IP where the requests was sent to.", + "default": null, + "anyOf": [ + { + "$ref": "#/definitions/String" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + ] + }, + "NetworkReportPhases": { + "description": " Describes which phase the error occurred in.", + "anyOf": [ + { + "type": "object", + "additionalProperties": false + }, + { + "type": "object", + "additionalProperties": false + }, + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string" + } + ] + }, "NsError": { "description": " NSError informaiton.", "anyOf": [ @@ -2986,6 +3072,14 @@ expression: "relay_event_schema::protocol::event_json_schema()" "null" ] }, + "protocol": { + "description": " HTTP protocol.", + "default": null, + "type": [ + "string", + "null" + ] + }, "query_string": { "description": " The query string component of the URL.\n\n Can be given as unparsed string, dictionary, or list of tuples.\n\n If the query string is not declared and part of the `url`, Sentry moves it to the\n query string.", "default": null, diff --git a/tests/integration/fixtures/__init__.py b/tests/integration/fixtures/__init__.py index 99e126bd7f..d7471e6336 100644 --- a/tests/integration/fixtures/__init__.py +++ b/tests/integration/fixtures/__init__.py @@ -154,6 +154,56 @@ def send_event( response.raise_for_status() return response.json() + def send_nel_event( + self, + project_id, + payload=None, + headers=None, + dsn_key_idx=0, + dsn_key=None, + ): + + if payload is None: + payload = [ + { + "age": 1200000, + "body": { + "elapsed_time": 37, + "method": "GET", + "phase": "application", + "protocol": "http/1.1", + "referrer": "https://example.com/nel/", + "sampling_fraction": 1, + "server_ip": "123.123.123.123", + "status_code": 500, + "type": "http.error", + }, + "type": "network-error", + "url": "https://example.com/index.html", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36", + } + ] + + if isinstance(payload, list): + kwargs = {"json": payload} + else: + raise ValueError(f"Invalid type {type(payload)} for payload.") + + headers = { + "Content-Type": "application/reports+json", + **(headers or {}), + } + + if dsn_key is None: + dsn_key = self.get_dsn_public_key(project_id, dsn_key_idx) + + url = f"/api/{project_id}/nel/?sentry_key={dsn_key}" + + response = self.post(url, headers=headers, **kwargs) + response.raise_for_status() + + return + def send_options(self, project_id, headers=None, dsn_key_idx=0): headers = { "X-Sentry-Auth": self.get_auth_header(project_id, dsn_key_idx), diff --git a/tests/integration/fixtures/processing.py b/tests/integration/fixtures/processing.py index 2f21a0d055..abec32ae44 100644 --- a/tests/integration/fixtures/processing.py +++ b/tests/integration/fixtures/processing.py @@ -338,8 +338,8 @@ def get_session(self): class EventsConsumer(ConsumerBase): - def get_event(self): - message = self.poll() + def get_event(self, timeout=None): + message = self.poll(timeout) assert message is not None assert message.error() is None diff --git a/tests/integration/test_store.py b/tests/integration/test_store.py index 89f30d7293..e6b396fd7b 100644 --- a/tests/integration/test_store.py +++ b/tests/integration/test_store.py @@ -375,7 +375,7 @@ def test_processing( @pytest.mark.parametrize( "window,max_rate_limit", [(86400, 2 * 86400), (2 * 86400, 86400)] ) -@pytest.mark.parametrize("event_type", ["default", "error", "transaction"]) +@pytest.mark.parametrize("event_type", ["default", "error", "transaction", "nel"]) def test_processing_quotas( mini_sentry, relay_with_processing, @@ -410,7 +410,7 @@ def test_processing_quotas( key_id = public_keys[0]["numericId"] # Default events are also mapped to "error" by Relay. - category = "error" if event_type == "default" else event_type + category = "error" if event_type == "default" or event_type == "nel" else event_type projectconfig["config"]["quotas"] = [ { @@ -434,17 +434,28 @@ def transform(e): return e for i in range(5): - # send using the first dsn - relay.send_event( - project_id, transform({"message": f"regular{i}"}), dsn_key_idx=0 - ) + if event_type == "nel": + relay.send_nel_event(project_id, dsn_key_idx=0) + else: + # send using the first dsn + relay.send_event( + project_id, transform({"message": f"regular{i}"}), dsn_key_idx=0 + ) - event, _ = events_consumer.get_event() - assert event["logentry"]["formatted"] == f"regular{i}" + event, _ = events_consumer.get_event(timeout=10) + if event_type == "nel": + assert event["logentry"]["formatted"] == "application / http.error" + else: + assert event["logentry"]["formatted"] == f"regular{i}" # this one will not get a 429 but still get rate limited (silently) because # of our caching - relay.send_event(project_id, transform({"message": "some_message"}), dsn_key_idx=0) + if event_type == "nel": + relay.send_nel_event(project_id, dsn_key_idx=0) + else: + relay.send_event( + project_id, transform({"message": "some_message"}), dsn_key_idx=0 + ) if outcomes_consumer is not None: outcomes_consumer.assert_rate_limited( @@ -473,12 +484,18 @@ def transform(e): for i in range(10): # now send using the second key - relay.send_event( - project_id, transform({"message": f"otherkey{i}"}), dsn_key_idx=1 - ) + if event_type == "nel": + relay.send_nel_event(project_id, dsn_key_idx=1) + else: + relay.send_event( + project_id, transform({"message": f"otherkey{i}"}), dsn_key_idx=1 + ) event, _ = events_consumer.get_event() - assert event["logentry"]["formatted"] == f"otherkey{i}" + if event_type == "nel": + assert event["logentry"]["formatted"] == "application / http.error" + else: + assert event["logentry"]["formatted"] == f"otherkey{i}" @pytest.mark.parametrize("violating_bucket", [[4.0, 5.0], [4.0, 5.0, 6.0]])