diff --git a/comp/forwarder/defaultforwarder/endpoints/endpoints.go b/comp/forwarder/defaultforwarder/endpoints/endpoints.go index 1dacd8fbacfdf..6506f394bb2ec 100644 --- a/comp/forwarder/defaultforwarder/endpoints/endpoints.go +++ b/comp/forwarder/defaultforwarder/endpoints/endpoints.go @@ -11,47 +11,53 @@ import "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transac var ( // V1SeriesEndpoint is a v1 endpoint used to send series - V1SeriesEndpoint = transaction.Endpoint{Route: "/api/v1/series", Name: "series_v1"} + V1SeriesEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/v1/series", Name: "series_v1"} // V1CheckRunsEndpoint is a v1 endpoint used to send checks results - V1CheckRunsEndpoint = transaction.Endpoint{Route: "/api/v1/check_run", Name: "check_run_v1"} + V1CheckRunsEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/v1/check_run", Name: "check_run_v1"} // V1IntakeEndpoint is a v1 endpoint, used by Agent v.5, still used for metadata - V1IntakeEndpoint = transaction.Endpoint{Route: "/intake/", Name: "intake"} - // V1SketchSeriesEndpoint is a v1 endpoint used to send sketches - V1SketchSeriesEndpoint = transaction.Endpoint{Route: "/api/v1/sketches", Name: "sketches_v1"} //nolint unused for now + V1IntakeEndpoint = transaction.Endpoint{Subdomain: "", Route: "/intake/", Name: "intake"} // V1ValidateEndpoint is a v1 endpoint used to validate API keys - V1ValidateEndpoint = transaction.Endpoint{Route: "/api/v1/validate", Name: "validate_v1"} + V1ValidateEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/v1/validate", Name: "validate_v1"} // V1MetadataEndpoint is a v1 endpoint used for metadata (only used for inventory metadata for now) - V1MetadataEndpoint = transaction.Endpoint{Route: "/api/v1/metadata", Name: "metadata_v1"} - + V1MetadataEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/v1/metadata", Name: "metadata_v1"} // SeriesEndpoint is the v2 endpoint used to send series - SeriesEndpoint = transaction.Endpoint{Route: "/api/v2/series", Name: "series_v2"} - // EventsEndpoint is the v2 endpoint used to send events - EventsEndpoint = transaction.Endpoint{Route: "/api/v2/events", Name: "events_v2"} - // ServiceChecksEndpoint is the v2 endpoint used to send service checks - ServiceChecksEndpoint = transaction.Endpoint{Route: "/api/v2/service_checks", Name: "services_checks_v2"} + SeriesEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/v2/series", Name: "series_v2"} // SketchSeriesEndpoint is the v2 endpoint used to send sketches - SketchSeriesEndpoint = transaction.Endpoint{Route: "/api/beta/sketches", Name: "sketches_v2"} - // HostMetadataEndpoint is the v2 endpoint used to send host medatada - HostMetadataEndpoint = transaction.Endpoint{Route: "/api/v2/host_metadata", Name: "host_metadata_v2"} + SketchSeriesEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/beta/sketches", Name: "sketches_v2"} + // ProcessStatusEndpoint is a v1 endpoint used to send process checks + ProcessStatusEndpoint = transaction.Endpoint{Subdomain: "process", Route: "/status", Name: "process_status"} + // ProcessesIntakeStatusEndpoint is a v1 endpoint used to send processes checks + ProcessesIntakeStatusEndpoint = transaction.Endpoint{Subdomain: "process", Route: "/intake/status", Name: "process_intake_status"} // ProcessesEndpoint is a v1 endpoint used to send processes checks - ProcessesEndpoint = transaction.Endpoint{Route: "/api/v1/collector", Name: "process"} + ProcessesEndpoint = transaction.Endpoint{Subdomain: "process", Route: "/api/v1/collector", Name: "process"} // work with processes subdomain (get 403) // ProcessDiscoveryEndpoint is a v1 endpoint used to sends process discovery checks - ProcessDiscoveryEndpoint = transaction.Endpoint{Route: "/api/v1/discovery", Name: "process_discovery"} + ProcessDiscoveryEndpoint = transaction.Endpoint{Subdomain: "process", Route: "/api/v1/discovery", Name: "process_discovery"} // work with processes subdomain (get 403) // ProcessLifecycleEndpoint is a v2 endpoint used to send process lifecycle events - ProcessLifecycleEndpoint = transaction.Endpoint{Route: "/api/v2/proclcycle", Name: "process_lifecycle"} + ProcessLifecycleEndpoint = transaction.Endpoint{Subdomain: "orchestrator", Route: "/api/v2/proclcycle", Name: "process_lifecycle"} // 404 not found // RtProcessesEndpoint is a v1 endpoint used to send real time process checks - RtProcessesEndpoint = transaction.Endpoint{Route: "/api/v1/collector", Name: "rtprocess"} + RtProcessesEndpoint = transaction.Endpoint{Subdomain: "process", Route: "/api/v1/collector", Name: "rtprocess"} // work with processes subdomain (get 403) // ContainerEndpoint is a v1 endpoint used to send container checks - ContainerEndpoint = transaction.Endpoint{Route: "/api/v1/container", Name: "container"} + ContainerEndpoint = transaction.Endpoint{Subdomain: "process", Route: "/api/v1/container", Name: "container"} // work with processes subdomain (get 403) // RtContainerEndpoint is a v1 endpoint used to send real time container checks - RtContainerEndpoint = transaction.Endpoint{Route: "/api/v1/container", Name: "rtcontainer"} + RtContainerEndpoint = transaction.Endpoint{Subdomain: "process", Route: "/api/v1/container", Name: "rtcontainer"} // ConnectionsEndpoint is a v1 endpoint used to send connection checks - ConnectionsEndpoint = transaction.Endpoint{Route: "/api/v1/connections", Name: "connections"} + ConnectionsEndpoint = transaction.Endpoint{Subdomain: "process", Route: "/api/v1/connections", Name: "connections"} // LegacyOrchestratorEndpoint is a v1 endpoint used to send orchestrator checks - LegacyOrchestratorEndpoint = transaction.Endpoint{Route: "/api/v1/orchestrator", Name: "orchestrator"} + LegacyOrchestratorEndpoint = transaction.Endpoint{Subdomain: "orchestrator", Route: "/api/v1/orchestrator", Name: "orchestrator"} // OrchestratorEndpoint is a v2 endpoint used to send orchestrator checks - OrchestratorEndpoint = transaction.Endpoint{Route: "/api/v2/orch", Name: "orchestrator"} + OrchestratorEndpoint = transaction.Endpoint{Subdomain: "orchestrator", Route: "/api/v2/orch", Name: "orchestrator"} // OrchestratorManifestEndpoint is a v2 endpoint used to send orchestrator manifests - OrchestratorManifestEndpoint = transaction.Endpoint{Route: "/api/v2/orchmanif", Name: "orchmanifest"} + OrchestratorManifestEndpoint = transaction.Endpoint{Subdomain: "orchestrator", Route: "/api/v2/orchmanif", Name: "orchmanifest"} // work with POST but got 401 + + /////////////// Unused Endpoints /////////////// + + // V1SketchSeriesEndpoint is a v1 endpoint used to send sketches + V1SketchSeriesEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/v1/sketches", Name: "sketches_v1"} //nolint + // EventsEndpoint is the v2 endpoint used to send events + EventsEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/v2/events", Name: "events_v2"} + // ServiceChecksEndpoint is the v2 endpoint used to send service checks + ServiceChecksEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/v2/service_checks", Name: "services_checks_v2"} + // HostMetadataEndpoint is the v2 endpoint used to send host medatada + HostMetadataEndpoint = transaction.Endpoint{Subdomain: "", Route: "/api/v2/host_metadata", Name: "host_metadata_v2"} ) diff --git a/comp/forwarder/defaultforwarder/transaction/endpoint.go b/comp/forwarder/defaultforwarder/transaction/endpoint.go index b766be196f8ff..21adcc485733a 100644 --- a/comp/forwarder/defaultforwarder/transaction/endpoint.go +++ b/comp/forwarder/defaultforwarder/transaction/endpoint.go @@ -5,8 +5,12 @@ package transaction +import "strings" + // Endpoint is an endpoint type Endpoint struct { + //Subdomain of the endpoint + Subdomain string // Route to hit in the HTTP transaction Route string // Name of the endpoint for the telemetry metrics @@ -17,3 +21,26 @@ type Endpoint struct { func (e Endpoint) String() string { return e.Route } + +// GetEndpoint returns the full endpoint URL +func (e Endpoint) GetEndpoint(domain string) string { + if e.Subdomain == "" { + e.Subdomain = "app" + } + + e.Subdomain = strings.TrimSuffix(e.Subdomain, "/") + e.Subdomain = strings.TrimPrefix(e.Subdomain, "/") + + domain = strings.TrimSuffix(domain, "/") + e.Route = strings.TrimPrefix(e.Route, "/") + + url := domain + "/" + e.Route + + if !strings.Contains(url, "http://") && !strings.Contains(url, "https://") { + url = "https://" + url + } + + url = strings.Replace(url, "app", e.Subdomain, 1) + + return url +} diff --git a/comp/forwarder/defaultforwarder/transaction/endpoint_test.go b/comp/forwarder/defaultforwarder/transaction/endpoint_test.go new file mode 100644 index 0000000000000..ea7d1c16ca4e1 --- /dev/null +++ b/comp/forwarder/defaultforwarder/transaction/endpoint_test.go @@ -0,0 +1,61 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package endpoints stores a collection of `transaction.Endpoint` mainly used by the forwarder package to send data to +// Datadog using the right request path for a given type of data. +package transaction + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetEndpoint(t *testing.T) { + tests := []struct { + name string + endpoint Endpoint + domain string + want string + }{ + { + name: "default subdomain applied when empty", + endpoint: Endpoint{Route: "docs"}, + domain: "https://dev.example.com", + want: "https://dev.example.com/docs", + }, + { + name: "explicit subdomain replacement", + endpoint: Endpoint{Subdomain: "admin", Route: "docs"}, + domain: "app.example.com", + want: "https://admin.example.com/docs", + }, + { + name: "no subdomain, app not in domain", + endpoint: Endpoint{Subdomain: "app/", Route: "docs"}, + domain: "myappsite.com", + want: "https://myappsite.com/docs", + }, + { + name: "subdomain app with app in domain", + endpoint: Endpoint{Subdomain: "app", Route: "support"}, + domain: "https://app.company.com", + want: "https://app.company.com/support", + }, + { + name: "complex route and domain", + endpoint: Endpoint{Subdomain: "api", Route: "/v1/users"}, + domain: "https://app.service.com", + want: "https://api.service.com/v1/users", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.endpoint.GetEndpoint(tt.domain) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/diagnose/README.md b/pkg/diagnose/README.md index 93145757c12d8..ec6f4c4deded0 100644 --- a/pkg/diagnose/README.md +++ b/pkg/diagnose/README.md @@ -62,3 +62,21 @@ func diagnose(diagCfg diagnosis.Config) []diagnosis.Diagnosis { ## Context of a diagnose function execution Normally, registered diagnose suite functions will be executed in context of the running agent service (or other services) but if ```Config.ForceLocal``` configuration is specified the registered diagnose function will be executed in the context of agent diagnose CLI command (if possible). + +## Which connectivity to endpoint are tested ? +With diagnose command, the Agent try to reach out a lot of endpoints, these ones are listed below: + +| Subdomain | Route | HTTP Method | Status Code expected | +|-----------|-------|-------------|----------------------| +| https://app.datadoghq.com | /api/v1/series | POST | 200 | +| https://app.datadoghq.com | /api/v1/check_run | POST | 200 | +| https://app.datadoghq.com | /intake/ | POST | 200 | +| https://app.datadoghq.com | /api/v1/validate | GET | 200 | +| https://app.datadoghq.com | /api/v1/metadata | POST | 200 | +| https://app.datadoghq.com | /api/v2/series | POST | 200 | +| https://app.datadoghq.com | /api/beta/sketches | POST | 200 | +| https://\-flare.agent.datadoghq.com | /support/flare | HEAD | 307 | +| https://process.datadoghq.com | /intake/status | GET | 200 | + + + diff --git a/pkg/diagnose/connectivity/core_endpoint.go b/pkg/diagnose/connectivity/core_endpoint.go index 45a7fa26ed754..b1337b28dba9a 100644 --- a/pkg/diagnose/connectivity/core_endpoint.go +++ b/pkg/diagnose/connectivity/core_endpoint.go @@ -150,7 +150,7 @@ func createDiagnosisString(diagnosis string, report string) string { // sendHTTPRequestToEndpoint creates an URL based on the domain and the endpoint information // then sends an HTTP Request with the method and payload inside the endpoint information func sendHTTPRequestToEndpoint(ctx context.Context, client *http.Client, domain string, endpointInfo endpointInfo, apiKey string) (int, []byte, string, error) { - url := createEndpointURL(domain, endpointInfo) + url := endpointInfo.Endpoint.GetEndpoint(domain) logURL := scrubber.ScrubLine(url) // Create a request for the backend @@ -168,18 +168,20 @@ func sendHTTPRequestToEndpoint(ctx context.Context, client *http.Client, domain resp, err := client.Do(req) - if err != nil { - return 0, nil, logURL, fmt.Errorf("cannot send the HTTP request to '%v' : %v", logURL, scrubber.ScrubLine(err.Error())) - } - defer func() { _ = resp.Body.Close() }() + if (err == nil) || (endpointInfo.Endpoint.Subdomain == "process" && resp.StatusCode == http.StatusBadRequest) { + defer func() { _ = resp.Body.Close() }() + // Get the endpoint response + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0, nil, logURL, fmt.Errorf("fail to read the response Body: %s", scrubber.ScrubLine(err.Error())) + } else if strings.Contains(string(body), "invalid message format") { + resp.StatusCode = -1 + } - // Get the endpoint response - body, err := io.ReadAll(resp.Body) - if err != nil { - return 0, nil, logURL, fmt.Errorf("fail to read the response Body: %s", scrubber.ScrubLine(err.Error())) + return resp.StatusCode, body, logURL, nil } - return resp.StatusCode, body, logURL, nil + return 0, nil, logURL, fmt.Errorf("cannot send the HTTP request to '%v' : %v", logURL, scrubber.ScrubLine(err.Error())) } // createEndpointUrl joins a domain with an endpoint @@ -211,9 +213,10 @@ func verifyEndpointResponse(diagCfg diagnosis.Config, statusCode int, responseBo if statusCode >= 400 { newErr = fmt.Errorf("bad request") verifyReport = fmt.Sprintf("Received response : '%v'\n", scrubbedResponseBody) - } + } else if statusCode != -1 { + verifyReport += fmt.Sprintf("Received status code %v from the endpoint", statusCode) - verifyReport += fmt.Sprintf("Received status code %v from the endpoint", statusCode) + } return verifyReport, newErr } diff --git a/pkg/diagnose/connectivity/core_endpoint_test.go b/pkg/diagnose/connectivity/core_endpoint_test.go index a88bd0f8990ab..6f673f34bb106 100644 --- a/pkg/diagnose/connectivity/core_endpoint_test.go +++ b/pkg/diagnose/connectivity/core_endpoint_test.go @@ -23,7 +23,8 @@ var ( apiKey1 = "api_key1" apiKey2 = "api_key2" - endpointInfoTest = endpointInfo{Endpoint: endpoints.V1ValidateEndpoint} + endpointInfoTest = endpointInfo{Endpoint: endpoints.V1ValidateEndpoint} + endpointProcessTest = endpointInfo{Endpoint: endpoints.ProcessStatusEndpoint} ) func TestCreateEndpointUrl(t *testing.T) { @@ -59,6 +60,12 @@ func TestSendHTTPRequestToEndpoint(t *testing.T) { assert.NoError(t, err) assert.Equal(t, statusCode, 400) assert.Equal(t, string(responseBody), "Bad Request") + + // With a different subdomain endpoint + statusCodeProcess, responseBodyProcess, _, errProcess := sendHTTPRequestToEndpoint(context.Background(), client, ts1.URL, endpointProcessTest, apiKey1) + assert.NoError(t, errProcess) + assert.Equal(t, statusCodeProcess, 200) + assert.Equal(t, string(responseBodyProcess), "OK") } func TestAcceptRedirection(t *testing.T) { diff --git a/pkg/diagnose/connectivity/endpoint_info.go b/pkg/diagnose/connectivity/endpoint_info.go index c90294d30cd19..f17bf26734128 100644 --- a/pkg/diagnose/connectivity/endpoint_info.go +++ b/pkg/diagnose/connectivity/endpoint_info.go @@ -41,15 +41,29 @@ func getEndpointsInfo(cfg config.Reader) []endpointInfo { {endpoints.V1SeriesEndpoint, "POST", emptyPayload}, {endpoints.V1CheckRunsEndpoint, "POST", checkRunPayload}, {endpoints.V1IntakeEndpoint, "POST", emptyPayload}, - - // This endpoint behaves differently depending on `site` when using `emptyPayload`. Do not modify `nil` here ! - {endpoints.V1ValidateEndpoint, "GET", nil}, + {endpoints.V1ValidateEndpoint, "GET", nil}, // This endpoint behaves differently depending on `site` when using `emptyPayload`. Do not modify `nil` here ! {endpoints.V1MetadataEndpoint, "POST", emptyPayload}, // v2 endpoints {endpoints.SeriesEndpoint, "POST", emptyPayload}, {endpoints.SketchSeriesEndpoint, "POST", emptyPayload}, + // Process endpoints + {endpoints.ProcessStatusEndpoint, "GET", nil}, + {endpoints.ProcessesIntakeStatusEndpoint, "GET", nil}, + {endpoints.ProcessesEndpoint, "POST", emptyPayload}, + {endpoints.ProcessDiscoveryEndpoint, "POST", emptyPayload}, + {endpoints.RtProcessesEndpoint, "POST", emptyPayload}, + {endpoints.ContainerEndpoint, "POST", emptyPayload}, + {endpoints.RtContainerEndpoint, "POST", emptyPayload}, + {endpoints.ConnectionsEndpoint, "POST", emptyPayload}, + + // Orchestrator endpoints + //{endpoints.ProcessLifecycleEndpoint, "POST", emptyPayload}, + {endpoints.LegacyOrchestratorEndpoint, "POST", emptyPayload}, + {endpoints.OrchestratorEndpoint, "POST", emptyPayload}, + {endpoints.OrchestratorManifestEndpoint, "POST", emptyPayload}, + // Flare endpoint {transaction.Endpoint{Route: helpers.GetFlareEndpoint(cfg), Name: "flare"}, "HEAD", nil}, }