diff --git a/Dockerfile b/Dockerfile index cc3377eb9..a921a2563 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,8 +31,8 @@ RUN make deps:openresty && make clean:dev COPY tasks/deps/trafficserver /app/tasks/deps/ RUN make deps:trafficserver && make clean:dev -COPY tasks/deps/rsyslog /app/tasks/deps/ -RUN make deps:rsyslog && make clean:dev +COPY tasks/deps/fluent-bit /app/tasks/deps/ +RUN make deps:fluent-bit && make clean:dev COPY src/api-umbrella-git-1.rockspec src/luarocks.lock /app/src/ COPY tasks/deps/luarocks /app/tasks/deps/ diff --git a/Taskfile.yml b/Taskfile.yml index f28b54490..0d57ddc54 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -60,6 +60,15 @@ tasks: generates: - ./build/work/stamp/deps/envoy-control-plane + deps:fluent-bit: + cmds: + - ./tasks/deps/fluent-bit + sources: + - ./tasks/deps/fluent-bit + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/fluent-bit + deps:luarocks: deps: - deps:openresty @@ -90,15 +99,6 @@ tasks: generates: - ./build/work/stamp/deps/perp - deps:rsyslog: - cmds: - - ./tasks/deps/rsyslog - sources: - - ./tasks/deps/rsyslog - - ./tasks/helpers.sh - generates: - - ./build/work/stamp/deps/rsyslog - deps:trafficserver: deps: - deps:openresty @@ -116,10 +116,10 @@ tasks: - task: deps:cue - task: deps:envoy - task: deps:envoy-control-plane + - task: deps:fluent-bit - task: deps:luarocks - task: deps:openresty - task: deps:perp - - task: deps:rsyslog - task: deps:trafficserver build-deps:crane: diff --git a/build/package_dependencies.sh b/build/package_dependencies.sh index 0a6490afc..ce4c8bb6f 100644 --- a/build/package_dependencies.sh +++ b/build/package_dependencies.sh @@ -34,9 +34,6 @@ if [[ "$ID_NORMALIZED" == "rhel" ]]; then # TrafficServer libxml2 - # rsyslog omelasticsearch - libcurl - # init.d script helpers initscripts @@ -190,13 +187,6 @@ elif [[ "$ID_NORMALIZED" == "debian" ]]; then libunwind8 libxml2 - # rsyslog - libestr0 - libfastjson4 - - # rsyslog omelasticsearch - "libcurl$libcurl_version" - # init.d script helpers sysvinit-utils lsb-base @@ -274,10 +264,6 @@ elif [[ "$ID_NORMALIZED" == "debian" ]]; then # lua-psl libpsl-dev - # rsyslog - libestr-dev - libfastjson-dev - # ngx_http_geoip2_module libmaxminddb-dev @@ -285,6 +271,11 @@ elif [[ "$ID_NORMALIZED" == "debian" ]]; then libhwloc-dev libjemalloc-dev libunwind-dev + + # Fluent Bit + bison + cmake + flex ) test_runtime_dependencies=( unbound diff --git a/config/opensearch_templates_v2_es7.json.etlua b/config/opensearch_templates_v3.json.etlua similarity index 94% rename from config/opensearch_templates_v2_es7.json.etlua rename to config/opensearch_templates_v3.json.etlua index fb6cd045a..29be2bff2 100644 --- a/config/opensearch_templates_v2_es7.json.etlua +++ b/config/opensearch_templates_v3.json.etlua @@ -1,12 +1,13 @@ -[ - { - "id": "<%- config["opensearch"]["index_name_prefix"] %>-log-v2-template", +{ + "<%- config["opensearch"]["index_name_prefix"] %>-log-v3-template": { + "index_patterns": [ + "<%- config["opensearch"]["index_name_prefix"] %>-logs-v3-*" + ], + "data_stream": {}, "template": { - "template": "<%- config["opensearch"]["index_name_prefix"] %>-logs-v2-*", "settings": { "index": { - "number_of_shards": 1, - "codec": "best_compression" + "refresh_interval": "10s" }, "analysis": { "normalizer": { @@ -28,6 +29,9 @@ "mappings": { "dynamic": "strict", "properties": { + "@timestamp": { + "type": "date" + }, "api_backend_id": { "type": "keyword", "normalizer": "lowercase_normalizer" @@ -66,9 +70,6 @@ "type": "keyword", "normalizer": "lowercase_normalizer" }, - "request_at": { - "type": "date" - }, "request_basic_auth_username": { "type": "keyword", "normalizer": "lowercase_normalizer" @@ -85,12 +86,16 @@ "type": "keyword", "normalizer": "lowercase_normalizer" }, + "request_id": { + "type": "keyword" + }, "request_ip": { "type": "keyword", "normalizer": "lowercase_normalizer" }, "request_ip_city": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "request_ip_country": { "type": "keyword", @@ -233,4 +238,4 @@ } } } -] +} diff --git a/config/schema.cue b/config/schema.cue index a928921ad..16add59b7 100644 --- a/config/schema.cue +++ b/config/schema.cue @@ -418,7 +418,7 @@ import "path" match_x_forwarded_host?: bool } - rsyslog: { + fluent_bit: { host: string | *"127.0.0.1" port: uint16 | *14014 } @@ -457,18 +457,7 @@ import "path" "http://opensearch:9200", ] index_name_prefix: string | *"api-umbrella" - index_partition: string | *"daily" - api_version: uint | *7 - template_version: uint | *2 - aws_signing_proxy: { - host: string | *"127.0.0.1" - port: uint16 | *14017 - workers: uint | "auto" | *1 - worker_connections: uint | *8192 - listen_so_keepalive: string | *"on" - listen_backlog?: uint - error_log_level: string | *"notice" - } + template_version: uint | *3 } #analytics_output_name: "opensearch" diff --git a/config/test.yml b/config/test.yml index 3a44aaa2b..d0bdbfd36 100644 --- a/config/test.yml +++ b/config/test.yml @@ -91,7 +91,7 @@ router: # 1-2 seconds, but to make testing easier, force immediate refreshes in the # test environment. refresh_local_cache_interval: 0 -rsyslog: +fluent_bit: port: 13014 geoip: db_update_frequency: false @@ -100,6 +100,8 @@ postgresql: password: dev_password migrations: password: dev_password +opensearch: + index_name_prefix: "api-umbrella-test" unbound: port: 13100 control_port: 13101 diff --git a/docker-compose.yml b/docker-compose.yml index 7dea11213..1b0639863 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: ports: - "14011:5432" opensearch: - image: public.ecr.aws/opensearchproject/opensearch:2.0.1 + image: public.ecr.aws/opensearchproject/opensearch:2.11.1 environment: OPENSEARCH_JAVA_OPTS: "-Xms32m -Xmx256m" DISABLE_INSTALL_DEMO_CONFIG: "true" diff --git a/src/api-umbrella/cli/reopen_logs.lua b/src/api-umbrella/cli/reopen_logs.lua index 0b770dc8c..e1d201509 100644 --- a/src/api-umbrella/cli/reopen_logs.lua +++ b/src/api-umbrella/cli/reopen_logs.lua @@ -38,14 +38,6 @@ local function reopen_nginx(perp_base) end end -local function reopen_rsyslog(perp_base) - local _, err = shell_blocking_capture_combined({ "perpctl", "-b", perp_base, "hup", "rsyslog" }) - if err then - print("Failed to reopen logs for rsyslog\n" .. err) - os.exit(1) - end -end - return function() local running, pid = status() if not running then @@ -59,6 +51,5 @@ return function() if config["_service_router_enabled?"] then reopen_nginx(perp_base) - reopen_rsyslog(perp_base) end end diff --git a/src/api-umbrella/cli/write_config_files.lua b/src/api-umbrella/cli/write_config_files.lua index 10b04c011..d1be26369 100644 --- a/src/api-umbrella/cli/write_config_files.lua +++ b/src/api-umbrella/cli/write_config_files.lua @@ -241,16 +241,13 @@ local function activate_services() available_services = invert_table(available_services) local active_services = {} - if config["_service_opensearch_aws_signing_proxy_enabled?"] then - active_services["opensearch-aws-signing-proxy"] = 1 - end if config["_service_router_enabled?"] then if config["geoip"]["_auto_updater_enabled"] then active_services["geoip-auto-updater"] = 1 end active_services["envoy-control-plane"] = 1 + active_services["fluent-bit"] = 1 active_services["nginx"] = 1 - active_services["rsyslog"] = 1 active_services["trafficserver"] = 1 end if config["_service_egress_enabled?"] then diff --git a/src/api-umbrella/http-api/health.lua b/src/api-umbrella/http-api/health.lua index 4adf1ad92..06ad9b685 100644 --- a/src/api-umbrella/http-api/health.lua +++ b/src/api-umbrella/http-api/health.lua @@ -1,10 +1,10 @@ local active_config_exists = require("api-umbrella.proxy.stores.active_config_store").exists local config = require("api-umbrella.utils.load_config")() local http = require "resty.http" -local icu_date = require "icu-date-ffi" local json_encode = require "api-umbrella.utils.json_encode" local opensearch = require "api-umbrella.utils.opensearch" +local jobs_dict = ngx.shared.jobs local opensearch_query = opensearch.query local ngx_var = ngx.var @@ -45,18 +45,9 @@ local function status_response(quick) response["details"]["analytics_db"] = opensearch_health["status"] -- Check to see if the OpenSearch index aliases have been setup. - local date = icu_date.new({ zone_id = "UTC" }) - local today = date:format(opensearch.partition_date_format) - local alias = config["opensearch"]["index_name_prefix"] .. "-logs-" .. today - local index = config["opensearch"]["index_name_prefix"] .. "-logs-v" .. config["opensearch"]["template_version"] .. "-" .. today - res, err = opensearch_query("/" .. index .. "/_alias/" .. alias) - if err then - ngx.log(ngx.ERR, "failed to fetch opensearch alias details: ", err) - elseif res.body_json then - local opensearch_alias = res.body_json - if not opensearch_alias["error"] then - response["details"]["analytics_db_setup"] = "green" - end + local created = jobs_dict:get("opensearch_templates_created") + if created then + response["details"]["analytics_db_setup"] = "green" end end diff --git a/src/api-umbrella/opensearch-aws-signing-proxy/init.lua b/src/api-umbrella/opensearch-aws-signing-proxy/init.lua deleted file mode 100644 index 36840792f..000000000 --- a/src/api-umbrella/opensearch-aws-signing-proxy/init.lua +++ /dev/null @@ -1 +0,0 @@ -require "api-umbrella.utils.load_config" diff --git a/src/api-umbrella/opensearch-aws-signing-proxy/proxy.lua b/src/api-umbrella/opensearch-aws-signing-proxy/proxy.lua deleted file mode 100644 index a5fccd61c..000000000 --- a/src/api-umbrella/opensearch-aws-signing-proxy/proxy.lua +++ /dev/null @@ -1,53 +0,0 @@ -local ngx_exit = ngx.exit -local ngx_say = ngx.say -local ngx_var = ngx.var -local req_set_header = ngx.req.set_header - -local config = require("api-umbrella.utils.load_config")() - -local username = config["opensearch"]["aws_signing_proxy"]["username"] -if not username then - ngx_say("opensearch.aws_signing_proxy.username must be configured in /etc/api-umbrella/api-umbrella.yml") - return ngx_exit(ngx.HTTP_INTERNAL_SERVER_ERROR) -end - -local password = config["opensearch"]["aws_signing_proxy"]["password"] -if not password then - ngx_say("opensearch.aws_signing_proxy.password must be configured in /etc/api-umbrella/api-umbrella.yml") - return ngx_exit(ngx.HTTP_INTERNAL_SERVER_ERROR) -end - -local aws_region = config["opensearch"]["aws_signing_proxy"]["aws_region"] -if not aws_region then - ngx_say("opensearch.aws_signing_proxy.aws_region must be configured in /etc/api-umbrella/api-umbrella.yml") - return ngx_exit(ngx.HTTP_INTERNAL_SERVER_ERROR) -end - -local aws_access_key_id = config["opensearch"]["aws_signing_proxy"]["aws_access_key_id"] -if not aws_access_key_id then - ngx_say("opensearch.aws_signing_proxy.aws_access_key_id must be configured in /etc/api-umbrella/api-umbrella.yml") - return ngx_exit(ngx.HTTP_INTERNAL_SERVER_ERROR) -end - -local aws_secret_access_key = config["opensearch"]["aws_signing_proxy"]["aws_secret_access_key"] -if not aws_secret_access_key then - ngx_say("opensearch.aws_signing_proxy.aws_access_key_id must be configured in /etc/api-umbrella/api-umbrella.yml") - return ngx_exit(ngx.HTTP_INTERNAL_SERVER_ERROR) -end - -local remote_username = ngx_var.remote_user -local remote_password = ngx_var.remote_passwd -if not remote_username or not remote_password then - ngx.header["WWW-Authenticate"] = 'Basic realm="Restricted"' - return ngx_exit(ngx.HTTP_UNAUTHORIZED) -end - -if remote_username ~= username or remote_password ~= password then - return ngx_exit(ngx.HTTP_FORBIDDEN) -end - -local host = config["opensearch"]["aws_signing_proxy"]["aws_host"] -req_set_header("Host", host) - -local signing = require "api-umbrella.utils.aws_signing_v4" -signing.sign_request(aws_region, aws_access_key_id, aws_secret_access_key) diff --git a/src/api-umbrella/proxy/hooks/init_preload_modules.lua b/src/api-umbrella/proxy/hooks/init_preload_modules.lua index 00c7e63d4..ed0d2a45f 100644 --- a/src/api-umbrella/proxy/hooks/init_preload_modules.lua +++ b/src/api-umbrella/proxy/hooks/init_preload_modules.lua @@ -70,7 +70,6 @@ require "api-umbrella.utils.worker_group" require "api-umbrella.utils.xpcall_error_handler" require "cjson" require "etlua" -require "icu-date-ffi" require "libcidr-ffi" require "lustache" require "ngx.re" diff --git a/src/api-umbrella/proxy/hooks/log_initial_proxy.lua b/src/api-umbrella/proxy/hooks/log_initial_proxy.lua index 4457dcf91..086f222ee 100644 --- a/src/api-umbrella/proxy/hooks/log_initial_proxy.lua +++ b/src/api-umbrella/proxy/hooks/log_initial_proxy.lua @@ -1,17 +1,26 @@ -local flatten_headers = require "api-umbrella.utils.flatten_headers" local log_utils = require "api-umbrella.proxy.log_utils" -local xpcall_error_handler = require "api-umbrella.utils.xpcall_error_handler" +local ignore_request = log_utils.ignore_request local ngx_ctx = ngx.ctx -local ngx_var = ngx.var -local req_get_headers = ngx.req.get_headers -local resp_get_headers = ngx.resp.get_headers -if log_utils.ignore_request(ngx_ctx) then +if ignore_request(ngx_ctx) then return end +local flatten_headers = require "api-umbrella.utils.flatten_headers" +local json_encode = require "api-umbrella.utils.json_encode" +local xpcall_error_handler = require "api-umbrella.utils.xpcall_error_handler" + +local cache_new_city_geocode = log_utils.cache_new_city_geocode +local ngx_var = ngx.var +local normalized_data = log_utils.normalized_data +local req_get_headers = ngx.req.get_headers +local resp_get_headers = ngx.resp.get_headers local sec_to_ms = log_utils.sec_to_ms +local send_message = log_utils.send_message +local set_computed_url_fields = log_utils.set_computed_url_fields +local set_computed_user_agent_fields = log_utils.set_computed_user_agent_fields +local set_request_ip_geo_fields = log_utils.set_request_ip_geo_fields local function build_log_data() -- Fetch all the request and response headers. @@ -19,13 +28,13 @@ local function build_log_data() local response_headers = flatten_headers(resp_get_headers()); -- Put together the basic log data. - local id = ngx_var.x_api_umbrella_request_id + local request_id = ngx_var.x_api_umbrella_request_id local data = { api_backend_resolved_host = ngx_ctx.x_api_umbrella_backend_resolved_host, api_backend_response_code_details = ngx_ctx.x_api_umbrella_backend_response_code_details, api_backend_response_flags = ngx_ctx.x_api_umbrella_backend_response_flags, - denied_reason = ngx_ctx.gatekeeper_denied_code, - id = id, + gatekeeper_denied_code = ngx_ctx.gatekeeper_denied_code, + request_id = request_id, request_accept = request_headers["accept"], request_accept_encoding = request_headers["accept-encoding"], request_basic_auth_username = ngx_var.remote_user, @@ -80,19 +89,18 @@ local function build_log_data() data["response_cache_flags"] = string.sub(via, -8, -3) end - log_utils.set_request_ip_geo_fields(data, ngx_var) - log_utils.set_computed_timestamp_fields(data) - log_utils.set_computed_url_fields(data, ngx_ctx) - log_utils.set_computed_user_agent_fields(data) + set_request_ip_geo_fields(data, ngx_var) + set_computed_url_fields(data, ngx_ctx) + set_computed_user_agent_fields(data) - return log_utils.normalized_data(data) + return normalized_data(data) end local function log_request() - -- Build the log message and send to rsyslog for processing. + -- Build the log message and send to Fluent Bit for processing. local data = build_log_data() - local syslog_message = log_utils.build_syslog_message(data) - local _, err = log_utils.send_syslog_message(syslog_message) + local message = json_encode(data) + local _, err = send_message(message) if err then ngx.log(ngx.ERR, "failed to log message: ", err) return @@ -100,7 +108,7 @@ local function log_request() -- After logging, cache any new cities we see from GeoIP in our database. if data["request_ip_lat"] then - log_utils.cache_new_city_geocode(data) + cache_new_city_geocode(data) end end diff --git a/src/api-umbrella/proxy/jobs/opensearch_setup.lua b/src/api-umbrella/proxy/jobs/opensearch_setup.lua index eb5fbf522..9c5acd6cf 100644 --- a/src/api-umbrella/proxy/jobs/opensearch_setup.lua +++ b/src/api-umbrella/proxy/jobs/opensearch_setup.lua @@ -1,5 +1,3 @@ -local config = require("api-umbrella.utils.load_config")() -local icu_date = require "icu-date-ffi" local interval_lock = require "api-umbrella.utils.interval_lock" local opensearch = require "api-umbrella.utils.opensearch" local opensearch_templates = require "api-umbrella.proxy.opensearch_templates_data" @@ -44,10 +42,10 @@ function _M.create_templates() if created then return end if opensearch_templates then - for _, template in ipairs(opensearch_templates) do - local _, err = opensearch_query("/_template/" .. template["id"], { + for template_id, template in pairs(opensearch_templates) do + local _, err = opensearch_query("/_index_template/" .. template_id, { method = "PUT", - body = template["template"], + body = template, }) if err then ngx.log(ngx.ERR, "failed to update opensearch template: ", err) @@ -61,69 +59,10 @@ function _M.create_templates() end end -function _M.create_aliases() - local date = icu_date.new({ zone_id = "UTC" }) - local today = date:format(opensearch.partition_date_format) - - date:add(icu_date.fields.DATE, 1) - local tomorrow = date:format(opensearch.partition_date_format) - - local aliases = { - { - alias = config["opensearch"]["index_name_prefix"] .. "-logs-" .. today, - index = config["opensearch"]["index_name_prefix"] .. "-logs-v" .. config["opensearch"]["template_version"] .. "-" .. today, - }, - { - alias = config["opensearch"]["index_name_prefix"] .. "-logs-write-" .. today, - index = config["opensearch"]["index_name_prefix"] .. "-logs-v" .. config["opensearch"]["template_version"] .. "-" .. today, - }, - } - - -- Create the aliases needed for the next day if we're at the end of the - -- month. - if tomorrow ~= today then - table.insert(aliases, { - alias = config["opensearch"]["index_name_prefix"] .. "-logs-" .. tomorrow, - index = config["opensearch"]["index_name_prefix"] .. "-logs-v" .. config["opensearch"]["template_version"] .. "-" .. tomorrow, - }) - table.insert(aliases, { - alias = config["opensearch"]["index_name_prefix"] .. "-logs-write-" .. tomorrow, - index = config["opensearch"]["index_name_prefix"] .. "-logs-v" .. config["opensearch"]["template_version"] .. "-" .. tomorrow, - }) - end - - for _, alias in ipairs(aliases) do - -- Only create aliases if they don't already exist. - local exists_res, exists_err = opensearch_query("/_alias/" .. alias["alias"], { - method = "HEAD", - }) - if exists_err then - ngx.log(ngx.ERR, "failed to check opensearch index alias: ", exists_err) - elseif exists_res.status == 404 then - -- Make sure the index exists. - local _, create_err = opensearch_query("/" .. alias["index"], { - method = "PUT", - }) - if create_err then - ngx.log(ngx.ERR, "failed to create opensearch index: ", create_err) - end - - -- Create the alias for the index. - local _, alias_err = opensearch_query("/" .. alias["index"] .. "/_alias/" .. alias["alias"], { - method = "PUT", - }) - if alias_err then - ngx.log(ngx.ERR, "failed to create opensearch index alias: ", alias_err) - end - end - end -end - local function setup() local _, err = _M.wait_for_opensearch() if not err then _M.create_templates() - _M.create_aliases() else ngx.log(ngx.ERR, "timed out waiting for eleasticsearch before setup, rerunning...") sleep(5) diff --git a/src/api-umbrella/proxy/log_utils.lua b/src/api-umbrella/proxy/log_utils.lua index 251b68b2d..e66556857 100644 --- a/src/api-umbrella/proxy/log_utils.lua +++ b/src/api-umbrella/proxy/log_utils.lua @@ -1,7 +1,5 @@ local config = require("api-umbrella.utils.load_config")() local escape_uri_non_ascii = require "api-umbrella.utils.escape_uri_non_ascii" -local icu_date = require "icu-date-ffi" -local json_encode = require "api-umbrella.utils.json_encode" local logger = require "resty.logger.socket" local pg_utils = require "api-umbrella.utils.pg_utils" local plutils = require "pl.utils" @@ -9,23 +7,14 @@ local round = require "api-umbrella.utils.round" local shared_dict_retry_set = require("api-umbrella.utils.shared_dict_retry").set local user_agent_parser = require "api-umbrella.proxy.user_agent_parser" +local logger_init = logger.init +local logger_initted = logger.initted +local logger_log = logger.log local re_gsub = ngx.re.gsub local split = plutils.split +local string_len = string.len local timer_at = ngx.timer.at -local syslog_facility = 16 -- local0 -local syslog_severity = 6 -- info -local syslog_priority = (syslog_facility * 8) + syslog_severity -local syslog_version = 1 - -local ZONE_OFFSET = icu_date.fields.ZONE_OFFSET -local DST_OFFSET = icu_date.fields.DST_OFFSET -local DAY_OF_WEEK = icu_date.fields.DAY_OF_WEEK --- Setup the date object in the analytics timezone, and set the first day of --- the week to Mondays for ISO week calculations. -local date = icu_date.new({ zone_id = config["analytics"]["timezone"] }) -date:set_attribute(icu_date.attributes.FIRST_DAY_OF_WEEK, 2) - local geocode_city_cache_dict = ngx.shared.geocode_city_cache local _M = {} @@ -118,15 +107,12 @@ function _M.set_url_hierarchy(data) local path_parts = split(cleaned_path, "/", true, 6) -- Setup top-level host hierarchy for OpenSearch storage. - data["request_url_hierarchy"] = {} local host_level = data["request_url_host"] if #path_parts > 0 then host_level = host_level .. "/" end data["request_url_hierarchy_level0"] = host_level - table.insert(data["request_url_hierarchy"], "0/" .. host_level) - local path_tree = "/" for index, _ in ipairs(path_parts) do local path_level = path_parts[index] @@ -144,13 +130,8 @@ function _M.set_url_hierarchy(data) path_level = path_level .. "/" end - -- Store in the request_url_path_level(1-6) fields for SQL storage. + -- Store in the request_url_path_level(1-6) fields. data["request_url_hierarchy_level" .. index] = path_level - - -- Store as an array for OpenSearch storage. - path_tree = path_tree .. path_level - local path_token = index .. "/" .. data["request_url_host"] .. path_tree - table.insert(data["request_url_hierarchy"], path_token) end end @@ -262,29 +243,6 @@ function _M.set_request_ip_geo_fields(data, ngx_var) end end -function _M.set_computed_timestamp_fields(data) - -- Generate a string of current timestamp in the analytics timezone. - -- - -- Note that we use os.date instead of icu-date's "format" function, since in - -- some microbenchmarks, this approach is faster. - date:set_millis(data["timestamp_utc"]) - local tz_offset = date:get(ZONE_OFFSET) + date:get(DST_OFFSET) - local tz_time = os.date("!%Y-%m-%d %H:%M:00", (date:get_millis() + tz_offset) / 1000) - - -- Determine the first day in the ISO week (the most recent Monday). - date:set(DAY_OF_WEEK, 2) - local week_tz_offset = date:get(ZONE_OFFSET) + date:get(DST_OFFSET) - local tz_week = os.date("!%Y-%m-%d", (date:get_millis() + week_tz_offset) / 1000) - - data["timestamp_tz_offset"] = tz_offset - data["timestamp_tz_year"] = string.sub(tz_time, 1, 4) .. "-01-01" -- YYYY-01-01 - data["timestamp_tz_month"] = string.sub(tz_time, 1, 7) .. "-01" -- YYYY-MM-01 - data["timestamp_tz_week"] = tz_week -- YYYY-MM-DD of first day in ISO week. - data["timestamp_tz_date"] = string.sub(tz_time, 1, 10) -- YYYY-MM-DD - data["timestamp_tz_hour"] = string.sub(tz_time, 1, 13) .. ":00:00" -- YYYY-MM-DD HH:00:00 - data["timestamp_tz_minute"] = tz_time -- YYYY-MM-DD HH:MM:00 -end - function _M.set_computed_url_fields(data, ngx_ctx) data["request_url_host"] = lowercase_truncate(data["request_url_host"], 200) @@ -306,11 +264,6 @@ function _M.set_computed_url_fields(data, ngx_ctx) data["request_url_query"] = escape_uri_non_ascii(parts[2]) end - data["legacy_request_url"] = data["request_url_scheme"] .. "://" .. data["request_url_host"] .. data["request_url_path"] - if data["request_url_query"] then - data["legacy_request_url"] = data["legacy_request_url"] .. "?" .. data["request_url_query"] - end - _M.set_url_hierarchy(data) end @@ -331,8 +284,8 @@ function _M.normalized_data(data) api_backend_resolved_host = remove_envoy_empty_header_value(lowercase_truncate(data["api_backend_resolved_host"], 200)), api_backend_response_code_details = remove_envoy_empty_header_value(truncate(data["api_backend_response_code_details"], 100)), api_backend_response_flags = remove_envoy_empty_header_value(truncate(data["api_backend_response_flags"], 20)), - denied_reason = lowercase_truncate(data["denied_reason"], 50), - id = lowercase_truncate(data["id"], 20), + gatekeeper_denied_code = lowercase_truncate(data["gatekeeper_denied_code"], 50), + request_id = lowercase_truncate(data["request_id"], 20), request_accept = truncate(data["request_accept"], 200), request_accept_encoding = truncate(data["request_accept_encoding"], 200), request_basic_auth_username = truncate(data["request_basic_auth_username"], 200), @@ -348,9 +301,8 @@ function _M.normalized_data(data) request_origin = truncate(data["request_origin"], 200), request_referer = truncate(data["request_referer"], 200), request_size = tonumber(data["request_size"]), - request_url_hierarchy = data["request_url_hierarchy"], - request_url_host = lowercase_truncate(data["request_url_host"], 200), - request_url_path = truncate(data["request_url_path"], 4000), + request_host = lowercase_truncate(data["request_url_host"], 200), + request_path = truncate(data["request_url_path"], 4000), request_url_hierarchy_level0 = truncate(data["request_url_hierarchy_level0"], 200), request_url_hierarchy_level1 = truncate(data["request_url_hierarchy_level1"], 200), request_url_hierarchy_level2 = truncate(data["request_url_hierarchy_level2"], 200), @@ -358,9 +310,8 @@ function _M.normalized_data(data) request_url_hierarchy_level4 = truncate(data["request_url_hierarchy_level4"], 200), request_url_hierarchy_level5 = truncate(data["request_url_hierarchy_level5"], 200), request_url_hierarchy_level6 = truncate(data["request_url_hierarchy_level6"], 200), - request_url_port = tonumber(data["request_url_port"]), request_url_query = truncate(data["request_url_query"], 4000), - request_url_scheme = lowercase_truncate(data["request_url_scheme"], 10), + request_scheme = lowercase_truncate(data["request_url_scheme"], 10), request_user_agent = truncate(data["request_user_agent"], 400), request_user_agent_family = truncate(data["request_user_agent_family"], 100), request_user_agent_type = truncate(data["request_user_agent_type"], 100), @@ -377,72 +328,42 @@ function _M.normalized_data(data) response_size = tonumber(data["response_size"]), response_status = tonumber(data["response_status"]), response_transfer_encoding = truncate(data["response_transfer_encoding"], 200), - timer_response = tonumber(data["timer_response"]), - timestamp_tz_date = uppercase_truncate(data["timestamp_tz_date"], 20), - timestamp_tz_hour = uppercase_truncate(data["timestamp_tz_hour"], 20), - timestamp_tz_minute = uppercase_truncate(data["timestamp_tz_minute"], 20), - timestamp_tz_month = uppercase_truncate(data["timestamp_tz_month"], 20), - timestamp_tz_offset = tonumber(data["timestamp_tz_offset"]), - timestamp_tz_week = uppercase_truncate(data["timestamp_tz_week"], 20), - timestamp_tz_year = uppercase_truncate(data["timestamp_tz_year"], 20), - timestamp_utc = tonumber(data["timestamp_utc"]), + response_time = tonumber(data["timer_response"]), + ["@timestamp"] = tonumber(data["timestamp_utc"]), user_id = lowercase_truncate(data["user_id"], 36), -- Deprecated - legacy_api_key = truncate(data["legacy_api_key"], 40), - legacy_request_url = truncate(data["legacy_request_url"], 8000), - legacy_user_email = truncate(data["legacy_user_email"], 200), - legacy_user_registration_source = truncate(data["legacy_user_registration_source"], 200), + api_key = truncate(data["legacy_api_key"], 40), + user_email = truncate(data["legacy_user_email"], 200), + user_registration_source = truncate(data["legacy_user_registration_source"], 200), } - if normalized["request_url_hierarchy"] then - for index, path in ipairs(normalized["request_url_hierarchy"]) do - normalized["request_url_hierarchy"][index] = truncate(path, 400) - end - end - return normalized end -function _M.build_syslog_message(data) - local syslog_message = "<" .. syslog_priority .. ">" - .. syslog_version - .. " " .. os.date("!%Y-%m-%dT%TZ", data["timestamp_utc"] / 1000) -- timestamp - .. " -" -- hostname - .. " api-umbrella" -- app-name - .. " -" -- procid - .. " -" -- msgid - .. " -" -- structured-data - .. " @cee:" -- CEE-enhanced logging for rsyslog to parse JSON - .. json_encode({ raw = data }) -- JSON data - .. "\n" - - return syslog_message -end - -function _M.send_syslog_message(syslog_message) - -- Check the syslog message length to ensure it doesn't exceed the configured - -- rsyslog maxMessageSize value. +function _M.send_message(message) + -- Check the message length to ensure it doesn't exceed the configured + -- Fluent Bit Buffer_Size. -- -- In general, this shouldn't be possible, since URLs can't exceed 8KB, and -- we truncate the various headers that users can control for logging -- purposes. However, this provides an extra sanity check to ensure this -- doesn't unexpectedly pop up (eg, if we add additional headers we forget to -- truncate). - local syslog_message_length = string.len(syslog_message) - if syslog_message_length > 32000 then - ngx.log(ngx.ERR, "request syslog message longer than expected - analytics logging may fail: ", syslog_message_length) + local message_length = string_len(message) + if message_length > 32000 then + ngx.log(ngx.ERR, "request message longer than expected - analytics logging may fail: ", message_length) end -- Init the resty logger socket. - if not logger.initted() then - local ok, err = logger.init{ - host = config["rsyslog"]["host"], - port = config["rsyslog"]["port"], + if not logger_initted() then + local ok, err = logger_init({ + host = config["fluent_bit"]["host"], + port = config["fluent_bit"]["port"], flush_limit = 4096, -- 4KB drop_limit = 10485760, -- 10MB periodic_flush = 0.1, - } + }) if not ok then ngx.log(ngx.ERR, "failed to initialize the logger: ", err) @@ -450,7 +371,7 @@ function _M.send_syslog_message(syslog_message) end end - return logger.log(syslog_message) + return logger_log(message) end return _M diff --git a/src/api-umbrella/proxy/opensearch_templates_data.lua b/src/api-umbrella/proxy/opensearch_templates_data.lua index 6c987bc18..d10347f62 100644 --- a/src/api-umbrella/proxy/opensearch_templates_data.lua +++ b/src/api-umbrella/proxy/opensearch_templates_data.lua @@ -5,17 +5,7 @@ local xpcall_error_handler = require "api-umbrella.utils.xpcall_error_handler" local opensearch_templates -local path = os.getenv("API_UMBRELLA_SRC_ROOT") .. "/config/opensearch_templates_v" .. config["opensearch"]["template_version"] -if config["opensearch"]["api_version"] >= 7 then - path = path .. "_es7.json.etlua" -elseif config["opensearch"]["api_version"] >= 5 then - path = path .. "_es5.json.etlua" -elseif config["opensearch"]["api_version"] >= 2 then - path = path .. "_es2.json.etlua" -else - error("Unsupported version of opensearch: " .. (config["opensearch"]["api_version"] or "")) -end - +local path = os.getenv("API_UMBRELLA_SRC_ROOT") .. "/config/opensearch_templates_v" .. config["opensearch"]["template_version"] .. ".json.etlua" local f, err = io.open(path, "rb") if err then ngx.log(ngx.ERR, "failed to open file: ", err) @@ -35,7 +25,7 @@ else -- In the test environment, disable replicas and reduce shards to speed -- things up. if config["app_env"] == "test" then - for _, template in ipairs(opensearch_templates) do + for _, template in pairs(opensearch_templates) do if not template["template"]["settings"] then template["template"]["settings"] = {} end diff --git a/src/api-umbrella/utils/aws_signing_v4.lua b/src/api-umbrella/utils/aws_signing_v4.lua deleted file mode 100644 index 59b0233f5..000000000 --- a/src/api-umbrella/utils/aws_signing_v4.lua +++ /dev/null @@ -1,186 +0,0 @@ -local nettle_hmac = require "resty.nettle.hmac" -local resty_sha256 = require "resty.sha256" -local to_hex = require("resty.string").to_hex - -local escape_uri = ngx.escape_uri -local gsub = ngx.re.gsub -local ngx_var = ngx.var -local now = ngx.now -local req_get_body_data = ngx.req.get_body_data -local req_get_headers = ngx.req.get_headers -local req_get_uri_args = ngx.req.get_uri_args -local req_read_body = ngx.req.read_body -local req_set_header = ngx.req.set_header - -local AWS_SERVICE = "es" -local UNSIGNED_HEADERS = { - authorization = 1, - expect = 1, -} - -local _M = {} - -local function hmac(secret_key, value) - assert(secret_key) - assert(value) - - local hmac_sha256 = nettle_hmac.sha256.new(secret_key) - hmac_sha256:update(value) - local binary = hmac_sha256:digest() - - return binary -end - -local function sha256_hexdigest(value) - local sha256 = resty_sha256:new() - sha256:update(value or "") - return to_hex(sha256:final()) -end - -local function canonical_header_name(name) - return string.lower(name) -end - -local function canonical_header_value(value) - return gsub(value, [[\s+]], " ", "jo") -end - -local function escape_uri_component(value) - if(value == true) then - return "" - else - return escape_uri(value or "") - end -end - -local function get_headers() - local headers = {} - - local raw_headers = req_get_headers() - for name, value in pairs(raw_headers) do - if type(value) == "table" then - for multi_name, multi_value in pairs(value) do - table.insert(headers, { - name = canonical_header_name(multi_name), - value = canonical_header_value(multi_value), - }) - end - else - table.insert(headers, { - name = canonical_header_name(name), - value = canonical_header_value(value), - }) - end - end - - return headers -end - -local function get_canonical_headers(headers) - local canonical = {} - for _, header in ipairs(headers) do - if not UNSIGNED_HEADERS[header.name] then - table.insert(canonical, header.name .. ":" .. header.value) - end - end - - table.sort(canonical) - return table.concat(canonical, "\n") -end - -local function get_signed_headers(headers) - local signed = {} - for _, header in ipairs(headers) do - if not UNSIGNED_HEADERS[header.name] then - table.insert(signed, header.name) - end - end - - table.sort(signed) - return table.concat(signed, ";") -end - -local function get_canonical_query_string() - local canonical = {} - local args = req_get_uri_args() - for name, value in pairs(args) do - if type(value) == "table" then - for multi_name, multi_value in pairs(value) do - table.insert(canonical, escape_uri_component(multi_name) .. "=" .. escape_uri_component(multi_value)) - end - else - table.insert(canonical, escape_uri_component(name) .. "=" .. escape_uri_component(value)) - end - end - - table.sort(canonical) - return table.concat(canonical, "&") -end - -local function get_canonical_request(headers, signed_headers, content_sha256) - return table.concat({ - ngx_var.request_method, - gsub(escape_uri(ngx_var.uri), [[%2F]], "/", "ijo"), - get_canonical_query_string(), - get_canonical_headers(headers) .. "\n", - signed_headers, - content_sha256, - }, "\n") -end - -local function get_credential_scope(aws_region, date) - return table.concat({ - date, - aws_region, - AWS_SERVICE, - "aws4_request", - }, "/") -end - -local function get_string_to_sign(datetime, credential_scope, canonical_request) - return table.concat({ - "AWS4-HMAC-SHA256", - datetime, - credential_scope, - sha256_hexdigest(canonical_request), - }, "\n") -end - -local function get_signature(aws_region, aws_secret_access_key, date, string_to_sign) - local k_date = hmac("AWS4" .. aws_secret_access_key, date) - local k_region = hmac(k_date, aws_region) - local k_service = hmac(k_region, AWS_SERVICE) - local k_credentials = hmac(k_service, "aws4_request") - return to_hex(hmac(k_credentials, string_to_sign)) -end - -local function get_authorization(aws_access_key_id, credential_scope, signed_headers, signature) - return table.concat({ - "AWS4-HMAC-SHA256 Credential=" .. aws_access_key_id .. "/" .. credential_scope, - "SignedHeaders=" .. signed_headers, - "Signature=" .. signature, - }, ", ") -end - -function _M.sign_request(aws_region, aws_access_key_id, aws_secret_access_key) - local datetime = os.date("!%Y%m%dT%H%M%SZ", now()) - local date = string.sub(datetime, 1, 8) - req_set_header("X-Amz-Date", os.date("!%Y%m%dT%H%M%SZ", now())) - - req_read_body() - local body = req_get_body_data() - local content_sha256 = sha256_hexdigest(body) - req_set_header("X-Amz-Content-Sha256", content_sha256) - - local headers = get_headers() - local signed_headers = get_signed_headers(headers) - local credential_scope = get_credential_scope(aws_region, date) - - local canonical_request = get_canonical_request(headers, signed_headers, content_sha256) - local string_to_sign = get_string_to_sign(datetime, credential_scope, canonical_request) - local signature = get_signature(aws_region, aws_secret_access_key, date, string_to_sign) - local authorization = get_authorization(aws_access_key_id, credential_scope, signed_headers, signature) - req_set_header("Authorization", authorization) -end - -return _M diff --git a/src/api-umbrella/utils/generate_runtime_config.lua b/src/api-umbrella/utils/generate_runtime_config.lua index 621d855d3..e4def17ff 100644 --- a/src/api-umbrella/utils/generate_runtime_config.lua +++ b/src/api-umbrella/utils/generate_runtime_config.lua @@ -431,13 +431,8 @@ local function set_computed_config(config) }, opensearch = { _first_server = config["opensearch"]["_servers"][1], - ["_index_partition_monthly?"] = (config["opensearch"]["index_partition"] == "monthly"), - ["_index_partition_daily?"] = (config["opensearch"]["index_partition"] == "daily"), - ["_template_version_v1?"] = (config["opensearch"]["template_version"] == 1), - ["_template_version_v2?"] = (config["opensearch"]["template_version"] == 2), }, ["_service_egress_enabled?"] = array_includes(config["services"], "egress"), - ["_service_opensearch_aws_signing_proxy_enabled?"] = array_includes(config["services"], "opensearch_aws_signing_proxy"), ["_service_router_enabled?"] = array_includes(config["services"], "router"), ["_service_web_enabled?"] = array_includes(config["services"], "web"), router = { diff --git a/src/api-umbrella/utils/opensearch.lua b/src/api-umbrella/utils/opensearch.lua index 9df56a215..e99d69a5b 100644 --- a/src/api-umbrella/utils/opensearch.lua +++ b/src/api-umbrella/utils/opensearch.lua @@ -1,6 +1,5 @@ local config = require("api-umbrella.utils.load_config")() local http = require "resty.http" -local icu_date = require "icu-date-ffi" local is_empty = require "api-umbrella.utils.is_empty" local json_decode = require("cjson").decode local json_encode = require "api-umbrella.utils.json_encode" @@ -10,14 +9,6 @@ local server = config["opensearch"]["_first_server"] local _M = {} -if config["opensearch"]["index_partition"] == "monthly" then - _M.partition_date_format = icu_date.formats.pattern("yyyy-MM") -elseif config["opensearch"]["index_partition"] == "daily" then - _M.partition_date_format = icu_date.formats.pattern("yyyy-MM-dd") -else - error("Unknown opensearch.index_partition configuration value") -end - function _M.query(path, options) local httpc = http.new() diff --git a/tasks/deps/fluent-bit b/tasks/deps/fluent-bit new file mode 100755 index 000000000..c9de4c7db --- /dev/null +++ b/tasks/deps/fluent-bit @@ -0,0 +1,156 @@ +#!/usr/bin/env bash + +set -e -u -x +source ./tasks/helpers.sh + +fluent_bit_version="2.2.2" +fluent_bit_hash="8e7e951b2907e9d29508699c71c8949a4a22d750d54ffa5ee5b96537e59371dd" + +task_working_dir +download "https://github.com/fluent/fluent-bit/archive/refs/tags/v${fluent_bit_version}.tar.gz" "sha256" "$fluent_bit_hash" +extract_download "80a36d3682-v${fluent_bit_version}.tar.gz" + +cd "fluent-bit-${fluent_bit_version}/build" + +cmake \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_PREFIX_EMBEDDED" \ + -DFLB_CUSTOM_CALYPTIA="Off" \ + -DFLB_EXAMPLES="Off" \ + -DFLB_HTTP_SERVER="On" \ + -DFLB_JEMALLOC="On" \ + -DFLB_LUAJIT="Off" \ + -DFLB_PROXY_GO="Off" \ + -DFLB_RECORD_ACCESSOR="On" \ + -DFLB_RELEASE="On" \ + -DFLB_STREAM_PROCESSOR="Off" \ + -DFLB_WASM="Off" \ + -DFLB_WINDOWS_DEFAULTS="Off" \ + -DFLB_FILTER_ALTER_SIZE="Off" \ + -DFLB_FILTER_AWS="Off" \ + -DFLB_FILTER_CHECKLIST="Off" \ + -DFLB_FILTER_ECS="Off" \ + -DFLB_FILTER_EXPECT="Off" \ + -DFLB_FILTER_GEOIP2="Off" \ + -DFLB_FILTER_GREP="Off" \ + -DFLB_FILTER_KUBERNETES="Off" \ + -DFLB_FILTER_LOG_TO_METRICS="Off" \ + -DFLB_FILTER_LUA="Off" \ + -DFLB_FILTER_LUA_USE_MPACK="Off" \ + -DFLB_FILTER_MODIFY="Off" \ + -DFLB_FILTER_MULTILINE="Off" \ + -DFLB_FILTER_NEST="Off" \ + -DFLB_FILTER_NIGHTFALL="Off" \ + -DFLB_FILTER_PARSER="Off" \ + -DFLB_FILTER_RECORD_MODIFIER="Off" \ + -DFLB_FILTER_REWRITE_TAG="On" \ + -DFLB_FILTER_STDOUT="Off" \ + -DFLB_FILTER_SYSINFO="Off" \ + -DFLB_FILTER_TENSORFLOW="Off" \ + -DFLB_FILTER_THROTTLE="Off" \ + -DFLB_FILTER_THROTTLE_SIZE="Off" \ + -DFLB_FILTER_TYPE_CONVERTER="Off" \ + -DFLB_FILTER_WASM="Off" \ + -DFLB_IN_CALYPTIA_FLEET="Off" \ + -DFLB_IN_COLLECTD="Off" \ + -DFLB_IN_CPU="Off" \ + -DFLB_IN_DISK="Off" \ + -DFLB_IN_DOCKER="Off" \ + -DFLB_IN_DOCKER_EVENTS="Off" \ + -DFLB_IN_DUMMY="Off" \ + -DFLB_IN_ELASTICSEARCH="Off" \ + -DFLB_IN_EMITTER="On" \ + -DFLB_IN_EVENT_TEST="Off" \ + -DFLB_IN_EVENT_TYPE="Off" \ + -DFLB_IN_EXEC="Off" \ + -DFLB_IN_EXEC_WASI="Off" \ + -DFLB_IN_FLUENTBIT_METRICS="On" \ + -DFLB_IN_FORWARD="Off" \ + -DFLB_IN_HEAD="Off" \ + -DFLB_IN_HEALTH="Off" \ + -DFLB_IN_HTTP="Off" \ + -DFLB_IN_KAFKA="Off" \ + -DFLB_IN_KMSG="Off" \ + -DFLB_IN_KUBERNETES_EVENTS="Off" \ + -DFLB_IN_LIB="Off" \ + -DFLB_IN_MEM="Off" \ + -DFLB_IN_MQTT="Off" \ + -DFLB_IN_NETIF="Off" \ + -DFLB_IN_NGINX_EXPORTER_METRICS="Off" \ + -DFLB_IN_NODE_EXPORTER_METRICS="Off" \ + -DFLB_IN_OPENTELEMETRY="Off" \ + -DFLB_IN_PODMAN_METRICS="Off" \ + -DFLB_IN_PROC="Off" \ + -DFLB_IN_PROCESS_EXPORTER_METRICS="Off" \ + -DFLB_IN_PROMETHEUS_SCRAPE="Off" \ + -DFLB_IN_RANDOM="Off" \ + -DFLB_IN_SERIAL="Off" \ + -DFLB_IN_SPLUNK="Off" \ + -DFLB_IN_STATSD="Off" \ + -DFLB_IN_STDIN="Off" \ + -DFLB_IN_STORAGE_BACKLOG="On" \ + -DFLB_IN_SYSLOG="Off" \ + -DFLB_IN_SYSTEMD="Off" \ + -DFLB_IN_TAIL="On" \ + -DFLB_IN_TCP="On" \ + -DFLB_IN_THERMAL="Off" \ + -DFLB_IN_UDP="Off" \ + -DFLB_IN_UNIX_SOCKET="Off" \ + -DFLB_IN_WINDOWS_EXPORTER_METRICS="Off" \ + -DFLB_IN_WINEVTLOG="Off" \ + -DFLB_IN_WINLOG="Off" \ + -DFLB_IN_WINSTAT="Off" \ + -DFLB_OUT_AZURE="Off" \ + -DFLB_OUT_AZURE_BLOB="Off" \ + -DFLB_OUT_AZURE_KUSTO="Off" \ + -DFLB_OUT_AZURE_LOGS_INGESTION="Off" \ + -DFLB_OUT_BIGQUERY="Off" \ + -DFLB_OUT_CALYPTIA="Off" \ + -DFLB_OUT_CHRONICLE="Off" \ + -DFLB_OUT_CLOUDWATCH_LOGS="Off" \ + -DFLB_OUT_COUNTER="Off" \ + -DFLB_OUT_DATADOG="Off" \ + -DFLB_OUT_ES="Off" \ + -DFLB_OUT_EXIT="Off" \ + -DFLB_OUT_FILE="Off" \ + -DFLB_OUT_FLOWCOUNTER="Off" \ + -DFLB_OUT_FORWARD="Off" \ + -DFLB_OUT_GELF="Off" \ + -DFLB_OUT_HTTP="Off" \ + -DFLB_OUT_INFLUXDB="Off" \ + -DFLB_OUT_KAFKA="Off" \ + -DFLB_OUT_KAFKA_REST="Off" \ + -DFLB_OUT_KINESIS_FIREHOSE="Off" \ + -DFLB_OUT_KINESIS_STREAMS="Off" \ + -DFLB_OUT_LIB="Off" \ + -DFLB_OUT_LOGDNA="Off" \ + -DFLB_OUT_LOKI="Off" \ + -DFLB_OUT_NATS="Off" \ + -DFLB_OUT_NRLOGS="Off" \ + -DFLB_OUT_NULL="Off" \ + -DFLB_OUT_OPENSEARCH="On" \ + -DFLB_OUT_OPENTELEMETRY="Off" \ + -DFLB_OUT_ORACLE_LOG_ANALYTICS="Off" \ + -DFLB_OUT_PGSQL="Off" \ + -DFLB_OUT_PLOT="Off" \ + -DFLB_OUT_PROMETHEUS_EXPORTER="Off" \ + -DFLB_OUT_PROMETHEUS_REMOTE_WRITE="Off" \ + -DFLB_OUT_RETRY="Off" \ + -DFLB_OUT_S3="On" \ + -DFLB_OUT_SKYWALKING="Off" \ + -DFLB_OUT_SLACK="Off" \ + -DFLB_OUT_SPLUNK="Off" \ + -DFLB_OUT_STACKDRIVER="Off" \ + -DFLB_OUT_STDOUT="On" \ + -DFLB_OUT_SYSLOG="Off" \ + -DFLB_OUT_TCP="Off" \ + -DFLB_OUT_TD="Off" \ + -DFLB_OUT_UDP="Off" \ + -DFLB_OUT_VIVO_EXPORTER="Off" \ + -DFLB_OUT_WEBSOCKET="Off" \ + -DFLB_PROCESSOR_ATTRIBUTES="Off" \ + -DFLB_PROCESSOR_LABELS="Off" \ + ../ +make +make install + +stamp diff --git a/tasks/outdated.thor b/tasks/outdated.thor index 5442171bb..26f511d68 100644 --- a/tasks/outdated.thor +++ b/tasks/outdated.thor @@ -20,6 +20,9 @@ class Outdated < Thor "envoy_control_plane" => { :git => "https://github.com/GUI/envoy-control-plane.git", }, + "fluent-bit" => { + :git => "https://github.com/fluent/fluent-bit.git", + }, "glauth" => { :git => "https://github.com/glauth/glauth.git", }, @@ -56,9 +59,6 @@ class Outdated < Thor "perp" => { :http => "http://b0llix.net/perp/site.cgi?page=download", }, - "rsyslog" => { - :git => "https://github.com/rsyslog/rsyslog.git", - }, "shellcheck" => { :git => "https://github.com/koalaman/shellcheck.git", }, diff --git a/templates/etc/fluent-bit/fluent-bit.yaml.etlua b/templates/etc/fluent-bit/fluent-bit.yaml.etlua new file mode 100644 index 000000000..3bc0a5a73 --- /dev/null +++ b/templates/etc/fluent-bit/fluent-bit.yaml.etlua @@ -0,0 +1,95 @@ +service: + log_level: info + storage.path: /tmp/fluentbit + storage.sync: normal + storage.checksum: off + storage.max_chunks_up: 256 + storage.backlog.mem_limit: 5M + +pipeline: + inputs: + - name: tcp + listen: <%- json_encode(config["fluent_bit"]["host"]) %> + port: <%- json_encode(config["fluent_bit"]["port"]) %> + format: json + tag: logs + # mem_buf_limit: 500M + # storage.type: filesystem + # storage.pause_on_chunks_overlimit: on + + - name: fluentbit_metrics + scrape_interval: 10 + tag: internal_metrics + + filters: + - name: rewrite_tag + match: logs + rule: "$gatekeeper_denied_code ^. denied_logs false" + + outputs: + - name: stdout + # match: logs denied_logs + match: "*" + json_date_key: false + format: json_lines + + - name: opensearch + match: logs + # aws_auth: on + # aws_region: us-west-2 + host: <%- json_encode(config["opensearch"]["_first_server"]["host"]) %> + port: <%- json_encode(config["opensearch"]["_first_server"]["port"]) %> + tls: <%- config["opensearch"]["_first_server"]["_https?"] and "on" or "off" %> + <% if config["opensearch"]["_first_server"]["user"] then %> + http_user: <%- json_encode(config["opensearch"]["_first_server"]["user"]) %> + <% end %> + <% if config["opensearch"]["_first_server"]["password"] then %> + http_passwd: <%- json_encode(config["opensearch"]["_first_server"]["password"]) %>" + <% end %> + index: <%- json_encode(config["opensearch"]["index_name_prefix"] .. "-logs-v" .. config["opensearch"]["template_version"] .. "-all") %> + suppress_type_name: on + id_key: request_id + # workers: 6 + compress: gzip + trace_error: on + # storage.total_limit_size: 4GB + # generate_id: on + # buffer_size: false + + - name: opensearch + match: denied_logs + # aws_auth: on + # aws_region: us-west-2 + host: <%- json_encode(config["opensearch"]["_first_server"]["host"]) %> + port: <%- json_encode(config["opensearch"]["_first_server"]["port"]) %> + tls: <%- config["opensearch"]["_first_server"]["_https?"] and "on" or "off" %> + <% if config["opensearch"]["_first_server"]["user"] then %> + http_user: <%- json_encode(config["opensearch"]["_first_server"]["user"]) %> + <% end %> + <% if config["opensearch"]["_first_server"]["password"] then %> + http_passwd: <%- json_encode(config["opensearch"]["_first_server"]["password"]) %>" + <% end %> + index: <%- json_encode(config["opensearch"]["index_name_prefix"] .. "-logs-v" .. config["opensearch"]["template_version"] .. "-denied") %> + suppress_type_name: on + id_key: request_id + # workers: 6 + compress: gzip + trace_error: on + # storage.total_limit_size: 4GB + # generate_id: on + # buffer_size: false + + # - name: s3 + # match: logs denied_logs + # compression: on + # upload_chunk_size: 30M + # total_file_size: 250M + # upload_timeout: 60m + # s3_key_format: "/$TAG/%Y/%m/%d/%Y-%m-%dT%H:%M:%SZ-$UUID.gz + # send_content_md5: true + # auto_retry_requests: true + # preserve_data_ordering: true + # retry_limit: 5 + + + diff --git a/templates/etc/nginx/opensearch-aws-signing-proxy.conf.etlua b/templates/etc/nginx/opensearch-aws-signing-proxy.conf.etlua deleted file mode 100644 index b01aae6a7..000000000 --- a/templates/etc/nginx/opensearch-aws-signing-proxy.conf.etlua +++ /dev/null @@ -1,70 +0,0 @@ -worker_processes <%- config["opensearch"]["aws_signing_proxy"]["workers"] %>; -error_log stderr <%- config["opensearch"]["aws_signing_proxy"]["error_log_level"] %>; -daemon off; -pid <%- config["run_dir"] %>/opensearch-aws-signing-proxy.pid; - -<% if config["user"] then %> - user <%- config["user"] %> <%- config["group"] %>; -<% end %> - -events { - worker_connections <%- config["opensearch"]["aws_signing_proxy"]["worker_connections"] %>; -} - -env API_UMBRELLA_SRC_ROOT; -env API_UMBRELLA_RUNTIME_CONFIG; - -pcre_jit on; - -http { - access_log <%- config["log_dir"] %>/opensearch-aws-signing-proxy/<%- config["nginx"]["access_log_filename"] %> combined; - - client_body_temp_path <%- config["tmp_dir"] %>/opensearch-aws-signing-proxy-client_body_temp; - proxy_temp_path <%- config["tmp_dir"] %>/opensearch-aws-signing-proxy-proxy_temp; - fastcgi_temp_path <%- config["tmp_dir"] %>/opensearch-aws-signing-proxy-fastcgi_temp; - uwsgi_temp_path <%- config["tmp_dir"] %>/opensearch-aws-signing-proxy-uwsgi_temp; - scgi_temp_path <%- config["tmp_dir"] %>/opensearch-aws-signing-proxy-scgi_temp; - server_tokens off; - - <% if config["nginx"]["_lua_ssl_trusted_certificate"] then %> - lua_ssl_trusted_certificate <%- config["nginx"]["_lua_ssl_trusted_certificate"] %>; - lua_ssl_verify_depth <%- config["nginx"]["lua_ssl_verify_depth"] %>; - <% end %> - - lua_package_path '<%- config["_package_path"] %>'; - lua_package_cpath '<%- config["_package_cpath"] %>'; - lua_check_client_abort on; - if_modified_since off; - - lua_shared_dict locks <%- config["nginx"]["shared_dicts"]["locks"]["size"] %>; - - <% if config["dns_resolver"]["_nameservers_nginx"] then %> - resolver <%- config["dns_resolver"]["_nameservers_nginx"] %>; - resolver_timeout 12s; - <% end %> - - include ./mime.conf; - include ./realip.conf; - - client_max_body_size 10m; - client_body_buffer_size 10m; - - init_by_lua_file '<%- config["_src_root_dir"] %>/src/api-umbrella/opensearch-aws-signing-proxy/init.lua'; - - server { - listen <%- config["opensearch"]["aws_signing_proxy"]["host"] %>:<%- config["opensearch"]["aws_signing_proxy"]["port"] %> so_keepalive=<%- config["opensearch"]["aws_signing_proxy"]["listen_so_keepalive"] %><% if config["opensearch"]["aws_signing_proxy"]["listen_backlog"] then %> backlog=<%- config["opensearch"]["aws_signing_proxy"]["listen_backlog"] %><% end %>; - - <% if config["_development_env?"] then %> - lua_code_cache off; - <% end %> - - location / { - access_by_lua_file '<%- config["_src_root_dir"] %>/src/api-umbrella/opensearch-aws-signing-proxy/proxy.lua'; - - proxy_buffering off; - proxy_request_buffering off; - set $backend_upstream "https://<%- config["opensearch"]["aws_signing_proxy"]["aws_host"] %>:443"; - proxy_pass $backend_upstream; - } - } -} diff --git a/templates/etc/perp/rsyslog/rc.env.etlua b/templates/etc/perp/fluent-bit/rc.env.etlua similarity index 100% rename from templates/etc/perp/rsyslog/rc.env.etlua rename to templates/etc/perp/fluent-bit/rc.env.etlua diff --git a/templates/etc/perp/opensearch-aws-signing-proxy/rc.log b/templates/etc/perp/fluent-bit/rc.log similarity index 100% rename from templates/etc/perp/opensearch-aws-signing-proxy/rc.log rename to templates/etc/perp/fluent-bit/rc.log diff --git a/templates/etc/perp/rsyslog/rc.main.etlua b/templates/etc/perp/fluent-bit/rc.main.etlua similarity index 88% rename from templates/etc/perp/rsyslog/rc.main.etlua rename to templates/etc/perp/fluent-bit/rc.main.etlua index 602c12d46..f7c56ccba 100755 --- a/templates/etc/perp/rsyslog/rc.main.etlua +++ b/templates/etc/perp/fluent-bit/rc.main.etlua @@ -35,8 +35,6 @@ if [ "${1}" = "start" ]; then exec \ runtool ${run_args[@]+"${run_args[@]}"} \ - rsyslogd \ - -n \ - -f "<%- config['etc_dir'] %>/rsyslog.conf" \ - -i "<%- config['run_dir'] %>/rsyslogd.pid" + fluent-bit \ + --config "<%- config['etc_dir'] %>/fluent-bit/fluent-bit.yaml" fi diff --git a/templates/etc/perp/opensearch-aws-signing-proxy/rc.env.etlua b/templates/etc/perp/opensearch-aws-signing-proxy/rc.env.etlua deleted file mode 100644 index af194cc8d..000000000 --- a/templates/etc/perp/opensearch-aws-signing-proxy/rc.env.etlua +++ /dev/null @@ -1 +0,0 @@ -API_UMBRELLA_RUNTIME_CONFIG=<%- config["_runtime_config_path"] %> diff --git a/templates/etc/perp/opensearch-aws-signing-proxy/rc.main.etlua b/templates/etc/perp/opensearch-aws-signing-proxy/rc.main.etlua deleted file mode 100755 index 8a767e850..000000000 --- a/templates/etc/perp/opensearch-aws-signing-proxy/rc.main.etlua +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# Redirect stderr to stdout -exec 2>&1 - -if [ "${1}" = "start" ]; then - echo "starting ${2}..." - api_umbrella_user="<%- config['user'] %>" - api_umbrella_group="<%- config['group'] %>" - - destination="<%- config['log']['destination'] %>" - if [ "$destination" = "console" ]; then - if [ -n "$api_umbrella_user" ]; then - chown "$api_umbrella_user":"$api_umbrella_group" /dev/stdout - fi - - ln -sf /dev/stdout "<%- config['log_dir'] %>/opensearch-aws-signing-proxy/access.log" - fi - - run_args=("-e" "rc.env" "-c" "<%- config['_src_root_dir'] %>") - exec runtool "${run_args[@]}" nginx -p "<%- config['_src_root_dir'] %>/" -c "<%- config['etc_dir'] %>/nginx/opensearch-aws-signing-proxy.conf" -fi - -exit 0 diff --git a/templates/etc/perp/rsyslog/rc.log b/templates/etc/perp/rsyslog/rc.log deleted file mode 100755 index 0bfdd70ef..000000000 --- a/templates/etc/perp/rsyslog/rc.log +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -set -e -u -exec ../rc.log "$@" diff --git a/templates/etc/rsyslog.conf.etlua b/templates/etc/rsyslog.conf.etlua deleted file mode 100644 index e41876d58..000000000 --- a/templates/etc/rsyslog.conf.etlua +++ /dev/null @@ -1,91 +0,0 @@ -# Allow for longer message sizes to handle the JSON data containing long URLs -# or HTTP headers. -global(maxMessageSize="32k") - -global(workDirectory="<%- config["db_dir"] %>/rsyslog") - -# Load Modules -module(load="imfile") -module(load="imtcp" MaxSessions="500") -module(load="mmjsonparse") -module(load="mmutf8fix") -<% if config["log"]["destination"] == "console" then %> - module(load="omstdout") -<% end %> -<% if config["analytics"]["_output_opensearch?"] then %> - module(load="omelasticsearch") -<% end %> - -# Define templates for OpenSearch output. -<% if config["opensearch"]["_index_partition_monthly?"] then %> - template(name="opensearch-index" type="string" string="<%- config["opensearch"]["index_name_prefix"] %>-logs-write-%timereported:1:7:date-rfc3339,date-utc%") -<% end %> -<% if config["opensearch"]["_index_partition_daily?"] then %> - template(name="opensearch-index" type="string" string="<%- config["opensearch"]["index_name_prefix"] %>-logs-write-%timereported:1:10:date-rfc3339,date-utc%") -<% end %> -template(name="opensearch-id" type="subtree" subtree="$!raw!id") -template(name="opensearch-json-record" type="subtree" subtree="$!usr!es") -template(name="all-json-record" type="list") { - property(name="$!raw") constant(value="\n") -} - -template(name="template-stdout" type="list") { - constant(value="[") - property(name="syslogtag") - constant(value="]") - property(name="msg" spifno1stsp="on") - property(name="msg" droplastlf="on") - constant(value="\n") -} - -ruleset(name="analytics-ruleset" queue.type="FixedArray" queue.size="5000") { - include(file="<%- config["etc_dir"] %>/rsyslog.d/analytics.conf") -} - -<% if config["log"]["destination"] == "console" then %> - ruleset(name="logs-ruleset" queue.type="FixedArray" queue.size="5000") { - action( - name="output-logs-stdout" - type="omstdout" - template="template-stdout" - ) - } - - # Workaround for error.log currently being hard-coded to output to a file - # that won't work if symlinked to /dev/stdout (it insists on rolling the file - # since it doesn't think it can write to it). - # - # This workaround should no longer be needed once Trafficserver 10 is - # released and things can be configured to output to stdout/stderr directly: - # https://github.com/apache/trafficserver/pull/7937 - input(type="imfile" file="<%- config["log_dir"] %>/trafficserver/error.log" tag="trafficserver-error" ruleset="logs-ruleset") -<% end %> - -# TCP Input -input(type="imtcp" address="<%- config["rsyslog"]["host"] %>" port="<%- config["rsyslog"]["port"] %>" ruleset="analytics-ruleset") - -ruleset(name="stats-ruleset" queue.type="FixedArray" queue.size="5000") { - <% if config["log"]["destination"] == "console" then %> - action( - name="output-stats-stdout" - type="omstdout" - template="template-stdout" - ) - <% else %> - action( - name="output-stats-file" - type="omfile" - file="<%- config["log_dir"] %>/rsyslog/stats.log" - ) - <% end %> -} - -# Output queue statistics periodically so the health of the queue can be -# inspected. -module( - load="impstats" - interval="60" - resetCounters="on" - log.syslog="on" - ruleset="stats-ruleset" -) diff --git a/templates/etc/rsyslog.d/analytics.conf.etlua b/templates/etc/rsyslog.d/analytics.conf.etlua deleted file mode 100644 index 8e8d953b5..000000000 --- a/templates/etc/rsyslog.d/analytics.conf.etlua +++ /dev/null @@ -1,253 +0,0 @@ -# Ensure we have only valid UTF-8 characters in the log message (otherwise -# OpenSearch is unhappy). -action(name="analytics-utf8fix" type="mmutf8fix") - -# Parse the JSON data out of the @cee-enhanced logs. -action(name="analytics-jsonparse" type="mmjsonparse") - -<% if config["analytics"]["_output_opensearch?"] then %> - # Define the OpenSearch-based output. - # This aliases some of the logged fields to different names for backwards - # compatibility with the OpenSearch data. - if($!raw!api_backend_id != "") then { - set $!usr!es!api_backend_id = $!raw!api_backend_id; - } - if($!raw!api_backend_url_match_id != "") then { - set $!usr!es!api_backend_url_match_id = $!raw!api_backend_url_match_id; - } - if($!raw!legacy_api_key != "") then { - set $!usr!es!api_key = $!raw!legacy_api_key; - } - <% if config["opensearch"]["_template_version_v2?"] then %> - if($!raw!api_backend_resolved_host != "") then { - set $!usr!es!api_backend_resolved_host = $!raw!api_backend_resolved_host; - } - if($!raw!api_backend_response_code_details != "") then { - set $!usr!es!api_backend_response_code_details = $!raw!api_backend_response_code_details; - } - if($!raw!api_backend_response_flags != "") then { - set $!usr!es!api_backend_response_flags = $!raw!api_backend_response_flags; - } - <% end %> - if($!raw!denied_reason != "") then { - set $!usr!es!gatekeeper_denied_code = $!raw!denied_reason; - } - if($!raw!request_accept != "") then { - set $!usr!es!request_accept = $!raw!request_accept; - } - if($!raw!request_accept_encoding != "") then { - set $!usr!es!request_accept_encoding = $!raw!request_accept_encoding; - } - if($!raw!timestamp_utc != "") then { - set $!usr!es!request_at = $!raw!timestamp_utc; - } - if($!raw!request_basic_auth_username != "") then { - set $!usr!es!request_basic_auth_username = $!raw!request_basic_auth_username; - } - if($!raw!request_connection != "") then { - set $!usr!es!request_connection = $!raw!request_connection; - } - if($!raw!request_content_type != "") then { - set $!usr!es!request_content_type = $!raw!request_content_type; - } - <% if config["opensearch"]["_template_version_v1?"] then %> - if($!raw!request_url_hierarchy != "") then { - set $!usr!es!request_hierarchy = $!raw!request_url_hierarchy; - } - <% end %> - <% if config["opensearch"]["_template_version_v2?"] then %> - if($!raw!request_url_hierarchy_level0 != "") then { - set $!usr!es!request_url_hierarchy_level0 = $!raw!request_url_hierarchy_level0; - } - if($!raw!request_url_hierarchy_level1 != "") then { - set $!usr!es!request_url_hierarchy_level1 = $!raw!request_url_hierarchy_level1; - } - if($!raw!request_url_hierarchy_level2 != "") then { - set $!usr!es!request_url_hierarchy_level2 = $!raw!request_url_hierarchy_level2; - } - if($!raw!request_url_hierarchy_level3 != "") then { - set $!usr!es!request_url_hierarchy_level3 = $!raw!request_url_hierarchy_level3; - } - if($!raw!request_url_hierarchy_level4 != "") then { - set $!usr!es!request_url_hierarchy_level4 = $!raw!request_url_hierarchy_level4; - } - if($!raw!request_url_hierarchy_level5 != "") then { - set $!usr!es!request_url_hierarchy_level5 = $!raw!request_url_hierarchy_level5; - } - if($!raw!request_url_hierarchy_level6 != "") then { - set $!usr!es!request_url_hierarchy_level6 = $!raw!request_url_hierarchy_level6; - } - <% end %> - if($!raw!request_url_host != "") then { - set $!usr!es!request_host = $!raw!request_url_host; - } - if($!raw!request_ip != "") then { - set $!usr!es!request_ip = $!raw!request_ip; - } - if($!raw!request_ip_city != "") then { - set $!usr!es!request_ip_city = $!raw!request_ip_city; - } - if($!raw!request_ip_country != "") then { - set $!usr!es!request_ip_country = $!raw!request_ip_country; - } - if($!raw!request_ip_region != "") then { - set $!usr!es!request_ip_region = $!raw!request_ip_region; - } - if($!raw!request_method != "") then { - set $!usr!es!request_method = $!raw!request_method; - } - if($!raw!request_origin != "") then { - set $!usr!es!request_origin = $!raw!request_origin; - } - if($!raw!request_url_path != "") then { - set $!usr!es!request_path = $!raw!request_url_path; - } - if($!raw!request_referer != "") then { - set $!usr!es!request_referer = $!raw!request_referer; - } - if($!raw!request_url_scheme != "") then { - set $!usr!es!request_scheme = $!raw!request_url_scheme; - } - if($!raw!request_size != "") then { - set $!usr!es!request_size = $!raw!request_size; - } - <% if config["opensearch"]["_template_version_v1?"] then %> - if($!raw!legacy_request_url != "") then { - set $!usr!es!request_url = $!raw!legacy_request_url; - } - <% end %> - if($!raw!request_url_query != "") then { - set $!usr!es!request_url_query = $!raw!request_url_query; - } - if($!raw!request_user_agent != "") then { - set $!usr!es!request_user_agent = $!raw!request_user_agent; - } - if($!raw!request_user_agent_family != "") then { - set $!usr!es!request_user_agent_family = $!raw!request_user_agent_family; - } - if($!raw!request_user_agent_type != "") then { - set $!usr!es!request_user_agent_type = $!raw!request_user_agent_type; - } - if($!raw!response_age != "") then { - set $!usr!es!response_age = $!raw!response_age; - } - if($!raw!response_cache != "") then { - set $!usr!es!response_cache = $!raw!response_cache; - } - <% if config["opensearch"]["_template_version_v2?"] then %> - if($!raw!response_cache_flags != "") then { - set $!usr!es!response_cache_flags = $!raw!response_cache_flags; - } - <% end %> - if($!raw!response_content_encoding != "") then { - set $!usr!es!response_content_encoding = $!raw!response_content_encoding; - } - if($!raw!response_content_length != "") then { - set $!usr!es!response_content_length = $!raw!response_content_length; - } - if($!raw!response_content_type != "") then { - set $!usr!es!response_content_type = $!raw!response_content_type; - } - <% if config["opensearch"]["_template_version_v2?"] then %> - if($!raw!response_custom1 != "") then { - set $!usr!es!response_custom1 = $!raw!response_custom1; - } - if($!raw!response_custom2 != "") then { - set $!usr!es!response_custom2 = $!raw!response_custom2; - } - if($!raw!response_custom3 != "") then { - set $!usr!es!response_custom3 = $!raw!response_custom3; - } - <% end %> - if($!raw!response_server != "") then { - set $!usr!es!response_server = $!raw!response_server; - } - if($!raw!response_size != "") then { - set $!usr!es!response_size = $!raw!response_size; - } - if($!raw!response_status != "") then { - set $!usr!es!response_status = $!raw!response_status; - } - if($!raw!timer_response != "") then { - set $!usr!es!response_time = $!raw!timer_response; - } - if($!raw!response_transfer_encoding != "") then { - set $!usr!es!response_transfer_encoding = $!raw!response_transfer_encoding; - } - if($!raw!legacy_user_email != "") then { - set $!usr!es!user_email = $!raw!legacy_user_email; - } - if($!raw!user_id != "") then { - set $!usr!es!user_id = $!raw!user_id; - } - if($!raw!legacy_user_registration_source != "") then { - set $!usr!es!user_registration_source = $!raw!legacy_user_registration_source; - } - - # Output to OpenSearch. - # A disk-assisted memory queue is used for buffering. - action( - name="analytics-output-opensearch" - type="omelasticsearch" - server="<%- config["opensearch"]["_first_server"]["host"] %>" - serverport="<%- config["opensearch"]["_first_server"]["port"] %>" - <% if config["opensearch"]["_first_server"]["_https?"] then %> - usehttps="on" - <% end %> - <% if config["opensearch"]["_first_server"]["user"] then %> - uid="<%- config["opensearch"]["_first_server"]["user"] %>" - <% end %> - <% if config["opensearch"]["_first_server"]["password"] then %> - pwd="<%- config["opensearch"]["_first_server"]["password"] %>" - <% end %> - searchIndex="opensearch-index" - dynSearchIndex="on" - searchType="" - template="opensearch-json-record" - - # Enable bulk indexing, so batches of records are sent as a single HTTP - # request. - bulkmode="on" - bulkId="opensearch-id" - dynBulkId="on" - - errorfile="<%- config["log_dir"] %>/rsyslog/opensearch_error.log" - - # Allow bulk indexing of batches *up to* this size. - # - # Note that as long as OpenSearch is keeping up and data isn't being - # queued by rsyslog, then rsyslog will send data as quickly as it can to - # OpenSearch (so the batch sizes might be much smaller). See - # http://www.gossamer-threads.com/lists/rsyslog/users/17550 - queue.dequeuebatchsize="5000" - - # For the in-memory queue, use a linked-list (so the memory doesn't have to - # be pre-allocated based on a fixed size). - queue.type="LinkedList" - - # Set a filename, so the queue is disk assisted. This allows for offloading - # the data from the memory queue to disk if the queue becomes bigger than - # expected. - queue.filename="queue-opensearch" - - # Set thresholds for when the memory queue is too big and should use the - # disk (note the disk queue size is not bounded by the queue.size, that only - # applies to the memory portion). - queue.size="15000" - queue.highwatermark="10000" - queue.lowwatermark="2000" - - # Persist data to disk on this interval (in seconds). We're okay with some - # loss in the event of unexpected failures. - queue.checkpointinterval="10" - - # Persist data to disk on graceful shutdowns. - queue.saveonshutdown="on" - - # If OpenSearch is inaccessible, retry on this interval (in seconds) - # indefinitely (so we don't stop logging to OpenSearch in case it goes - # down for a longer period of time). - action.resumeInterval="30" - action.resumeRetryCount="-1" - ) -<% end %> diff --git a/templates/etc/trafficserver/records.config.etlua b/templates/etc/trafficserver/records.config.etlua index 1a6de7465..74527cd59 100644 --- a/templates/etc/trafficserver/records.config.etlua +++ b/templates/etc/trafficserver/records.config.etlua @@ -58,7 +58,7 @@ CONFIG proxy.config.output.logfile.rolling_enabled INT 0 # If outputting to stdout, the error.log file can't be symlinked to # stdout/stderr, so we still need to output to a log file, but enable # Trafficserver's log rotation so these don't grow very big. We'll then use - # rsyslog to tail these files and output to stdout for us. + # fluent-bit to tail these files and output to stdout for us. # # This workaround should no longer be needed once Trafficserver 10 is # released and things can be configured to output to stdout/stderr directly: diff --git a/test/proxy/logging/test_basics.rb b/test/proxy/logging/test_basics.rb index eb15924a5..fb6751a5c 100644 --- a/test/proxy/logging/test_basics.rb +++ b/test/proxy/logging/test_basics.rb @@ -38,18 +38,19 @@ def test_logs_expected_fields_for_non_chunked_non_gzip hit = result[:hit] expected_fields = [ + "@timestamp", "api_backend_id", - "api_backend_url_match_id", - "api_key", "api_backend_resolved_host", "api_backend_response_code_details", + "api_backend_url_match_id", + "api_key", "request_accept", "request_accept_encoding", - "request_at", "request_basic_auth_username", "request_connection", "request_content_type", "request_host", + "request_id", "request_ip", "request_method", "request_origin", diff --git a/test/support/api_umbrella_test_helpers/setup.rb b/test/support/api_umbrella_test_helpers/setup.rb index 73aa3981d..289e8d23f 100644 --- a/test/support/api_umbrella_test_helpers/setup.rb +++ b/test/support/api_umbrella_test_helpers/setup.rb @@ -99,7 +99,7 @@ def setup_server # suite will work with. We completely delete the indices here (rather # than relying on LogItem.clean_indices!), so that we're sure each # test gets fresh index template and mappings setup. - client.indices.delete :index => "api-umbrella*-2013-*,api-umbrella*-2014-*,api-umbrella*-2015-*" + client.indices.delete :index => "api-umbrella-test-*" self.setup_complete = true end @@ -368,7 +368,6 @@ def override_config_set(config, options = {}) # since otherwise it might leave symlinked files in place when # switching back to file output, which would lead to the output going # to unexpected places. - FileUtils.rm_f(File.join($config["log_dir"], "opensearch-aws-signing-proxy/access.log")) FileUtils.rm_f(File.join($config["log_dir"], "nginx-web-app/access.log")) FileUtils.rm_f(File.join($config["log_dir"], "nginx/access.log")) FileUtils.rm_f(File.join($config["log_dir"], "rsyslog/opensearch_error.log")) diff --git a/test/support/models/log_item.rb b/test/support/models/log_item.rb index 27b0fae07..4fd3d0b36 100644 --- a/test/support/models/log_item.rb +++ b/test/support/models/log_item.rb @@ -2,8 +2,6 @@ class LogItem include ActiveAttr::Model mattr_accessor :client - mattr_reader(:setup_indices) { Concurrent::Hash.new } - mattr_reader(:setup_indices_lock) { Monitor.new } attribute :_id attribute :api_backend_id @@ -154,39 +152,8 @@ def save index_time = Time.at(index_time / 1000.0).utc end - partition_date_format = case $config.fetch("opensearch").fetch("index_partition") - when "monthly" - "%Y-%m" - when "daily" - "%Y-%m-%d" - end - partition = index_time.utc.strftime(partition_date_format) - prefix = "#{$config.fetch("opensearch").fetch("index_name_prefix")}-logs" - index_name = "#{prefix}-v#{$config["opensearch"]["template_version"]}-#{partition}" - index_name_read = "#{prefix}-#{partition}" - index_name_write = "#{prefix}-write-#{partition}" - - unless self.setup_indices[index_name] - self.setup_indices_lock.synchronize do - unless self.setup_indices[index_name] - begin - self.client.indices.create(:index => index_name, :body => { - :aliases => { - index_name_read => {}, - index_name_write => {}, - }, - }) - rescue OpenSearch::Transport::Transport::Errors::BadRequest => e - unless e.message.include?("index_already_exists_exception") - raise e - end - end - - self.setup_indices[index_name] = true - end - end - end + index_name = "#{prefix}-v#{$config["opensearch"]["template_version"]}-all" self.client.index({ :index => index_name,