WIP: New OpenSearch template and logging approach with fluent-bit.

- New OpenSearch template based on using data streams, which eliminates
  the need for manually managing time-partitioned indices and aliases.
- Begin to remove code supporting older elasticsearch templates, since
  we'll just support the latest version now.
- For logging to OpenSearch switch from rsyslog to fluent-bit, since
  fluent-bit can log directly to AWS OpenSearch with the AWS
  authentication/signing (so we don't need a custom proxy layer to do
  the AWS signing).

Various things are still broken, but the basics are sort of there.

Dockerfile

@@ -31,8 +31,8 @@ RUN make deps:openresty && make clean:dev
 COPY tasks/deps/trafficserver /app/tasks/deps/
 RUN make deps:trafficserver && make clean:dev
 
-COPY tasks/deps/rsyslog /app/tasks/deps/
-RUN make deps:rsyslog && make clean:dev
+COPY tasks/deps/fluent-bit /app/tasks/deps/
+RUN make deps:fluent-bit && make clean:dev
 
 COPY src/api-umbrella-git-1.rockspec src/luarocks.lock /app/src/
 COPY tasks/deps/luarocks /app/tasks/deps/

Taskfile.yml

@@ -60,6 +60,15 @@ tasks:
     generates:
       - ./build/work/stamp/deps/envoy-control-plane
 
+  deps:fluent-bit:
+    cmds:
+      - ./tasks/deps/fluent-bit
+    sources:
+      - ./tasks/deps/fluent-bit
+      - ./tasks/helpers.sh
+    generates:
+      - ./build/work/stamp/deps/fluent-bit
+
   deps:luarocks:
     deps:
       - deps:openresty
@@ -90,15 +99,6 @@ tasks:
     generates:
       - ./build/work/stamp/deps/perp
 
-  deps:rsyslog:
-    cmds:
-      - ./tasks/deps/rsyslog
-    sources:
-      - ./tasks/deps/rsyslog
-      - ./tasks/helpers.sh
-    generates:
-      - ./build/work/stamp/deps/rsyslog
-
   deps:trafficserver:
     deps:
       - deps:openresty
@@ -116,10 +116,10 @@ tasks:
       - task: deps:cue
       - task: deps:envoy
       - task: deps:envoy-control-plane
+      - task: deps:fluent-bit
       - task: deps:luarocks
       - task: deps:openresty
       - task: deps:perp
-      - task: deps:rsyslog
       - task: deps:trafficserver
 
   build-deps:crane:

build/package_dependencies.sh

@@ -34,9 +34,6 @@ if [[ "$ID_NORMALIZED" == "rhel" ]]; then
     # TrafficServer
     libxml2
 
-    # rsyslog omelasticsearch
-    libcurl
-
     # init.d script helpers
     initscripts
 
@@ -190,13 +187,6 @@ elif [[ "$ID_NORMALIZED" == "debian" ]]; then
     libunwind8
     libxml2
 
-    # rsyslog
-    libestr0
-    libfastjson4
-
-    # rsyslog omelasticsearch
-    "libcurl$libcurl_version"
-
     # init.d script helpers
     sysvinit-utils
     lsb-base
@@ -274,17 +264,18 @@ elif [[ "$ID_NORMALIZED" == "debian" ]]; then
     # lua-psl
     libpsl-dev
 
-    # rsyslog
-    libestr-dev
-    libfastjson-dev
-
     # ngx_http_geoip2_module
     libmaxminddb-dev
 
     # TrafficServer
     libhwloc-dev
     libjemalloc-dev
     libunwind-dev
+
+    # Fluent Bit
+    bison
+    cmake
+    flex
   )
   test_runtime_dependencies=(
     unbound

config/opensearch_templates_v2_es7.json.etlua → config/opensearch_templates_v3.json.etlua

@@ -1,12 +1,13 @@
-[
-  {
-    "id": "<%- config["opensearch"]["index_name_prefix"] %>-log-v2-template",
+{
+  "<%- config["opensearch"]["index_name_prefix"] %>-log-v3-template": {
+    "index_patterns": [
+      "<%- config["opensearch"]["index_name_prefix"] %>-logs-v3-*"
+    ],
+    "data_stream": {},
     "template": {
-      "template": "<%- config["opensearch"]["index_name_prefix"] %>-logs-v2-*",
       "settings": {
         "index": {
-          "number_of_shards": 1,
-          "codec": "best_compression"
+          "refresh_interval": "10s"
         },
         "analysis": {
           "normalizer": {
@@ -28,6 +29,9 @@
       "mappings": {
         "dynamic": "strict",
         "properties": {
+          "@timestamp": {
+            "type": "date"
+          },
           "api_backend_id": {
             "type": "keyword",
             "normalizer": "lowercase_normalizer"
@@ -66,9 +70,6 @@
             "type": "keyword",
             "normalizer": "lowercase_normalizer"
           },
-          "request_at": {
-            "type": "date"
-          },
           "request_basic_auth_username": {
             "type": "keyword",
             "normalizer": "lowercase_normalizer"
@@ -85,12 +86,16 @@
             "type": "keyword",
             "normalizer": "lowercase_normalizer"
           },
+          "request_id": {
+            "type": "keyword"
+          },
           "request_ip": {
             "type": "keyword",
             "normalizer": "lowercase_normalizer"
           },
           "request_ip_city": {
-            "type": "keyword"
+            "type": "keyword",
+            "normalizer": "lowercase_normalizer"
           },
           "request_ip_country": {
             "type": "keyword",
@@ -233,4 +238,4 @@
       }
     }
   }
-]
+}

config/schema.cue

@@ -418,7 +418,7 @@ import "path"
     match_x_forwarded_host?: bool
   }
 
-  rsyslog: {
+  fluent_bit: {
     host: string | *"127.0.0.1"
     port: uint16 | *14014
   }
@@ -457,18 +457,7 @@ import "path"
       "http://opensearch:9200",
     ]
     index_name_prefix: string | *"api-umbrella"
-    index_partition: string | *"daily"
-    api_version: uint | *7
-    template_version: uint | *2
-    aws_signing_proxy: {
-      host: string | *"127.0.0.1"
-      port: uint16 | *14017
-      workers: uint | "auto" | *1
-      worker_connections: uint | *8192
-      listen_so_keepalive: string | *"on"
-      listen_backlog?: uint
-      error_log_level: string | *"notice"
-    }
+    template_version: uint | *3
   }
 
   #analytics_output_name: "opensearch"

config/test.yml

@@ -91,7 +91,7 @@ router:
     # 1-2 seconds, but to make testing easier, force immediate refreshes in the
     # test environment.
     refresh_local_cache_interval: 0
-rsyslog:
+fluent_bit:
   port: 13014
 geoip:
   db_update_frequency: false
@@ -100,6 +100,8 @@ postgresql:
   password: dev_password
   migrations:
     password: dev_password
+opensearch:
+  index_name_prefix: "api-umbrella-test"
 unbound:
   port: 13100
   control_port: 13101

docker-compose.yml

@@ -33,7 +33,7 @@ services:
     ports:
       - "14011:5432"
   opensearch:
-    image: public.ecr.aws/opensearchproject/opensearch:2.0.1
+    image: public.ecr.aws/opensearchproject/opensearch:2.11.1
     environment:
       OPENSEARCH_JAVA_OPTS: "-Xms32m -Xmx256m"
       DISABLE_INSTALL_DEMO_CONFIG: "true"

src/api-umbrella/cli/reopen_logs.lua

@@ -38,14 +38,6 @@ local function reopen_nginx(perp_base)
   end
 end
 
-local function reopen_rsyslog(perp_base)
-  local _, err = shell_blocking_capture_combined({ "perpctl", "-b", perp_base, "hup", "rsyslog" })
-  if err then
-    print("Failed to reopen logs for rsyslog\n" .. err)
-    os.exit(1)
-  end
-end
-
 return function()
   local running, pid = status()
   if not running then
@@ -59,6 +51,5 @@ return function()
 
   if config["_service_router_enabled?"] then
     reopen_nginx(perp_base)
-    reopen_rsyslog(perp_base)
   end
 end

src/api-umbrella/cli/write_config_files.lua

@@ -241,16 +241,13 @@ local function activate_services()
   available_services = invert_table(available_services)
 
   local active_services = {}
-  if config["_service_opensearch_aws_signing_proxy_enabled?"] then
-    active_services["opensearch-aws-signing-proxy"] = 1
-  end
   if config["_service_router_enabled?"] then
     if config["geoip"]["_auto_updater_enabled"] then
       active_services["geoip-auto-updater"] = 1
     end
     active_services["envoy-control-plane"] = 1
+    active_services["fluent-bit"] = 1
     active_services["nginx"] = 1
-    active_services["rsyslog"] = 1
     active_services["trafficserver"] = 1
   end
   if config["_service_egress_enabled?"] then

src/api-umbrella/http-api/health.lua

@@ -1,10 +1,10 @@
 local active_config_exists = require("api-umbrella.proxy.stores.active_config_store").exists
 local config = require("api-umbrella.utils.load_config")()
 local http = require "resty.http"
-local icu_date = require "icu-date-ffi"
 local json_encode = require "api-umbrella.utils.json_encode"
 local opensearch = require "api-umbrella.utils.opensearch"
 
+local jobs_dict = ngx.shared.jobs
 local opensearch_query = opensearch.query
 local ngx_var = ngx.var
 
@@ -45,18 +45,9 @@ local function status_response(quick)
     response["details"]["analytics_db"] = opensearch_health["status"]
 
     -- Check to see if the OpenSearch index aliases have been setup.
-    local date = icu_date.new({ zone_id = "UTC" })
-    local today = date:format(opensearch.partition_date_format)
-    local alias = config["opensearch"]["index_name_prefix"] .. "-logs-" .. today
-    local index = config["opensearch"]["index_name_prefix"] .. "-logs-v" .. config["opensearch"]["template_version"] .. "-" .. today
-    res, err = opensearch_query("/" .. index .. "/_alias/" .. alias)
-    if err then
-      ngx.log(ngx.ERR, "failed to fetch opensearch alias details: ", err)
-    elseif res.body_json then
-      local opensearch_alias = res.body_json
-      if not opensearch_alias["error"] then
-        response["details"]["analytics_db_setup"] = "green"
-      end
+    local created = jobs_dict:get("opensearch_templates_created")
+    if created then
+      response["details"]["analytics_db_setup"] = "green"
     end
   end
 

src/api-umbrella/proxy/hooks/init_preload_modules.lua

@@ -70,7 +70,6 @@ require "api-umbrella.utils.worker_group"
 require "api-umbrella.utils.xpcall_error_handler"
 require "cjson"
 require "etlua"
-require "icu-date-ffi"
 require "libcidr-ffi"
 require "lustache"
 require "ngx.re"