From 6db0c1c0f6ce6a94f5da8f5095d4bd4fb0e582de Mon Sep 17 00:00:00 2001 From: mborne Date: Tue, 1 Aug 2023 21:48:24 +0200 Subject: [PATCH] remove fluentd --- README.md | 1 - fluentd/.env | 1 - fluentd/Dockerfile | 12 ----- fluentd/README.md | 59 ----------------------- fluentd/conf/fluent.conf | 75 ------------------------------ fluentd/docker-compose.yml | 29 ------------ fluentd/helm/values-opensearch.yml | 19 -------- opensearch/docker-compose.yml | 6 --- 8 files changed, 202 deletions(-) delete mode 100644 fluentd/.env delete mode 100644 fluentd/Dockerfile delete mode 100644 fluentd/README.md delete mode 100644 fluentd/conf/fluent.conf delete mode 100644 fluentd/docker-compose.yml delete mode 100644 fluentd/helm/values-opensearch.yml diff --git a/README.md b/README.md index 7371488..068720b 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,6 @@ This is my playground to learn and illustrate how to deploy application with [do | [Kibana](kibana/README.md) | ELK - User Interface | ☑ | ☐ | | [OpenSearch](opensearch/README.md) | Forked from [Elasticsearch](https://www.elastic.co/fr/elasticsearch/) by AWS | ☑ | ☑ | | [Netdata](netdata/README.md) | A monitoring tool with an easy setup | ☑ | ☐ | -| [Fluend](fluentd/README.md) | An attempt to configure [Fluend](https://www.fluentd.org/) to collect containers logs | ☑ | ☐ | ### GeoSpatial services diff --git a/fluentd/.env b/fluentd/.env deleted file mode 100644 index 0376f3c..0000000 --- a/fluentd/.env +++ /dev/null @@ -1 +0,0 @@ -DEVBOX_PORT_PREFIX=127.0.0.1: diff --git a/fluentd/Dockerfile b/fluentd/Dockerfile deleted file mode 100644 index 1c7565a..0000000 --- a/fluentd/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM fluent/fluentd:v1.12.0-debian-1.0 -USER root - -RUN apt-get update && apt-get install -y libsystemd-dev make gcc g++ libc6-dev ruby-dev - -RUN ["gem", "install", "fluent-plugin-opensearch", "--no-document"] -RUN ["gem", "install", "fluent-plugin-systemd", "--no-document", "-v","1.0.2"] - -RUN mkdir -p /data && chmod 777 /data -VOLUME /data - -#USER fluent diff --git a/fluentd/README.md b/fluentd/README.md deleted file mode 100644 index d39c562..0000000 --- a/fluentd/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Fluentd - -**WORK IN PROGRESS** - -Container running [fluentd](https://www.fluentd.org/) to collect logs. - -## Pre-requisite - -* [OpenSearch](../opensearch/README.md) ( see [fluentd/conf/fluent.conf](fluentd/conf/fluent.conf) ) - -## Usage with docker - -* Read [conf/fluent.conf](conf/fluent.conf) - -* Start containers : - -```bash -docker compose up -d -``` - -* To test with `docker run` : - -```bash -docker run --rm --log-driver=fluentd --log-opt tag="docker.echo.{{.ID}}" ubuntu echo '{"message":"hello world!"}' -``` - -* To configure driver globally with `/etc/docker/daemon.json` : - -```json -{ - "log-driver": "fluentd", - "log-opts": { - "fluentd-address": "localhost:24224", - "tag": "docker.{{.Name}}.{{.ID}}" - } -} -``` - -## Usage with Kubernetes - -* Add repository : `helm repo add fluent https://fluent.github.io/helm-charts` -* Update helm repositories : `helm repo update` -* Create namespace : `kubectl create namespace fluent-system` -* Deploy fluent-bit : - -```bash -helm -n fluent-system install -i fluent-bit fluent/fluent-bit -# or -helm -n fluent-system install -f fluent/helm/values-opensearch.yml fluent-bit fluent/fluent-bit -``` - -Note : - -* see [fluent-bit/values.yaml](https://github.com/fluent/helm-charts/blob/main/charts/fluent-bit/values.yaml) - -## Resources - -* [docs.fluentd.org - container-deployment/docker-compose](https://docs.fluentd.org/container-deployment/docker-compose) -* [www.fluentd.org - Docker Logging](https://www.fluentd.org/guides/recipes/docker-logging) diff --git a/fluentd/conf/fluent.conf b/fluentd/conf/fluent.conf deleted file mode 100644 index ae6fa89..0000000 --- a/fluentd/conf/fluent.conf +++ /dev/null @@ -1,75 +0,0 @@ - - @type forward - port 24224 - bind 0.0.0.0 - - - - @type tail - path /var/log/auth.log - pos_file /data/auth.pos - tag authlog - - @type syslog - message_format rfc3164 - with_priority false - - - - - - @type tail - path /var/log/syslog.log - pos_file /data/syslog.pos - tag syslog - - @type syslog - message_format rfc3164 - with_priority false - - - -# -# @type systemd -# filters [{ "_SYSTEMD_UNIT": "docker.service" }] -# -# @type local -# persistent true -# path /data/systemd-docker.pos -# -# read_from_head true -# tag docker -# - - - - @type parser - format json - key_name log - reserve_data true - suppress_parse_error_log true - - - -# https://docs.fluentd.org/output/opensearch - - @type opensearch - host opensearch - port 9200 - logstash_format true - - chunk_limit_size 2MB - flush_mode interval - flush_interval 5.0 - flush_thread_count 1 - flush_thread_interval 5.0 - flush_thread_burst_interval 5.0 - - - -# -# log_level debug -# # log_level error -# - - diff --git a/fluentd/docker-compose.yml b/fluentd/docker-compose.yml deleted file mode 100644 index c94bbce..0000000 --- a/fluentd/docker-compose.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: '3.8' - -services: - - fluentd: - build: . - container_name: fluentd - volumes: - - ./conf:/fluentd/etc:ro - - /var/log:/var/log:ro - - fluent-data:/data - ports: - - "${DEVBOX_PORT_PREFIX}24224:24224" - - "${DEVBOX_PORT_PREFIX}24224:24224/udp" - userns_mode: 'host' - privileged: true - # ensure it can start with global fluentd logging - logging: - driver: syslog - restart: unless-stopped - -volumes: - fluent-data: - name: fluent-data - -networks: - default: - name: devbox - external: true diff --git a/fluentd/helm/values-opensearch.yml b/fluentd/helm/values-opensearch.yml deleted file mode 100644 index ffe9608..0000000 --- a/fluentd/helm/values-opensearch.yml +++ /dev/null @@ -1,19 +0,0 @@ -# https://github.com/fluent/helm-charts/blob/main/charts/fluent-bit/values.yaml - -config: - ## https://docs.fluentbit.io/manual/pipeline/outputs - outputs: | - [OUTPUT] - Name opensearch - Match kube.* - Host opensearch-cluster-master.opensearch.svc.cluster.local - Logstash_Format On - Retry_Limit False - - [OUTPUT] - Name opensearch - Match host.* - Host opensearch-cluster-master.opensearch.svc.cluster.local - Logstash_Format On - Logstash_Prefix node - Retry_Limit False diff --git a/opensearch/docker-compose.yml b/opensearch/docker-compose.yml index 42a4b3c..836bb76 100644 --- a/opensearch/docker-compose.yml +++ b/opensearch/docker-compose.yml @@ -26,9 +26,6 @@ services: - ${DEVBOX_PORT_PREFIX}9200:9200 # required for Performance Analyzer - ${DEVBOX_PORT_PREFIX}9600:9600 - # ensure it can start with global fluentd logging - logging: - driver: syslog labels: - "traefik.enable=true" - "traefik.http.routers.os.rule=Host(`os.${DEVBOX_HOSTNAME}`)" @@ -40,9 +37,6 @@ services: environment: - 'OPENSEARCH_HOSTS=["http://opensearch:9200"]' - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true" - # ensure it can start with global fluentd logging - logging: - driver: syslog labels: - "traefik.enable=true" - "traefik.http.routers.os-dashboard.rule=Host(`os-dashboard.${DEVBOX_HOSTNAME}`)"