From e3e9be9c161da914fc16ede60ee41ca5314d3611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 18 Jan 2024 14:23:36 -0300 Subject: [PATCH 1/2] feat: add loki to docker-compose (#639) --- .gitignore | 9 ++- config/config.exs | 12 +++ metrics/docker-compose.yml | 74 +++++++++++++++++-- .../provisioning/datasources/loki_ds.yml | 6 ++ metrics/loki/loki.yml | 49 ++++++++++++ metrics/promtail/promtail.yml | 18 +++++ 6 files changed, 158 insertions(+), 10 deletions(-) create mode 100644 metrics/grafana/provisioning/datasources/loki_ds.yml create mode 100644 metrics/loki/loki.yml create mode 100644 metrics/promtail/promtail.yml diff --git a/.gitignore b/.gitignore index 7e82c30ad..bb7281b4d 100644 --- a/.gitignore +++ b/.gitignore @@ -44,19 +44,20 @@ priv .vscode/ # spec-test vectors -test/spec/vectors +/test/spec/vectors -native/libp2p_port/libp2p_port +/native/libp2p_port/libp2p_port # Proto generated code. *.pb.ex *.pb.go # local db. -level_db +/level_db +/logs # Generated tests -test/generated +/test/generated # profiling artifacts callgrind.out.* diff --git a/config/config.exs b/config/config.exs index 9a9c9a280..4f5572d7b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -4,6 +4,18 @@ import Config # Configure logging config :logger, level: :info, truncate: :infinity +## TODO: we might want to enable this with a CLI flag +## Uncomment to log to a file +# config :logger, :default_handler, +# config: [ +# file: ~c"logs/system.log", +# filesync_repeat_interval: 5000, +# file_check: 5000, +# max_no_bytes: 10_000_000, +# max_no_files: 5, +# compress_on_rotate: true +# ] + # Configures the phoenix endpoint config :lambda_ethereum_consensus, BeaconApi.Endpoint, http: [port: 4000], diff --git a/metrics/docker-compose.yml b/metrics/docker-compose.yml index 7519f9e28..a0fd06bdf 100644 --- a/metrics/docker-compose.yml +++ b/metrics/docker-compose.yml @@ -1,4 +1,4 @@ -version: '0.1' +version: '3' name: 'lambda-ethereum-consensus-grafana' services: @@ -6,14 +6,14 @@ services: image: prom/prometheus container_name: prometheus hostname: prometheus - ports: - - "9090:9090" volumes: + # prometheus configuration - ./prometheus:/etc/prometheus + # prometheus data - prometheus-data:/prometheus command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml networks: - open: + grafana-prometheus: aliases: - prometheus extra_hosts: @@ -25,17 +25,79 @@ services: ports: - "3000:3000" volumes: + # grafana configuration - ./grafana/provisioning:/etc/grafana/provisioning + # grafana data - grafana-data:/var/lib/grafana + environment: + # WARNING: use this for same-machine access ONLY + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_DISABLE_LOGIN_FORM: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" networks: - open: + grafana-prometheus: + aliases: + - grafana + grafana-loki: aliases: - grafana + # Since the Loki containers are running as user 10001 and the mounted data volume is owned by root, + # Loki would not have permissions to create the directories. + # Therefore the init container changes permissions of the mounted directory. + loki-init: + image: &lokiImage grafana/loki:2.9.2 + user: root + entrypoint: + - "chown" + - "10001:10001" + - "/tmp/loki" + volumes: + - loki-data:/tmp/loki + + loki: + image: *lokiImage + container_name: loki + volumes: + # loki configuration + - ./loki:/etc/loki + # loki data + - loki-data:/tmp/loki + command: --config.file=/etc/loki/loki.yml + networks: + grafana-loki: + aliases: + - loki + loki-promtail: + aliases: + - loki + + promtail: + image: grafana/promtail + container_name: promtail + volumes: + # promtail configuration + - ./promtail:/etc/promtail + # logs to scrape + - ../logs:/var/log/consensus + # promtail data + - promtail-data:/tmp/promtail + command: --config.file=/etc/promtail/promtail.yml + networks: + loki-promtail: + aliases: + - promtail + networks: - open: + grafana-prometheus: + driver: bridge + grafana-loki: + driver: bridge + loki-promtail: driver: bridge volumes: prometheus-data: grafana-data: + loki-data: + promtail-data: diff --git a/metrics/grafana/provisioning/datasources/loki_ds.yml b/metrics/grafana/provisioning/datasources/loki_ds.yml new file mode 100644 index 000000000..6fcfd57c7 --- /dev/null +++ b/metrics/grafana/provisioning/datasources/loki_ds.yml @@ -0,0 +1,6 @@ +datasources: + - name: Loki + access: proxy + type: loki + url: http://loki:3100 + isDefault: false diff --git a/metrics/loki/loki.yml b/metrics/loki/loki.yml new file mode 100644 index 000000000..0abb31135 --- /dev/null +++ b/metrics/loki/loki.yml @@ -0,0 +1,49 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +#analytics: +# reporting_enabled: false diff --git a/metrics/promtail/promtail.yml b/metrics/promtail/promtail.yml new file mode 100644 index 000000000..667d4d907 --- /dev/null +++ b/metrics/promtail/promtail.yml @@ -0,0 +1,18 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/promtail/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: system + static_configs: + - targets: + - localhost + labels: + job: lambda_ethereum_consensus + __path__: /var/log/consensus/*log From 82f2e007939c93f9498570b33a72f4e3b363036c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 18 Jan 2024 14:38:11 -0300 Subject: [PATCH 2/2] feat: label each log message and show in grafana (#647) --- config/config.exs | 17 +- .../grafana/provisioning/dashboards/home.json | 510 ++++++++++-------- metrics/promtail/promtail.yml | 17 + mix.exs | 3 +- mix.lock | 1 + 5 files changed, 310 insertions(+), 238 deletions(-) diff --git a/config/config.exs b/config/config.exs index 4f5572d7b..69a2e77f1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -4,8 +4,8 @@ import Config # Configure logging config :logger, level: :info, truncate: :infinity -## TODO: we might want to enable this with a CLI flag -## Uncomment to log to a file +# # Uncomment to log to a file +# # TODO: we might want to enable this with a CLI flag # config :logger, :default_handler, # config: [ # file: ~c"logs/system.log", @@ -16,6 +16,19 @@ config :logger, level: :info, truncate: :infinity # compress_on_rotate: true # ] +# # NOTE: We want to log UTC timestamps, for convenience +# config :logger, utc_log: true + +# config :logger, :default_formatter, +# format: {LogfmtEx, :format}, +# colors: [enabled: false], +# metadata: [:mfa] + +# config :logfmt_ex, :opts, +# message_key: "msg", +# timestamp_key: "ts", +# timestamp_format: :iso8601 + # Configures the phoenix endpoint config :lambda_ethereum_consensus, BeaconApi.Endpoint, http: [port: 4000], diff --git a/metrics/grafana/provisioning/dashboards/home.json b/metrics/grafana/provisioning/dashboards/home.json index 6dd20e3a5..26ceab217 100644 --- a/metrics/grafana/provisioning/dashboards/home.json +++ b/metrics/grafana/provisioning/dashboards/home.json @@ -138,62 +138,8 @@ }, { "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] + "type": "loki", + "uid": "P8E80F9AEF21F6940" }, "gridPos": { "h": 6, @@ -201,56 +147,32 @@ "x": 12, "y": 0 }, - "id": 4, + "id": 23, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false }, "targets": [ { "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(network_request_count{result=\"success\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "success", - "range": true, - "refId": "Success" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(network_request_count{result=~\"error|retry\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "error", - "range": true, - "refId": "Error", - "useBackend": false + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "builder", + "expr": "{job=\"lambda_ethereum_consensus\"} |= ``", + "maxLines": 100, + "queryType": "range", + "refId": "Raw logs" } ], - "title": "P2P Requests", - "type": "timeseries" + "title": "Raw logs", + "type": "logs" }, { "datasource": { @@ -280,6 +202,7 @@ "y": 6 }, "id": 22, + "maxDataPoints": 50, "options": { "calculate": false, "calculation": { @@ -408,6 +331,146 @@ "x": 12, "y": 6 }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (direction) (rate(port_message_count{direction=\"->elixir\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "incoming", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (direction) (rate(port_message_count{direction=\"elixir->\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "outgoing", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by () (rate(port_message_count{}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "total", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(port_message_count{}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{function}}", + "range": true, + "refId": "D" + } + ], + "title": "Libp2pPort Messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, "id": 3, "options": { "legend": { @@ -456,6 +519,97 @@ "title": "Peer additions", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 14, + "maxDataPoints": 25, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "show": true, + "showLegend": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "show": true, + "sort": "none", + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "network_pubsub_topic_active_active", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "PubSub topics joined", + "type": "heatmap" + }, { "datasource": { "type": "prometheus", @@ -522,7 +676,7 @@ "h": 6, "w": 24, "x": 0, - "y": 12 + "y": 18 }, "id": 2, "options": { @@ -657,7 +811,7 @@ "h": 6, "w": 5, "x": 0, - "y": 18 + "y": 24 }, "id": 7, "options": { @@ -717,7 +871,7 @@ "h": 6, "w": 7, "x": 5, - "y": 18 + "y": 24 }, "id": 5, "options": { @@ -819,9 +973,9 @@ "h": 6, "w": 12, "x": 12, - "y": 18 + "y": 24 }, - "id": 12, + "id": 4, "options": { "legend": { "calcs": [], @@ -835,65 +989,41 @@ } }, "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (direction) (rate(port_message_count{direction=\"->elixir\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "incoming", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by (direction) (rate(port_message_count{direction=\"elixir->\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "outgoing", - "range": true, - "refId": "B" - }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by () (rate(port_message_count{}[$__rate_interval]))", + "exemplar": true, + "expr": "rate(network_request_count{result=\"success\"}[$__rate_interval])", "hide": false, - "instant": false, - "legendFormat": "total", + "interval": "", + "legendFormat": "success", "range": true, - "refId": "C" + "refId": "Success" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "disableTextWrap": false, "editorMode": "code", - "expr": "rate(port_message_count{}[$__rate_interval])", + "exemplar": false, + "expr": "sum(rate(network_request_count{result=~\"error|retry\"}[$__rate_interval]))", + "fullMetaSearch": false, "hide": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "{{function}}", + "interval": "", + "legendFormat": "error", "range": true, - "refId": "D" + "refId": "Error", + "useBackend": false } ], - "title": "Libp2pPort Messages", + "title": "P2P Requests", "type": "timeseries" }, { @@ -959,7 +1089,7 @@ "h": 6, "w": 24, "x": 0, - "y": 24 + "y": 30 }, "id": 13, "options": { @@ -997,96 +1127,6 @@ "title": "Peers (Gossip)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 14, - "options": { - "calculate": false, - "cellGap": 1, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Oranges", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "show": true, - "showLegend": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "show": true, - "sort": "none", - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false - } - }, - "pluginVersion": "10.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "network_pubsub_topic_active_active", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "{{topic}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "PubSub topics joined", - "type": "heatmap" - }, { "datasource": { "type": "prometheus", @@ -1942,6 +1982,6 @@ "timezone": "", "title": "Node", "uid": "90EXFQnIk", - "version": 4, + "version": 2, "weekStart": "" } diff --git a/metrics/promtail/promtail.yml b/metrics/promtail/promtail.yml index 667d4d907..99633e926 100644 --- a/metrics/promtail/promtail.yml +++ b/metrics/promtail/promtail.yml @@ -16,3 +16,20 @@ scrape_configs: labels: job: lambda_ethereum_consensus __path__: /var/log/consensus/*log + pipeline_stages: + - logfmt: + mapping: + ts: + level: + msg: + mfa: + process: registered_name + - timestamp: + format: "2006-01-02T15:04:05.000" + source: ts + - labels: + level: + mfa: + process: + - output: + source: msg diff --git a/mix.exs b/mix.exs index 65060d9ad..755b1f0d7 100644 --- a/mix.exs +++ b/mix.exs @@ -47,12 +47,13 @@ defmodule LambdaEthereumConsensus.MixProject do {:telemetry_poller, "~> 1.0"}, {:telemetry_metrics, "~> 0.6"}, {:telemetry_metrics_prometheus, "~> 1.1.0"}, + {:aja, "~> 0.6"}, + {:logfmt_ex, "~> 0.4.2"}, {:ex2ms, "~> 1.6", runtime: false}, {:eflambe, "~> 0.3.1"}, {:patch, "~> 0.12.0", only: [:test]}, {:stream_data, "~> 0.5", only: [:test]}, {:benchee, "~> 1.2", only: [:dev]}, - {:aja, "~> 0.6"}, {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false} ] diff --git a/mix.lock b/mix.lock index 8ba40ff29..26f249815 100644 --- a/mix.lock +++ b/mix.lock @@ -27,6 +27,7 @@ "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, "joken": {:hex, :joken, "2.6.0", "b9dd9b6d52e3e6fcb6c65e151ad38bf4bc286382b5b6f97079c47ade6b1bcc6a", [:mix], [{:jose, "~> 1.11.5", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5a95b05a71cd0b54abd35378aeb1d487a23a52c324fa7efdffc512b655b5aaa7"}, "jose": {:hex, :jose, "1.11.6", "613fda82552128aa6fb804682e3a616f4bc15565a048dabd05b1ebd5827ed965", [:mix, :rebar3], [], "hexpm", "6275cb75504f9c1e60eeacb771adfeee4905a9e182103aa59b53fed651ff9738"}, + "logfmt_ex": {:hex, :logfmt_ex, "0.4.2", "e337b6072bd21ad61d8bbe38d9c591b5a8e4869ceba4967699d027baedf2eec8", [:mix], [], "hexpm", "7fad3704383d4595adf0da873e72c8b393120e67b1257f9102da881fde9d4249"}, "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},