From fb031fee2c4c0f282dddef22e1b819c8a1b44357 Mon Sep 17 00:00:00 2001 From: TzeYiing Date: Wed, 20 Nov 2024 03:27:03 +0800 Subject: [PATCH] feat: refactor part 1 --- assets/css/app.scss | 3 ++ .../ecto/bigquery/ecto_bq_query_api_udf.ex | 14 ------ lib/logflare/logs/logs_search.ex | 4 +- .../logs/lql/bigquery/lql_ecto_helpers.ex | 2 +- .../logs/search/logs_search_operations.ex | 27 ++++++---- .../logs/search/logs_search_queries.ex | 49 +++++++++++++------ lib/logflare/source/bigquery/bigquery_udf.ex | 2 - .../source/bigquery/udf/search_fns.ex | 30 ------------ .../search_live/templates/logs_list.html.heex | 4 +- 9 files changed, 60 insertions(+), 75 deletions(-) diff --git a/assets/css/app.scss b/assets/css/app.scss index 9ad737513..5acfb5b21 100644 --- a/assets/css/app.scss +++ b/assets/css/app.scss @@ -274,6 +274,9 @@ a:focus, button.btn-link:focus { mark { background-color: #5eeb8f; + & .log-error { + background-color: #f14216; + } } .example-log-code { diff --git a/lib/logflare/ecto/bigquery/ecto_bq_query_api_udf.ex b/lib/logflare/ecto/bigquery/ecto_bq_query_api_udf.ex index 3136749d3..583abf62a 100644 --- a/lib/logflare/ecto/bigquery/ecto_bq_query_api_udf.ex +++ b/lib/logflare/ecto/bigquery/ecto_bq_query_api_udf.ex @@ -4,20 +4,6 @@ defmodule Logflare.Ecto.BQQueryAPI.UDF do """ import Logflare.Ecto.BQQueryAPI, only: [to_bq_interval_token: 1] - defmacro lf_timestamp_trunc(timestamp, interval) do - fragment_string = udf_function("LF_TIMESTAMP_TRUNC(?, ?)") - - interval = quoted_interval(interval) - - quote do - fragment( - unquote(fragment_string), - unquote(timestamp), - ^unquote(quote(do: to_bq_interval_token(unquote(interval)))) - ) - end - end - defmacro lf_timestamp_sub(date, count, interval) do fragment_string = udf_function("LF_TIMESTAMP_SUB(?, ?, ?)") diff --git a/lib/logflare/logs/logs_search.ex b/lib/logflare/logs/logs_search.ex index fe417970c..2cd0af2ec 100644 --- a/lib/logflare/logs/logs_search.ex +++ b/lib/logflare/logs/logs_search.ex @@ -79,7 +79,9 @@ defmodule Logflare.Logs.Search do def search_events(%SO{} = so) do so = %{so | type: :events} |> put_time_stats() - with %{error: nil} = so <- apply_query_defaults(so), + with + %{error: nil} = so <- put_chart_data_shape_id(so), + %{error: nil} = so <- apply_query_defaults(so), %{error: nil} = so <- apply_halt_conditions(so), %{error: nil} = so <- apply_local_timestamp_correction(so), %{error: nil} = so <- apply_timestamp_filter_rules(so), diff --git a/lib/logflare/logs/lql/bigquery/lql_ecto_helpers.ex b/lib/logflare/logs/lql/bigquery/lql_ecto_helpers.ex index 16754b90a..585b15600 100644 --- a/lib/logflare/logs/lql/bigquery/lql_ecto_helpers.ex +++ b/lib/logflare/logs/lql/bigquery/lql_ecto_helpers.ex @@ -59,7 +59,7 @@ defmodule Logflare.Lql.EctoHelpers do ) end end) - end + end end defp split_by_dots(str) do diff --git a/lib/logflare/logs/search/logs_search_operations.ex b/lib/logflare/logs/search/logs_search_operations.ex index 1b0a33caf..52bfb6927 100644 --- a/lib/logflare/logs/search/logs_search_operations.ex +++ b/lib/logflare/logs/search/logs_search_operations.ex @@ -65,13 +65,21 @@ defmodule Logflare.Logs.SearchOperations do end @spec apply_query_defaults(SO.t()) :: SO.t() - def apply_query_defaults(%SO{type: :events} = so) do + def apply_query_defaults(%SO{chart_data_shape_id: chart_data_shape_id} = so) do + dbg(so) query = from(so.source.bq_table_id) - |> select_default_fields(:events) - |> order_by(desc: :timestamp) + |> select( [t], [t.timestamp, t.id, t.event_message]) + |> case do + q when chart_data_shape_id == :elixir_logger_levels -> + q + |> Lql.EctoHelpers.unnest_and_join_nested_columns(:inner, "metadata") + |> select_merge([..., m], %{level: m.level}) + q -> q + end + |> order_by([t], desc: t.timestamp) |> limit(@default_limit) - + |> dbg() %{so | query: query} end @@ -132,21 +140,20 @@ defmodule Logflare.Logs.SearchOperations do SourceSchemas.get_source_schema_by(source_id: so.source.id) |> Map.get(:schema_flat_map) - [%{path: path}] = so.chart_rules - path_is_timestamp? = path == "timestamp" + dbg({so.type, Map.has_key?(flat_type_map, "metadata.level")}) chart_data_shape_id = cond do - path_is_timestamp? and Map.has_key?(flat_type_map, "metadata.status_code") -> + Map.has_key?(flat_type_map, "metadata.status_code") -> :netlify_status_codes - path_is_timestamp? and Map.has_key?(flat_type_map, "metadata.response.status_code") -> + Map.has_key?(flat_type_map, "metadata.response.status_code") -> :cloudflare_status_codes - path_is_timestamp? and Map.has_key?(flat_type_map, "metadata.proxy.statusCode") -> + Map.has_key?(flat_type_map, "metadata.proxy.statusCode") -> :vercel_status_codes - path_is_timestamp? and Map.has_key?(flat_type_map, "metadata.level") -> + Map.has_key?(flat_type_map, "metadata.level") -> :elixir_logger_levels true -> diff --git a/lib/logflare/logs/search/logs_search_queries.ex b/lib/logflare/logs/search/logs_search_queries.ex index 31165f528..77c33fec6 100644 --- a/lib/logflare/logs/search/logs_search_queries.ex +++ b/lib/logflare/logs/search/logs_search_queries.ex @@ -12,9 +12,23 @@ defmodule Logflare.Logs.SearchQueries do |> select([t], %{ timestamp: fragment( - "(`$$__DEFAULT_DATASET__$$`.LF_TIMESTAMP_TRUNC(?, ?)) as timestamp", + + """ + (case + when ? = 'DAY' then TIMESTAMP_TRUNC(?, DAY) + when ? = 'HOUR' then TIMESTAMP_TRUNC(?, HOUR) + when ? = 'MINUTE' then TIMESTAMP_TRUNC(?, MINUTE) + when ? = 'SECOND' then TIMESTAMP_TRUNC(?, SECOND) + end) as timestamp + """, + ^String.upcase("#{chart_period}"), + t.timestamp, + ^String.upcase("#{chart_period}"), + t.timestamp, + ^String.upcase("#{chart_period}"), t.timestamp, - ^String.upcase("#{chart_period}") + ^String.upcase("#{chart_period}"), + t.timestamp ) }) end @@ -24,9 +38,25 @@ defmodule Logflare.Logs.SearchQueries do |> select([t], %{ timestamp: fragment( - "(`$$__DEFAULT_DATASET__$$`.LF_TIMESTAMP_TRUNC_WITH_TIMEZONE(?, ?, ?)) as timestamp", + """ + (case + when ? = 'DAY' then TIMESTAMP_TRUNC(?, DAY , ?) + when ? = 'HOUR' then TIMESTAMP_TRUNC(?, HOUR , ?) + when ? = 'MINUTE' then TIMESTAMP_TRUNC(?, MINUTE , ?) + when ? = 'SECOND' then TIMESTAMP_TRUNC(?, SECOND , ?) + end) as timestamp + """, + ^String.upcase("#{chart_period}"), + t.timestamp, + ^timezone, + ^String.upcase("#{chart_period}"), t.timestamp, + ^timezone, ^String.upcase("#{chart_period}"), + t.timestamp, + ^timezone, + ^String.upcase("#{chart_period}"), + t.timestamp, ^timezone ) }) @@ -93,10 +123,6 @@ defmodule Logflare.Logs.SearchQueries do limit(query, ^number) end - def timestamp_truncator(period) when period in @chart_periods do - dynamic([t], lf_timestamp_trunc(t.timestamp, ^period)) - end - def where_streaming_buffer(query) do where(query, in_streaming_buffer()) end @@ -113,12 +139,6 @@ defmodule Logflare.Logs.SearchQueries do ) end - def select_timestamp_trunc(query, chart_period) do - select(query, [t, ...], %{ - timestamp: lf_timestamp_trunc(t.timestamp, ^chart_period) - }) - end - def select_merge_total(query) do select_merge(query, [t, ...], %{ total: fragment("COUNT(?) as total", t.timestamp) @@ -269,9 +289,6 @@ defmodule Logflare.Logs.SearchQueries do |> select([t], fragment("*")) end - def select_default_fields(query, :events) do - select(query, [:timestamp, :id, :event_message]) - end def source_table_streaming_buffer(bq_table_id) when is_binary(bq_table_id) do from(bq_table_id) diff --git a/lib/logflare/source/bigquery/bigquery_udf.ex b/lib/logflare/source/bigquery/bigquery_udf.ex index 328a3a137..90197a689 100644 --- a/lib/logflare/source/bigquery/bigquery_udf.ex +++ b/lib/logflare/source/bigquery/bigquery_udf.ex @@ -52,8 +52,6 @@ defmodule Logflare.User.BigQueryUDFs do when is_binary(bdid) and bdid != "" and is_binary(bqid) and bqid != "" do " #{SFns.lf_timestamp_sub(bqid, bdid)} - #{SFns.lf_timestamp_trunc(bqid, bdid)} - #{SFns.lf_timestamp_trunc_with_timezone(bqid, bdid)} #{SFns.lf_generate_timestamp_array(bqid, bdid)} " end diff --git a/lib/logflare/source/bigquery/udf/search_fns.ex b/lib/logflare/source/bigquery/udf/search_fns.ex index 5a027a02c..67a9e39f5 100644 --- a/lib/logflare/source/bigquery/udf/search_fns.ex +++ b/lib/logflare/source/bigquery/udf/search_fns.ex @@ -19,36 +19,6 @@ defmodule Logflare.User.BigQueryUDFs.SearchFns do " end - def lf_timestamp_trunc(bq_project_id, bq_dataset_id) do - " - CREATE OR REPLACE FUNCTION - `#{bq_project_id}`.#{bq_dataset_id}.LF_TIMESTAMP_TRUNC(_timestamp TIMESTAMP, _date_part STRING) AS ( - CASE _date_part - WHEN 'MICROSECOND' THEN TIMESTAMP_TRUNC(_timestamp, MICROSECOND) - WHEN 'MILLISECOND' THEN TIMESTAMP_TRUNC(_timestamp, MILLISECOND) - WHEN 'SECOND' THEN TIMESTAMP_TRUNC(_timestamp, SECOND) - WHEN 'MINUTE' THEN TIMESTAMP_TRUNC(_timestamp, MINUTE) - WHEN 'HOUR' THEN TIMESTAMP_TRUNC(_timestamp, HOUR) - WHEN 'DAY' THEN TIMESTAMP_TRUNC(_timestamp, DAY) - END); - " - end - - def lf_timestamp_trunc_with_timezone(bq_project_id, bq_dataset_id) do - " - CREATE OR REPLACE FUNCTION - `#{bq_project_id}`.#{bq_dataset_id}.LF_TIMESTAMP_TRUNC_WITH_TIMEZONE(_timestamp TIMESTAMP, _date_part STRING, timezone STRING) AS ( - CASE _date_part - WHEN 'MICROSECOND' THEN TIMESTAMP_TRUNC(_timestamp, MICROSECOND, timezone) - WHEN 'MILLISECOND' THEN TIMESTAMP_TRUNC(_timestamp, MILLISECOND, timezone) - WHEN 'SECOND' THEN TIMESTAMP_TRUNC(_timestamp, SECOND, timezone) - WHEN 'MINUTE' THEN TIMESTAMP_TRUNC(_timestamp, MINUTE, timezone) - WHEN 'HOUR' THEN TIMESTAMP_TRUNC(_timestamp, HOUR, timezone) - WHEN 'DAY' THEN TIMESTAMP_TRUNC(_timestamp, DAY, timezone) - END); - " - end - def lf_generate_timestamp_array(bq_project_id, bq_dataset_id) do " CREATE OR REPLACE FUNCTION diff --git a/lib/logflare_web/live/search_live/templates/logs_list.html.heex b/lib/logflare_web/live/search_live/templates/logs_list.html.heex index a33f1b61f..1feb78110 100644 --- a/lib/logflare_web/live/search_live/templates/logs_list.html.heex +++ b/lib/logflare_web/live/search_live/templates/logs_list.html.heex @@ -9,6 +9,8 @@ <% %{"timestamp" => timestamp, "event_message" => message} = log.body +#dbg(log.body) + log_level = get_in(log.body,["metadata", "level"]) || get_in(log.body,["level"]) || "info" formatted_timestamp = if @use_local_time do format_timestamp(timestamp, @user_local_timezone) @@ -16,7 +18,7 @@ format_timestamp(timestamp) <> " UTC" end %>
  • - <%= formatted_timestamp %> <%= message %> + <%= formatted_timestamp %> <%= message %> <%= live_modal_show_link(component: LogflareWeb.Search.LogEventViewerComponent, modal_id: :log_event_viewer, title: "Log Event", phx_value_log_event_id: log.id, phx_value_log_event_timestamp: log.body["timestamp"]) do %> view