Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release v1.3.27 #1660

Merged
merged 46 commits into from
Aug 2, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
389f0f3
chore: add failing test for supabase mode translation
Ziinc Jul 27, 2023
f665af7
feat: add initial implementation for supabase mode translation
Ziinc Jul 27, 2023
b0f1de7
feat: order by translation fixes
Ziinc Jul 27, 2023
2a57a2e
fix: improve parameter replacement to handle substrings better
Ziinc Jul 27, 2023
930d2a2
feat: add nested cte quotations conversion, literal value handling
Ziinc Jul 28, 2023
1859aaf
feat: cte cross join unnest removal
Ziinc Jul 28, 2023
24a8b3f
chore: refactored compound identifier handling
Ziinc Jul 28, 2023
29d86e1
chore: add failing test for order by compound identifier handling
Ziinc Jul 28, 2023
4a2aa50
feat: translation for compound identifiers in order by of a cte
Ziinc Jul 28, 2023
29fc788
chore: update test, formatting
Ziinc Jul 28, 2023
8bc22fe
feat: fix current_timestamp translation
Ziinc Jul 31, 2023
20c0eea
feat: fix parameter translation
Ziinc Jul 31, 2023
34ccd35
fix: translation for queries without from clause
Ziinc Jul 31, 2023
b4b5c9b
chore: update function name
Ziinc Jul 31, 2023
dc9f4eb
fix: table name replacement
Ziinc Jul 31, 2023
b464e2e
fix: cte order by translation without from clause in sandbox
Ziinc Jul 31, 2023
752010c
feat: centralized connection pooling using hash module name
Ziinc Jul 31, 2023
2fc1c9c
fix: startup backend source for postgres backend in supabase mode
Ziinc Jul 31, 2023
e52ddbf
fix: correct test schema is nil
Ziinc Jul 31, 2023
5d58749
docs: arameter posiiton
Ziinc Jul 31, 2023
ac80dc1
docs: spec for parameter_positions
Ziinc Jul 31, 2023
8526a35
feat: positional parameters for postgres queries
Ziinc Jul 31, 2023
2a23eff
chore: formatting
Ziinc Jul 31, 2023
8dc22b6
chore: fix failing test for repo name
Ziinc Jul 31, 2023
c44b4e1
fix: remove unused parameters
Ziinc Jul 31, 2023
46956ed
fix: cte cross join unnest alias path mappings
Ziinc Aug 1, 2023
4b26d44
chore: formatting
Ziinc Aug 1, 2023
d30a190
fix: cte alias referencing for multiple from tables
Ziinc Aug 1, 2023
2c1e3f8
feat: update logs.all endpoint
Ziinc Aug 1, 2023
3ee8ad1
chore: fix compilation warnings, remove unused code
Ziinc Aug 1, 2023
1fa0818
Merge pull request #1647 from Logflare/feat/sql-translation-supabase-…
Ziinc Aug 1, 2023
2957224
fix - Increases text area for Queries in Endpoints
filipecabaco Aug 1, 2023
d6e7c73
chore: upgrade logflare_logger_backend
chasers Aug 1, 2023
17deec3
fix: malformed name translation when global project id is not set
Ziinc Aug 1, 2023
827c54b
fix - Use Text instead of String on event_message
filipecabaco Aug 1, 2023
b593ba8
Merge branch 'staging' into chore/upgrade-logflare-logger
chasers Aug 1, 2023
32a2560
Merge pull request #1656 from Logflare/chore/upgrade-logflare-logger
chasers Aug 1, 2023
63bf888
fix: holy `:counters` batman
chasers Aug 1, 2023
7f16df2
Merge pull request #1658 from Logflare/fix/global-project-id-not-set-…
Ziinc Aug 2, 2023
b9c07ca
fix: cleanup - docs, specs, fn names, fn order
chasers Aug 2, 2023
d3be2ed
Merge branch 'staging' into fix/use-counters-for-buffercounter
chasers Aug 2, 2023
5db0763
fix: nit
chasers Aug 2, 2023
6db54cc
fix: another nit
chasers Aug 2, 2023
aaad700
fix: more nits
chasers Aug 2, 2023
22d2a3b
Merge pull request #1659 from Logflare/fix/use-counters-for-buffercou…
chasers Aug 2, 2023
8d2888e
Merge branch 'master' into staging
chasers Aug 2, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.3.26
1.3.27
6 changes: 4 additions & 2 deletions lib/logflare/application.ex
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ defmodule Logflare.Application do
{DynamicSupervisor,
strategy: :one_for_one, name: Logflare.Backends.Adaptor.PostgresAdaptor.Supervisor},
{Registry, name: Logflare.Backends.SourceRegistry, keys: :unique},
{Registry, name: Logflare.Backends.SourceDispatcher, keys: :duplicate}
{Registry, name: Logflare.Backends.SourceDispatcher, keys: :duplicate},
{Registry, name: Logflare.CounterRegistry, keys: :unique}
] ++ common_children()
end

Expand Down Expand Up @@ -140,7 +141,8 @@ defmodule Logflare.Application do
{DynamicSupervisor,
strategy: :one_for_one, name: Logflare.Backends.Adaptor.PostgresAdaptor.Supervisor},
{Registry, name: Logflare.Backends.SourceRegistry, keys: :unique},
{Registry, name: Logflare.Backends.SourceDispatcher, keys: :duplicate}
{Registry, name: Logflare.Backends.SourceDispatcher, keys: :duplicate},
{Registry, name: Logflare.CounterRegistry, keys: :unique}
] ++ conditional_children() ++ common_children()
end

Expand Down
7 changes: 4 additions & 3 deletions lib/logflare/backends/adaptor/postgres_adaptor/pg_repo.ex
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,10 @@ defmodule Logflare.Backends.Adaptor.PostgresAdaptor.PgRepo do
Retrieves the repo module. Requires `:source` to be preloaded.
"""
@spec get_repo_module(SourceBackend.t()) :: Ecto.Repo.t()
def get_repo_module(%SourceBackend{source: %Source{token: token}}) do
token = token |> Atom.to_string() |> String.replace("-", "")
Module.concat([Logflare.Repo.Postgres, "Adaptor#{token}"])
def get_repo_module(%SourceBackend{config: config, source: %Source{}}) do
data = inspect(config)
sha256 = :crypto.hash(:sha256, data) |> Base.encode16() |> String.downcase()
Module.concat([Logflare.Repo.Postgres, "Adaptor#{sha256}"])
end

@doc """
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ defmodule Logflare.Backends.Adaptor.PostgresAdaptor.Repo.Migrations.AddLogEvents
create table(unquote(table_name), primary_key: false) do
add(:id, :string, primary_key: true)
add(:body, :map)
add(:event_message, :string)
add(:event_message, :text)
add(:timestamp, :utc_datetime_usec)
end
end
Expand Down
35 changes: 28 additions & 7 deletions lib/logflare/endpoints.ex
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ defmodule Logflare.Endpoints do
alias Logflare.Utils
alias Logflare.Backends
alias Logflare.Backends.Adaptor.PostgresAdaptor
alias Logflare.SingleTenant

import Ecto.Query
@typep run_query_return :: {:ok, %{rows: [map()]}} | {:error, String.t()}
Expand Down Expand Up @@ -163,10 +164,17 @@ defmodule Logflare.Endpoints do

with {:ok, declared_params} <- Logflare.Sql.parameters(query_string),
{:ok, transformed_query} <-
Logflare.Sql.transform(endpoint_query.language, transform_input, user_id),
{:ok, result} <-
exec_query_on_backend(endpoint_query, transformed_query, declared_params, params) do
{:ok, result}
Logflare.Sql.transform(endpoint_query.language, transform_input, user_id) do
{endpoint, query_string} =
if SingleTenant.supabase_mode?() and SingleTenant.postgres_backend_adapter_opts() != nil do
# translate the query
{:ok, q} = Logflare.Sql.translate(:bq_sql, :pg_sql, transformed_query)
{Map.put(endpoint_query, :language, :pg_sql), q}
else
{endpoint_query, transformed_query}
end

exec_query_on_backend(endpoint, query_string, declared_params, params)
end
end

Expand Down Expand Up @@ -213,10 +221,10 @@ defmodule Logflare.Endpoints do
end

defp exec_query_on_backend(
%Query{language: :pg_sql} = endpoint_query,
%Query{query: query_string, language: :pg_sql} = endpoint_query,
transformed_query,
_declared_params,
_params
params
) do
# find compatible source backend
# TODO: move this to Backends module
Expand All @@ -233,7 +241,20 @@ defmodule Logflare.Endpoints do
other
end)

with {:ok, rows} <- PostgresAdaptor.execute_query(source_backend, transformed_query) do
# convert params to PG params style
positions =
Logflare.Sql.parameter_positions(query_string)
|> then(fn {:ok, params} ->
params
|> Enum.sort_by(&{elem(&1, 0)})
end)

args =
for {_pos, parameter} <- positions do
Map.get(params, parameter)
end

with {:ok, rows} <- PostgresAdaptor.execute_query(source_backend, {transformed_query, args}) do
{:ok, %{rows: rows}}
end
end
Expand Down
7 changes: 6 additions & 1 deletion lib/logflare/single_tenant.ex
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ defmodule Logflare.SingleTenant do
alias Logflare.Source.Supervisor
alias Logflare.Source.BigQuery.Schema
alias Logflare.LogEvent
alias Logflare.Backends
require Logger

@user_attrs %{
Expand Down Expand Up @@ -168,7 +169,11 @@ defmodule Logflare.SingleTenant do

if user do
for source <- Sources.list_sources_by_user(user) do
Supervisor.ensure_started(source.token)
if postgres_backend?() do
Backends.start_source_sup(source)
else
Supervisor.ensure_started(source.token)
end
end
end

Expand Down
Loading