From 6991bc214a7563b2eda2ce61016cc09e087f7841 Mon Sep 17 00:00:00 2001 From: Erik Vattekar Date: Tue, 26 Nov 2024 15:20:37 +0100 Subject: [PATCH 1/4] fix console team grapqhl api changed --- pkg/nc/nc.go | 35 +++++++++++++++++++++++++---------- pkg/nc/nc_test.go | 26 +++++++++++++++++++------- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/pkg/nc/nc.go b/pkg/nc/nc.go index 1545c29a..7c09e0f9 100644 --- a/pkg/nc/nc.go +++ b/pkg/nc/nc.go @@ -7,6 +7,8 @@ import ( "encoding/json" "fmt" "net/http" + + "github.com/btcsuite/btcutil/base58" ) type Fetcher interface { @@ -21,9 +23,17 @@ type Client struct { } type Team struct { - Slug string `json:"slug"` - GoogleGroupEmail string `json:"googleGroupEmail"` - Environments []Environment `json:"environments"` + Slug string `json:"slug"` + ExternalResources ExternalResources `json:"externalResources"` + Environments []Environment `json:"environments"` +} + +type ExternalResources struct { + GoogleGroup GoogleGroup `json:"googleGroup"` +} + +type GoogleGroup struct { + Email string `json:"email"` } type Environment struct { @@ -55,11 +65,16 @@ type TeamOutput struct { func (c *Client) GetTeamGoogleProjects(ctx context.Context) (map[string]TeamOutput, error) { gqlQuery := ` - query GCPTeams($limit: Int, $offset: Int){ - teams(limit: $limit, offset: $offset) { + query GCPTeams($first: Int, $after: Cursor){ + teams(first: $first, after: $after){ nodes { slug - googleGroupEmail + id + externalResources { + googleGroup { + email + } + } environments { name gcpProjectID @@ -81,14 +96,14 @@ func (c *Client) GetTeamGoogleProjects(ctx context.Context) (map[string]TeamOutp payload := map[string]any{ "query": gqlQuery, "variables": map[string]any{ - "limit": limit, - "offset": offset, + "first": limit, + "after": base58.Encode([]byte(fmt.Sprintf("v1:%d", offset))), }, } r := Response{} - err := c.sendRequestAndDeserialize(ctx, http.MethodPost, "/query", payload, &r) + err := c.sendRequestAndDeserialize(ctx, http.MethodPost, "/graphql", payload, &r) if err != nil { return nil, fmt.Errorf("fetching team google projects: %w", err) } @@ -97,7 +112,7 @@ func (c *Client) GetTeamGoogleProjects(ctx context.Context) (map[string]TeamOutp for _, env := range team.Environments { if env.Name == c.naisClusterName { mapping[team.Slug] = TeamOutput{ - GroupEmail: team.GoogleGroupEmail, + GroupEmail: team.ExternalResources.GoogleGroup.Email, GCPProjectID: env.GcpProjectID, } } diff --git a/pkg/nc/nc_test.go b/pkg/nc/nc_test.go index cc679042..069673e4 100644 --- a/pkg/nc/nc_test.go +++ b/pkg/nc/nc_test.go @@ -28,8 +28,12 @@ func TestClient_GetTeamGoogleProjects(t *testing.T) { Teams: nc.Teams{ Nodes: []nc.Team{ { - Slug: "team-1", - GoogleGroupEmail: "team-1@nav.no", + Slug: "team-1", + ExternalResources: nc.ExternalResources{ + GoogleGroup: nc.GoogleGroup{ + Email: "team-1@nav.no", + }, + }, Environments: []nc.Environment{ { Name: "env-1", @@ -38,8 +42,12 @@ func TestClient_GetTeamGoogleProjects(t *testing.T) { }, }, { - Slug: "team-2", - GoogleGroupEmail: "team-1@nav.no", + Slug: "team-2", + ExternalResources: nc.ExternalResources{ + GoogleGroup: nc.GoogleGroup{ + Email: "team-1@nav.no", + }, + }, Environments: []nc.Environment{ { Name: "env-2", @@ -69,8 +77,12 @@ func TestClient_GetTeamGoogleProjects(t *testing.T) { Teams: nc.Teams{ Nodes: []nc.Team{ { - Slug: "team-1", - GoogleGroupEmail: "team-1@nav.no", + Slug: "team-1", + ExternalResources: nc.ExternalResources{ + GoogleGroup: nc.GoogleGroup{ + Email: "team-1@nav.no", + }, + }, Environments: []nc.Environment{ { Name: "env-2", @@ -94,7 +106,7 @@ func TestClient_GetTeamGoogleProjects(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/query", r.URL.Path) + assert.Equal(t, "/graphql", r.URL.Path) assert.Equal(t, "Bearer super-secret", r.Header.Get("Authorization")) assert.Equal(t, http.MethodPost, r.Method) From c7bb91456aaf953bef99f8df52788c94adacfbcc Mon Sep 17 00:00:00 2001 From: Erik Vattekar Date: Tue, 26 Nov 2024 16:10:43 +0100 Subject: [PATCH 2/4] fix metabase table visibility bug. collection is set to 0 in metabase_metadata after for open databases --- pkg/database/gensql/metabase_metadata.sql.go | 2 +- pkg/database/queries/metabase_metadata.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/database/gensql/metabase_metadata.sql.go b/pkg/database/gensql/metabase_metadata.sql.go index e4a0cb9d..02bdc439 100644 --- a/pkg/database/gensql/metabase_metadata.sql.go +++ b/pkg/database/gensql/metabase_metadata.sql.go @@ -123,7 +123,7 @@ WITH sources_in_same_dataset AS ( SELECT table_name FROM sources_in_same_dataset sds JOIN metabase_metadata mbm ON mbm.dataset_id = sds.dataset_id -WHERE mbm.collection_id IS null +WHERE mbm.collection_id = 0 OR mbm.collection_id IS NULL ` type GetOpenMetabaseTablesInSameBigQueryDatasetParams struct { diff --git a/pkg/database/queries/metabase_metadata.sql b/pkg/database/queries/metabase_metadata.sql index 9f88ed23..4ce8a59d 100644 --- a/pkg/database/queries/metabase_metadata.sql +++ b/pkg/database/queries/metabase_metadata.sql @@ -72,4 +72,4 @@ WITH sources_in_same_dataset AS ( SELECT table_name FROM sources_in_same_dataset sds JOIN metabase_metadata mbm ON mbm.dataset_id = sds.dataset_id -WHERE mbm.collection_id IS null; +WHERE mbm.collection_id = 0 OR mbm.collection_id IS NULL; From a85c4f863d0f2d0061c58b24d585306a91177294 Mon Sep 17 00:00:00 2001 From: Erik Vattekar Date: Wed, 27 Nov 2024 10:32:06 +0100 Subject: [PATCH 3/4] metabase: isrestricted check must evaluate permission group id not collection id as collection is not removed when opening a restricted mb database --- pkg/database/gensql/metabase_metadata.sql.go | 2 +- pkg/database/queries/metabase_metadata.sql | 2 +- pkg/service/core/service_metabase.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/database/gensql/metabase_metadata.sql.go b/pkg/database/gensql/metabase_metadata.sql.go index 02bdc439..8ac765ed 100644 --- a/pkg/database/gensql/metabase_metadata.sql.go +++ b/pkg/database/gensql/metabase_metadata.sql.go @@ -123,7 +123,7 @@ WITH sources_in_same_dataset AS ( SELECT table_name FROM sources_in_same_dataset sds JOIN metabase_metadata mbm ON mbm.dataset_id = sds.dataset_id -WHERE mbm.collection_id = 0 OR mbm.collection_id IS NULL +WHERE mbm.permission_group_id = 0 ` type GetOpenMetabaseTablesInSameBigQueryDatasetParams struct { diff --git a/pkg/database/queries/metabase_metadata.sql b/pkg/database/queries/metabase_metadata.sql index 4ce8a59d..0a136ac1 100644 --- a/pkg/database/queries/metabase_metadata.sql +++ b/pkg/database/queries/metabase_metadata.sql @@ -72,4 +72,4 @@ WITH sources_in_same_dataset AS ( SELECT table_name FROM sources_in_same_dataset sds JOIN metabase_metadata mbm ON mbm.dataset_id = sds.dataset_id -WHERE mbm.collection_id = 0 OR mbm.collection_id IS NULL; +WHERE mbm.permission_group_id = 0; diff --git a/pkg/service/core/service_metabase.go b/pkg/service/core/service_metabase.go index bdf5f306..4de30088 100644 --- a/pkg/service/core/service_metabase.go +++ b/pkg/service/core/service_metabase.go @@ -925,7 +925,7 @@ func (s *metabaseService) SyncTableVisibility(ctx context.Context, meta *service } func isRestrictedDatabase(meta *service.MetabaseMetadata) bool { - if meta.CollectionID != nil && *meta.CollectionID != 0 { + if meta.PermissionGroupID != nil && *meta.PermissionGroupID != 0 { return true } From a6a1278156d59f98eb6a88d784c04633a053e1a8 Mon Sep 17 00:00:00 2001 From: Erik Vattekar Date: Wed, 27 Nov 2024 13:44:18 +0100 Subject: [PATCH 4/4] console sync, handle pagination correctly --- pkg/nc/nc.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/nc/nc.go b/pkg/nc/nc.go index 7c09e0f9..24ae0c8a 100644 --- a/pkg/nc/nc.go +++ b/pkg/nc/nc.go @@ -7,8 +7,6 @@ import ( "encoding/json" "fmt" "net/http" - - "github.com/btcsuite/btcutil/base58" ) type Fetcher interface { @@ -55,7 +53,8 @@ type Teams struct { } type PageInfo struct { - HasNextPage bool `json:"hasNextPage"` + HasNextPage bool `json:"hasNextPage"` + EndCursor *string `json:"endCursor"` } type TeamOutput struct { @@ -69,7 +68,6 @@ func (c *Client) GetTeamGoogleProjects(ctx context.Context) (map[string]TeamOutp teams(first: $first, after: $after){ nodes { slug - id externalResources { googleGroup { email @@ -82,27 +80,28 @@ func (c *Client) GetTeamGoogleProjects(ctx context.Context) (map[string]TeamOutp } pageInfo { hasNextPage + endCursor } } } ` const limit = 100 - offset := 0 + var offset *string mapping := map[string]TeamOutput{} for { + r := Response{} + payload := map[string]any{ "query": gqlQuery, "variables": map[string]any{ "first": limit, - "after": base58.Encode([]byte(fmt.Sprintf("v1:%d", offset))), + "after": offset, }, } - r := Response{} - err := c.sendRequestAndDeserialize(ctx, http.MethodPost, "/graphql", payload, &r) if err != nil { return nil, fmt.Errorf("fetching team google projects: %w", err) @@ -123,7 +122,7 @@ func (c *Client) GetTeamGoogleProjects(ctx context.Context) (map[string]TeamOutp break } - offset += limit + offset = r.Data.Teams.PageInfo.EndCursor } return mapping, nil