From 36be91f5c4269de34d2a770501a1de8b8e4001cf Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Sat, 2 Dec 2023 14:03:14 -0800 Subject: [PATCH 1/9] feat: adds support to list sites #345 --- cmd/root.go | 2 ++ cmd/sites/list.go | 36 +++++++++++++++++++++++++ cmd/sites/sites.go | 37 ++++++++++++++++++++++++++ internal/client/sites/sites.go | 48 ++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+) create mode 100644 cmd/sites/list.go create mode 100644 cmd/sites/sites.go create mode 100644 internal/client/sites/sites.go diff --git a/cmd/root.go b/cmd/root.go index d1f126c17..38e0b296f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -54,6 +54,7 @@ import ( res "github.com/apigee/apigeecli/cmd/res" "github.com/apigee/apigeecli/cmd/securityprofiles" "github.com/apigee/apigeecli/cmd/sharedflows" + "github.com/apigee/apigeecli/cmd/sites" "github.com/apigee/apigeecli/cmd/sync" targetservers "github.com/apigee/apigeecli/cmd/targetservers" "github.com/apigee/apigeecli/cmd/token" @@ -180,6 +181,7 @@ func init() { RootCmd.AddCommand(apicategories.Cmd) RootCmd.AddCommand(datastores.Cmd) RootCmd.AddCommand(securityprofiles.Cmd) + RootCmd.AddCommand(sites.Cmd) } func initConfig() { diff --git a/cmd/sites/list.go b/cmd/sites/list.go new file mode 100644 index 000000000..7ea7b1924 --- /dev/null +++ b/cmd/sites/list.go @@ -0,0 +1,36 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sites + +import ( + "internal/apiclient" + "internal/client/sites" + + "github.com/spf13/cobra" +) + +// ListCmd to list catalog items +var ListCmd = &cobra.Command{ + Use: "list", + Short: "Returns the API Portals associated with the org", + Long: "Returns the API Portals associated with the org", + Args: func(cmd *cobra.Command, args []string) (err error) { + return apiclient.SetApigeeOrg(org) + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + _, err = sites.List() + return + }, +} diff --git a/cmd/sites/sites.go b/cmd/sites/sites.go new file mode 100644 index 000000000..4f1791c7b --- /dev/null +++ b/cmd/sites/sites.go @@ -0,0 +1,37 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sites + +import ( + "github.com/spf13/cobra" +) + +// Cmd to manage +var Cmd = &cobra.Command{ + Use: "sites", + Short: "Manage Apigee API Portals", + Long: "Manage Apigee API Portals", +} + +var org, siteid, id, name string + +func init() { + Cmd.PersistentFlags().StringVarP(&org, "org", "o", + "", "Apigee organization name") + + Cmd.AddCommand(ListCmd) + + _ = Cmd.MarkFlagRequired("org") +} diff --git a/internal/client/sites/sites.go b/internal/client/sites/sites.go new file mode 100644 index 000000000..a349c2119 --- /dev/null +++ b/internal/client/sites/sites.go @@ -0,0 +1,48 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sites + +import ( + "internal/apiclient" + "net/url" + "path" + + "github.com/thedevsaddam/gojsonq" +) + +// List +func List() (respBody []byte, err error) { + u, _ := url.Parse(apiclient.BaseURL) + u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites") + respBody, err = apiclient.HttpClient(u.String()) + return respBody, err +} + +// GetSiteIDs +func GetSiteIDs() (siteIDs []string, err error) { + apiclient.ClientPrintHttpResponse.Set(false) + defer apiclient.ClientPrintHttpResponse.Set(apiclient.GetCmdPrintHttpResponseSetting()) + respBody, err := List() + if err != nil { + return nil, err + } + jq := gojsonq.New().JSONString(string(respBody)) + ids := jq.From("data").Pluck("id").([]interface{}) + siteIDs = make([]string, len(ids)) + for k, v := range ids { + siteIDs[k] = v.(string) + } + return siteIDs, nil +} From 69bbead007b7a46a7bcbd7872d3a829f1565977a Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Sat, 2 Dec 2023 14:10:02 -0800 Subject: [PATCH 2/9] bug: next page token not set --- internal/client/securityprofiles/securityprofiles.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/client/securityprofiles/securityprofiles.go b/internal/client/securityprofiles/securityprofiles.go index 6f080ea45..f75ae15a1 100644 --- a/internal/client/securityprofiles/securityprofiles.go +++ b/internal/client/securityprofiles/securityprofiles.go @@ -179,6 +179,7 @@ func Export(conn int, folder string, allRevisions bool) (err error) { return fmt.Errorf("failed to unmarshall: %w", err) } listsecprofiles.SecurityProfiles = append(listsecprofiles.SecurityProfiles, l.SecurityProfiles...) + pageToken = l.NextPageToken if l.NextPageToken == "" { break } From b2704788fb774f90d67a55464eb6656e7b3a7dfc Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Sat, 2 Dec 2023 14:10:42 -0800 Subject: [PATCH 3/9] feat: export apidocs to a file #344 --- cmd/apidocs/apidocs.go | 1 + cmd/apidocs/export.go | 54 +++++++++++++++++++ cmd/apidocs/list.go | 14 ++++- cmd/org/export.go | 13 ++++- cmd/org/variables.go | 1 + internal/client/apidocs/apidocs.go | 86 +++++++++++++++++++++++++++++- 6 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 cmd/apidocs/export.go diff --git a/cmd/apidocs/apidocs.go b/cmd/apidocs/apidocs.go index 4e2826b41..011f52d0d 100644 --- a/cmd/apidocs/apidocs.go +++ b/cmd/apidocs/apidocs.go @@ -39,6 +39,7 @@ func init() { Cmd.AddCommand(DocCmd) Cmd.AddCommand(CreateCmd) Cmd.AddCommand(UpdateCmd) + Cmd.AddCommand(ExpCmd) _ = Cmd.MarkFlagRequired("org") _ = Cmd.MarkFlagRequired("siteid") diff --git a/cmd/apidocs/export.go b/cmd/apidocs/export.go new file mode 100644 index 000000000..9849a25f2 --- /dev/null +++ b/cmd/apidocs/export.go @@ -0,0 +1,54 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apidocs + +import ( + "os" + + "internal/apiclient" + + "internal/client/apidocs" + + "github.com/spf13/cobra" +) + +// ExpCmd to export apidocs +var ExpCmd = &cobra.Command{ + Use: "export", + Short: "Export API Docs to a file", + Long: "Export API Docs to a file", + Args: func(cmd *cobra.Command, args []string) (err error) { + return apiclient.SetApigeeOrg(org) + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + if folder == "" { + folder, _ = os.Getwd() + } + if err = apiclient.FolderExists(folder); err != nil { + return err + } + return apidocs.Export(folder) + }, +} + +var ( + folder string +) + +func init() { + + ExpCmd.Flags().StringVarP(&folder, "folder", "f", + "", "folder to export API Docs") +} diff --git a/cmd/apidocs/list.go b/cmd/apidocs/list.go index 38a7f5284..2cc187982 100644 --- a/cmd/apidocs/list.go +++ b/cmd/apidocs/list.go @@ -30,7 +30,19 @@ var ListCmd = &cobra.Command{ return apiclient.SetApigeeOrg(org) }, RunE: func(cmd *cobra.Command, args []string) (err error) { - _, err = apidocs.List(siteid) + _, err = apidocs.List(siteid, pageSize, pageToken) return }, } + +var ( + pageSize int + pageToken string +) + +func init() { + ListCmd.Flags().IntVarP(&pageSize, "page-size", "", + -1, "The maximum number of versions to return") + ListCmd.Flags().StringVarP(&pageToken, "page-token", "", + "", "A page token, received from a previous call") +} diff --git a/cmd/org/export.go b/cmd/org/export.go index 450735eb0..a477c599f 100644 --- a/cmd/org/export.go +++ b/cmd/org/export.go @@ -26,6 +26,7 @@ import ( "internal/clilog" + "internal/client/apidocs" "internal/client/apis" "internal/client/appgroups" "internal/client/apps" @@ -180,6 +181,9 @@ var ExportCmd = &cobra.Command{ } } + clilog.Info.Println("Exporting API Portal apidocs Configuration...") + apidocs.Export(portalsFolderName) + if runtimeType == "HYBRID" { clilog.Info.Println("Exporting Sync Authorization Identities...") if respBody, err = sync.Get(); err != nil { @@ -307,8 +311,13 @@ func createFolders() (err error) { if err = os.Mkdir(proxiesFolderName, 0o755); err != nil { return err } - err = os.Mkdir(sharedFlowsFolderName, 0o755) - return err + if os.Mkdir(sharedFlowsFolderName, 0o755); err != nil { + return err + } + if os.Mkdir(portalsFolderName, 0o755); err != nil { + return err + } + return nil } func exportKVMEntries(scope string, env string, listKVMBytes []byte) (err error) { diff --git a/cmd/org/variables.go b/cmd/org/variables.go index dd377ec6c..175fa499a 100644 --- a/cmd/org/variables.go +++ b/cmd/org/variables.go @@ -32,6 +32,7 @@ const ( proxiesFolderName = "proxies" sharedFlowsFolderName = "sharedflows" + portalsFolderName = "portals" ) var conn int diff --git a/internal/client/apidocs/apidocs.go b/internal/client/apidocs/apidocs.go index affc67ff3..7b5b400d3 100644 --- a/internal/client/apidocs/apidocs.go +++ b/internal/client/apidocs/apidocs.go @@ -15,11 +15,15 @@ package apidocs import ( + "encoding/json" + "fmt" "net/url" "path" + "strconv" "strings" "internal/apiclient" + "internal/client/sites" ) type Action uint8 @@ -29,6 +33,37 @@ const ( UPDATE ) +const maxPageSize = 100 + +type listapidocs struct { + Status string `json:"status,omitempty"` + Message string `json:"message,omitempty"` + RequestID string `json:"requestId,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + Data []data `json:"data,omitempty"` + NextPageToken string `json:"nextPageToken,omitempty"` +} + +type data struct { + SiteID string `json:"siteId,omitempty"` + ID string `json:"id,omitempty"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + Published bool `json:"published,omitempty"` + AnonAllowed bool `json:"anonAllowed,omitempty"` + ApiProductName string `json:"apiProductName,omitempty"` + RequireCallbackUrl bool `json:"requireCallbackUrl,omitempty"` + ImageUrl string `json:"imageUrl,omitempty"` + CategoryIDs []string `json:"categoryIds,omitempty"` + Modified string `json:"modified,omitempty"` + Visibility bool `json:"visibility,omitempty"` + EdgeAPIProductName string `json:"edgeAPIProductName,omitempty"` + SpecID string `json:"specId,omitempty"` + GraphqlSchema string `json:"graphqlSchema,omitempty"` + GraphqlEndpointUrl string `json:"graphqlEndpointUrl,omitempty"` + GraphqlSchemaDisplayName string `json:"graphqlSchemaDisplayName,omitempty"` +} + // Create func Create(siteid string, title string, description string, published string, anonAllowed string, apiProductName string, requireCallbackUrl string, imageUrl string, @@ -113,9 +148,18 @@ func Get(siteid string, id string) (respBody []byte, err error) { } // List -func List(siteid string) (respBody []byte, err error) { +func List(siteid string, pageSize int, pageToken string) (respBody []byte, err error) { u, _ := url.Parse(apiclient.BaseURL) u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apidocs") + q := u.Query() + if pageSize != -1 { + q.Set("pageSize", strconv.Itoa(pageSize)) + } + if pageToken != "" { + q.Set("pageToken", pageToken) + } + + u.RawQuery = q.Encode() respBody, err = apiclient.HttpClient(u.String()) return respBody, err } @@ -150,6 +194,46 @@ func UpdateDocumentation(siteid string, id string, displayName string, openAPIDo return nil, nil } +// Export +func Export(folder string) (err error) { + apiclient.ClientPrintHttpResponse.Set(false) + defer apiclient.ClientPrintHttpResponse.Set(apiclient.GetCmdPrintHttpResponseSetting()) + + siteids, err := sites.GetSiteIDs() + if err != nil { + return err + } + + pageToken := "" + listdocs := listapidocs{} + + for _, siteid := range siteids { + for { + l := listapidocs{} + listRespBytes, err := List(siteid, maxPageSize, pageToken) + if err != nil { + return fmt.Errorf("failed to fetch apidocs: %w", err) + } + err = json.Unmarshal(listRespBytes, &l) + if err != nil { + return fmt.Errorf("failed to unmarshall: %w", err) + } + listdocs.Data = append(listdocs.Data, l.Data...) + pageToken = l.NextPageToken + if l.NextPageToken == "" { + break + } + } + } + + respBody, err := json.Marshal(listdocs.Data) + if err != nil { + return err + } + respBody, _ = apiclient.PrettifyJSON(respBody) + return apiclient.WriteByteArrayToFile(path.Join(folder, "apidocs.json"), false, respBody) +} + func getArrayStr(str []string) string { tmp := strings.Join(str, ",") tmp = strings.ReplaceAll(tmp, ",", "\",\"") From 1daa09454a2804ab590430aea40e4b3859401187 Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Sat, 2 Dec 2023 14:14:19 -0800 Subject: [PATCH 4/9] chore: fix linting issues #345 --- cmd/apidocs/export.go | 2 +- cmd/org/export.go | 8 +++++--- cmd/sites/sites.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/apidocs/export.go b/cmd/apidocs/export.go index 9849a25f2..ff21c56da 100644 --- a/cmd/apidocs/export.go +++ b/cmd/apidocs/export.go @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2023 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/org/export.go b/cmd/org/export.go index a477c599f..2c278ab5d 100644 --- a/cmd/org/export.go +++ b/cmd/org/export.go @@ -182,7 +182,9 @@ var ExportCmd = &cobra.Command{ } clilog.Info.Println("Exporting API Portal apidocs Configuration...") - apidocs.Export(portalsFolderName) + if err = apidocs.Export(portalsFolderName); proceedOnError(err) != nil { + return err + } if runtimeType == "HYBRID" { clilog.Info.Println("Exporting Sync Authorization Identities...") @@ -311,10 +313,10 @@ func createFolders() (err error) { if err = os.Mkdir(proxiesFolderName, 0o755); err != nil { return err } - if os.Mkdir(sharedFlowsFolderName, 0o755); err != nil { + if err = os.Mkdir(sharedFlowsFolderName, 0o755); err != nil { return err } - if os.Mkdir(portalsFolderName, 0o755); err != nil { + if err = os.Mkdir(portalsFolderName, 0o755); err != nil { return err } return nil diff --git a/cmd/sites/sites.go b/cmd/sites/sites.go index 4f1791c7b..2774ffd9a 100644 --- a/cmd/sites/sites.go +++ b/cmd/sites/sites.go @@ -25,7 +25,7 @@ var Cmd = &cobra.Command{ Long: "Manage Apigee API Portals", } -var org, siteid, id, name string +var org string func init() { Cmd.PersistentFlags().StringVarP(&org, "org", "o", From 3e04f1f60b8a743e5ede8376ebf97119555eecf6 Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:32:48 -0800 Subject: [PATCH 5/9] feat: add support to import apidocs #345 --- .gitignore | 3 ++ cmd/apidocs/apidocs.go | 1 + cmd/apidocs/export.go | 6 ++-- cmd/apidocs/import.go | 50 ++++++++++++++++++++++++++++++ internal/client/apidocs/apidocs.go | 29 ++++++++++++++++- internal/client/sites/sites.go | 3 +- 6 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 cmd/apidocs/import.go diff --git a/.gitignore b/.gitignore index f78d454de..7a54dd572 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ apiproxy/ #keys for envoy remote-service.* + +#json files +*.json diff --git a/cmd/apidocs/apidocs.go b/cmd/apidocs/apidocs.go index 011f52d0d..11dc44b34 100644 --- a/cmd/apidocs/apidocs.go +++ b/cmd/apidocs/apidocs.go @@ -40,6 +40,7 @@ func init() { Cmd.AddCommand(CreateCmd) Cmd.AddCommand(UpdateCmd) Cmd.AddCommand(ExpCmd) + Cmd.AddCommand(ImpCmd) _ = Cmd.MarkFlagRequired("org") _ = Cmd.MarkFlagRequired("siteid") diff --git a/cmd/apidocs/export.go b/cmd/apidocs/export.go index ff21c56da..60212cfc6 100644 --- a/cmd/apidocs/export.go +++ b/cmd/apidocs/export.go @@ -39,16 +39,14 @@ var ExpCmd = &cobra.Command{ if err = apiclient.FolderExists(folder); err != nil { return err } + apiclient.DisableCmdPrintHttpResponse() return apidocs.Export(folder) }, } -var ( - folder string -) +var folder string func init() { - ExpCmd.Flags().StringVarP(&folder, "folder", "f", "", "folder to export API Docs") } diff --git a/cmd/apidocs/import.go b/cmd/apidocs/import.go new file mode 100644 index 000000000..7d3f3ca75 --- /dev/null +++ b/cmd/apidocs/import.go @@ -0,0 +1,50 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apidocs + +import ( + "internal/apiclient" + + "internal/client/apidocs" + + "github.com/spf13/cobra" +) + +// ImpCmd to import products +var ImpCmd = &cobra.Command{ + Use: "import", + Short: "Import from a folder containing apidocs", + Long: "Import from a folder containing apidocs", + Args: func(cmd *cobra.Command, args []string) (err error) { + return apiclient.SetApigeeOrg(org) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return apidocs.Import(conn, folder) + }, +} + +var ( + conn int + filePath string +) + +func init() { + ImpCmd.Flags().StringVarP(&filePath, "folder", "f", + "", "Folder containing apidocs.json and apidocs__.json files") + ImpCmd.Flags().IntVarP(&conn, "conn", "c", + 4, "Number of connections") + + _ = ImpCmd.MarkFlagRequired("file") +} diff --git a/internal/client/apidocs/apidocs.go b/internal/client/apidocs/apidocs.go index 7b5b400d3..b3c495488 100644 --- a/internal/client/apidocs/apidocs.go +++ b/internal/client/apidocs/apidocs.go @@ -24,6 +24,7 @@ import ( "internal/apiclient" "internal/client/sites" + "internal/clilog" ) type Action uint8 @@ -173,7 +174,9 @@ func Delete(siteid string, id string) (respBody []byte, err error) { } // UpdateDocumentation -func UpdateDocumentation(siteid string, id string, displayName string, openAPIDoc string, graphQLDoc string, endpointUri string) (respBody []byte, err error) { +func UpdateDocumentation(siteid string, id string, displayName string, + openAPIDoc string, graphQLDoc string, endpointUri string, +) (respBody []byte, err error) { var payload string if openAPIDoc != "" { @@ -220,6 +223,17 @@ func Export(folder string) (err error) { } listdocs.Data = append(listdocs.Data, l.Data...) pageToken = l.NextPageToken + // write apidocs Documentation + for _, data := range l.Data { + respDocsBody, err := GetDocumentation(siteid, data.ID) + if err != nil { + return err + } + docFileName := fmt.Sprintf("apidocs_%s_%s.json", siteid, data.ID) + if err = apiclient.WriteByteArrayToFile(path.Join(folder, docFileName), false, respDocsBody); err != nil { + return err + } + } if l.NextPageToken == "" { break } @@ -234,8 +248,21 @@ func Export(folder string) (err error) { return apiclient.WriteByteArrayToFile(path.Join(folder, "apidocs.json"), false, respBody) } +func Import(conn int, folder string) (err error) { + entities, err := readAPIDocsFile(path.Join(folder, "apidocs.json")) + if err != nil { + clilog.Error.Println("Error reading file: ", err) + return err + } + return fmt.Errorf("not implemented") +} + func getArrayStr(str []string) string { tmp := strings.Join(str, ",") tmp = strings.ReplaceAll(tmp, ",", "\",\"") return tmp } + +func readAPIDocsFile(filePath string) (d []data, err error) { + return nil, nil +} diff --git a/internal/client/sites/sites.go b/internal/client/sites/sites.go index a349c2119..07a158d79 100644 --- a/internal/client/sites/sites.go +++ b/internal/client/sites/sites.go @@ -15,10 +15,11 @@ package sites import ( - "internal/apiclient" "net/url" "path" + "internal/apiclient" + "github.com/thedevsaddam/gojsonq" ) From 7725dde7c0f44ecd4576a153916b474de4bc44d9 Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:32:20 -0800 Subject: [PATCH 6/9] feat: import apidocs documentation #345 --- internal/client/apidocs/apidocs.go | 172 ++++++++++++++++++++++++++++- 1 file changed, 170 insertions(+), 2 deletions(-) diff --git a/internal/client/apidocs/apidocs.go b/internal/client/apidocs/apidocs.go index b3c495488..67a205435 100644 --- a/internal/client/apidocs/apidocs.go +++ b/internal/client/apidocs/apidocs.go @@ -15,12 +15,18 @@ package apidocs import ( + "encoding/base64" "encoding/json" + "errors" "fmt" + "io" "net/url" + "os" "path" + "path/filepath" "strconv" "strings" + "sync" "internal/apiclient" "internal/client/sites" @@ -45,6 +51,33 @@ type listapidocs struct { NextPageToken string `json:"nextPageToken,omitempty"` } +type documentation struct { + Status string `json:"status,omitempty"` + Message string `json:"message,omitempty"` + RequestID string `json:"requestId,omitempty"` + Data apidocsdata `json:"data,omitempty"` +} + +type apidocsdata struct { + OasDocumentation *oasDocumentation `json:"oasDocumentation,omitempty"` + GraphqlDocumentation *graphqlDocumentation `json:"graphqlDocumentation,omitempty"` +} + +type oasDocumentation struct { + Spec spec `json:"spec,omitempty"` + Format string `json:"format,omitempty"` +} + +type spec struct { + DisplayName string `json:"displayName,omitempty"` + Contents string `json:"contents,omitempty"` +} + +type graphqlDocumentation struct { + Schema spec `json:"schema,omitempty"` + EndpointUri string `json:"endpointUri,omitempty"` +} + type data struct { SiteID string `json:"siteId,omitempty"` ID string `json:"id,omitempty"` @@ -250,11 +283,110 @@ func Export(folder string) (err error) { func Import(conn int, folder string) (err error) { entities, err := readAPIDocsFile(path.Join(folder, "apidocs.json")) + var errs []string + if err != nil { clilog.Error.Println("Error reading file: ", err) return err } - return fmt.Errorf("not implemented") + for _, entity := range entities { + respBodyApiDocs, e := Create(entity.SiteID, entity.Title, entity.Description, + strconv.FormatBool(entity.Published), + strconv.FormatBool(entity.AnonAllowed), entity.ApiProductName, + strconv.FormatBool(entity.RequireCallbackUrl), entity.ImageUrl, entity.CategoryIDs) + if e != nil { + errs = append(errs, e.Error()) + continue + } + d := documentation{} + if e = json.Unmarshal(respBodyApiDocs, &d); e != nil { + errs = append(errs, e.Error()) + continue + } + // get all the files that match this siteid + siteIDDocsList := []string{} + err = filepath.Walk(folder, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + if filepath.Ext(path) != ".json" { + return nil + } + matchCriteria := fmt.Sprintf("apidocs_%s_*.json", entity.SiteID) + if ok, _ := filepath.Match(matchCriteria, path); ok { + siteIDDocsList = append(siteIDDocsList, path) + } + return nil + }) + if err != nil { + return err + } + if len(siteIDDocsList) == 0 { + clilog.Warning.Printf("No API Docs files were found for siteid %s in the folder", entity.SiteID) + continue + } + clilog.Debug.Printf("Found %d API Docs for siteid %s in the folder\n", len(siteIDDocsList), entity.SiteID) + for _, siteIDDoc := range siteIDDocsList { + doc, err := readAPIDocumentationFile(siteIDDoc) + if err != nil { + errs = append(errs, err.Error()) + continue + } + if doc.Data.GraphqlDocumentation != nil { + schema, err := base64.StdEncoding.DecodeString(doc.Data.GraphqlDocumentation.Schema.Contents) + if err != nil { + errs = append(errs, err.Error()) + continue + } + _, err = UpdateDocumentation(entity.SiteID, entity.ID, doc.Data.OasDocumentation.Spec.DisplayName, "", + string(schema), doc.Data.GraphqlDocumentation.EndpointUri) + } else { + oasdoc, err := base64.StdEncoding.DecodeString(doc.Data.OasDocumentation.Spec.Contents) + if err != nil { + errs = append(errs, err.Error()) + continue + } + _, err = UpdateDocumentation(entity.SiteID, entity.ID, doc.Data.OasDocumentation.Spec.DisplayName, string(oasdoc), + "", "") + } + if err != nil { + errs = append(errs, err.Error()) + continue + } + } + } + + if len(errs) > 0 { + return errors.New(strings.Join(errs, "\n")) + } + return nil +} + +func importDocs(wg *sync.WaitGroup, jobs <-chan string, errs chan<- error) { + defer wg.Done() + for { + job, ok := <-jobs + if !ok { + return + } + jsonFile, err := os.Open(job) + if err != nil { + errs <- err + } + byteValue, err := io.ReadAll(jsonFile) + if err != nil { + errs <- err + } + docsFile := documentation{} + err = json.Unmarshal(byteValue, &docsFile) + if err != nil { + errs <- err + } + jsonFile.Close() + } } func getArrayStr(str []string) string { @@ -264,5 +396,41 @@ func getArrayStr(str []string) string { } func readAPIDocsFile(filePath string) (d []data, err error) { - return nil, nil + jsonFile, err := os.Open(filePath) + if err != nil { + return nil, err + } + + defer jsonFile.Close() + + byteValue, err := io.ReadAll(jsonFile) + if err != nil { + return nil, err + } + + err = json.Unmarshal(byteValue, &d) + + if err != nil { + return nil, err + } + return d, nil +} + +func readAPIDocumentationFile(fileName string) (d documentation, err error) { + jsonFile, err := os.Open(fileName) + if err != nil { + return d, err + } + + defer jsonFile.Close() + + content, err := io.ReadAll(jsonFile) + if err != nil { + return d, err + } + err = json.Unmarshal(content, &d) + if err != nil { + return d, err + } + return d, nil } From 4ae1307d7ce5b9c18ad380c416325645a99b2ab4 Mon Sep 17 00:00:00 2001 From: srinandan <13950006+srinandan@users.noreply.github.com> Date: Wed, 6 Dec 2023 08:03:38 -0800 Subject: [PATCH 7/9] feat: adds ability to import apidoc for a siteid #345 --- cmd/apidocs/import.go | 17 ++- internal/client/apidocs/apidocs.go | 164 +++++++++-------------------- 2 files changed, 58 insertions(+), 123 deletions(-) diff --git a/cmd/apidocs/import.go b/cmd/apidocs/import.go index 7d3f3ca75..6e6aab53e 100644 --- a/cmd/apidocs/import.go +++ b/cmd/apidocs/import.go @@ -15,6 +15,7 @@ package apidocs import ( + "fmt" "internal/apiclient" "internal/client/apidocs" @@ -31,20 +32,16 @@ var ImpCmd = &cobra.Command{ return apiclient.SetApigeeOrg(org) }, RunE: func(cmd *cobra.Command, args []string) error { - return apidocs.Import(conn, folder) + if siteid == "" { + return fmt.Errorf("siteid is a mandatory parameter") + } + return apidocs.Import(siteid, folder) }, } -var ( - conn int - filePath string -) - func init() { - ImpCmd.Flags().StringVarP(&filePath, "folder", "f", - "", "Folder containing apidocs.json and apidocs__.json files") - ImpCmd.Flags().IntVarP(&conn, "conn", "c", - 4, "Number of connections") + ImpCmd.Flags().StringVarP(&folder, "folder", "f", + "", "Folder containing site_.json and apidocs__.json files") _ = ImpCmd.MarkFlagRequired("file") } diff --git a/internal/client/apidocs/apidocs.go b/internal/client/apidocs/apidocs.go index 67a205435..df43bf2d3 100644 --- a/internal/client/apidocs/apidocs.go +++ b/internal/client/apidocs/apidocs.go @@ -23,14 +23,11 @@ import ( "net/url" "os" "path" - "path/filepath" "strconv" "strings" - "sync" "internal/apiclient" "internal/client/sites" - "internal/clilog" ) type Action uint8 @@ -51,14 +48,14 @@ type listapidocs struct { NextPageToken string `json:"nextPageToken,omitempty"` } -type documentation struct { - Status string `json:"status,omitempty"` - Message string `json:"message,omitempty"` - RequestID string `json:"requestId,omitempty"` - Data apidocsdata `json:"data,omitempty"` +type apidocsdata struct { + Status string `json:"status,omitempty"` + Message string `json:"message,omitempty"` + RequestID string `json:"requestId,omitempty"` + Data documentation `json:"data,omitempty"` } -type apidocsdata struct { +type documentation struct { OasDocumentation *oasDocumentation `json:"oasDocumentation,omitempty"` GraphqlDocumentation *graphqlDocumentation `json:"graphqlDocumentation,omitempty"` } @@ -241,11 +238,10 @@ func Export(folder string) (err error) { } pageToken := "" - listdocs := listapidocs{} for _, siteid := range siteids { + l := listapidocs{} for { - l := listapidocs{} listRespBytes, err := List(siteid, maxPageSize, pageToken) if err != nil { return fmt.Errorf("failed to fetch apidocs: %w", err) @@ -254,7 +250,6 @@ func Export(folder string) (err error) { if err != nil { return fmt.Errorf("failed to unmarshall: %w", err) } - listdocs.Data = append(listdocs.Data, l.Data...) pageToken = l.NextPageToken // write apidocs Documentation for _, data := range l.Data { @@ -271,152 +266,95 @@ func Export(folder string) (err error) { break } } + respBody, err := json.Marshal(l.Data) + if err != nil { + return err + } + respBody, _ = apiclient.PrettifyJSON(respBody) + if err = apiclient.WriteByteArrayToFile(path.Join(folder, "site_"+siteid+".json"), false, respBody); err != nil { + return err + } } - - respBody, err := json.Marshal(listdocs.Data) - if err != nil { - return err - } - respBody, _ = apiclient.PrettifyJSON(respBody) - return apiclient.WriteByteArrayToFile(path.Join(folder, "apidocs.json"), false, respBody) + return nil } -func Import(conn int, folder string) (err error) { - entities, err := readAPIDocsFile(path.Join(folder, "apidocs.json")) +func Import(siteid string, folder string) (err error) { var errs []string - + docsList, err := readAPIDocsDataFile(path.Join(folder, "site_"+siteid+".json")) if err != nil { - clilog.Error.Println("Error reading file: ", err) return err } - for _, entity := range entities { - respBodyApiDocs, e := Create(entity.SiteID, entity.Title, entity.Description, - strconv.FormatBool(entity.Published), - strconv.FormatBool(entity.AnonAllowed), entity.ApiProductName, - strconv.FormatBool(entity.RequireCallbackUrl), entity.ImageUrl, entity.CategoryIDs) - if e != nil { - errs = append(errs, e.Error()) - continue - } - d := documentation{} - if e = json.Unmarshal(respBodyApiDocs, &d); e != nil { - errs = append(errs, e.Error()) + for _, doc := range docsList { + // 1. create the apidoc object + _, err = Create(siteid, doc.Title, doc.Description, strconv.FormatBool(doc.Published), + strconv.FormatBool(doc.AnonAllowed), doc.ApiProductName, + strconv.FormatBool(doc.RequireCallbackUrl), doc.ImageUrl, doc.CategoryIDs) + if err != nil { + errs = append(errs, err.Error()) continue + } - // get all the files that match this siteid - siteIDDocsList := []string{} - err = filepath.Walk(folder, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if filepath.Ext(path) != ".json" { - return nil - } - matchCriteria := fmt.Sprintf("apidocs_%s_*.json", entity.SiteID) - if ok, _ := filepath.Match(matchCriteria, path); ok { - siteIDDocsList = append(siteIDDocsList, path) - } - return nil - }) + // 2. find the documentation associated with this site + documentationFileName := fmt.Sprintf("apidocs_%s_%s", siteid, doc.ID) + apidocument, err := readAPIDocumentationFile(documentationFileName) if err != nil { - return err - } - if len(siteIDDocsList) == 0 { - clilog.Warning.Printf("No API Docs files were found for siteid %s in the folder", entity.SiteID) + errs = append(errs, err.Error()) continue } - clilog.Debug.Printf("Found %d API Docs for siteid %s in the folder\n", len(siteIDDocsList), entity.SiteID) - for _, siteIDDoc := range siteIDDocsList { - doc, err := readAPIDocumentationFile(siteIDDoc) + if apidocument.Data.GraphqlDocumentation != nil { + schema, err := base64.StdEncoding.DecodeString(apidocument.Data.GraphqlDocumentation.Schema.Contents) if err != nil { errs = append(errs, err.Error()) continue } - if doc.Data.GraphqlDocumentation != nil { - schema, err := base64.StdEncoding.DecodeString(doc.Data.GraphqlDocumentation.Schema.Contents) - if err != nil { - errs = append(errs, err.Error()) - continue - } - _, err = UpdateDocumentation(entity.SiteID, entity.ID, doc.Data.OasDocumentation.Spec.DisplayName, "", - string(schema), doc.Data.GraphqlDocumentation.EndpointUri) - } else { - oasdoc, err := base64.StdEncoding.DecodeString(doc.Data.OasDocumentation.Spec.Contents) - if err != nil { - errs = append(errs, err.Error()) - continue - } - _, err = UpdateDocumentation(entity.SiteID, entity.ID, doc.Data.OasDocumentation.Spec.DisplayName, string(oasdoc), - "", "") - } + _, err = UpdateDocumentation(siteid, doc.ID, apidocument.Data.OasDocumentation.Spec.DisplayName, "", + string(schema), apidocument.Data.GraphqlDocumentation.EndpointUri) + } else { + oasdoc, err := base64.StdEncoding.DecodeString(apidocument.Data.OasDocumentation.Spec.Contents) if err != nil { errs = append(errs, err.Error()) continue } + _, err = UpdateDocumentation(siteid, doc.ID, apidocument.Data.OasDocumentation.Spec.DisplayName, string(oasdoc), + "", "") + } + if err != nil { + errs = append(errs, err.Error()) + continue } } - if len(errs) > 0 { return errors.New(strings.Join(errs, "\n")) } return nil } -func importDocs(wg *sync.WaitGroup, jobs <-chan string, errs chan<- error) { - defer wg.Done() - for { - job, ok := <-jobs - if !ok { - return - } - jsonFile, err := os.Open(job) - if err != nil { - errs <- err - } - byteValue, err := io.ReadAll(jsonFile) - if err != nil { - errs <- err - } - docsFile := documentation{} - err = json.Unmarshal(byteValue, &docsFile) - if err != nil { - errs <- err - } - jsonFile.Close() - } -} - func getArrayStr(str []string) string { tmp := strings.Join(str, ",") tmp = strings.ReplaceAll(tmp, ",", "\",\"") return tmp } -func readAPIDocsFile(filePath string) (d []data, err error) { - jsonFile, err := os.Open(filePath) +func readAPIDocumentationFile(fileName string) (a apidocsdata, err error) { + jsonFile, err := os.Open(fileName) if err != nil { - return nil, err + return a, err } defer jsonFile.Close() - byteValue, err := io.ReadAll(jsonFile) + content, err := io.ReadAll(jsonFile) if err != nil { - return nil, err + return a, err } - - err = json.Unmarshal(byteValue, &d) - + err = json.Unmarshal(content, &a) if err != nil { - return nil, err + return a, err } - return d, nil + return a, nil } -func readAPIDocumentationFile(fileName string) (d documentation, err error) { +func readAPIDocsDataFile(fileName string) (d []data, err error) { jsonFile, err := os.Open(fileName) if err != nil { return d, err From 5c673b51785161db70897618af6482337b6234e5 Mon Sep 17 00:00:00 2001 From: Kurt Kanaskie Date: Tue, 12 Dec 2023 12:49:25 -0500 Subject: [PATCH 8/9] Fixed import, needed new to use doc ID and already encoded specs --- internal/client/apidocs/apidocs.go | 48 +++++++++++++++++++----------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/internal/client/apidocs/apidocs.go b/internal/client/apidocs/apidocs.go index df43bf2d3..6ad009809 100644 --- a/internal/client/apidocs/apidocs.go +++ b/internal/client/apidocs/apidocs.go @@ -15,7 +15,7 @@ package apidocs import ( - "encoding/base64" + // "encoding/base64" "encoding/json" "errors" "fmt" @@ -95,6 +95,14 @@ type data struct { GraphqlSchemaDisplayName string `json:"graphqlSchemaDisplayName,omitempty"` } +type apidocResponse struct { + Status string `json:"status,omitempty"` + Message string `json:"message,omitempty"` + RequestID string `json:"requestId,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + Data data `json:"data,omitempty"` +} + // Create func Create(siteid string, title string, description string, published string, anonAllowed string, apiProductName string, requireCallbackUrl string, imageUrl string, @@ -211,17 +219,18 @@ func UpdateDocumentation(siteid string, id string, displayName string, if openAPIDoc != "" { payload = "{\"oasDocumentation\":{\"spec\":{\"displayName\":\"" + - displayName + "\",\"contents\":" + openAPIDoc + "}}}" + displayName + "\",\"contents\":\"" + openAPIDoc + "\"}}}" } if graphQLDoc != "" { payload = "{\"graphqlDocumentation\":{\"endpointUri\":\"" + endpointUri + "\",\"schema\":{\"displayName\":\"" + displayName + - "\",\"contents\":" + graphQLDoc + "}}}" + "\",\"contents\":\"" + graphQLDoc + "\"}}}" } u, _ := url.Parse(apiclient.BaseURL) u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apidocs", id, "documentation") + fmt.Printf("Kurt: update %s\n%s\n", u.String(), payload) respBody, err = apiclient.HttpClient(u.String(), payload, "PATCH") return nil, nil @@ -280,13 +289,14 @@ func Export(folder string) (err error) { func Import(siteid string, folder string) (err error) { var errs []string + var respBody []byte docsList, err := readAPIDocsDataFile(path.Join(folder, "site_"+siteid+".json")) if err != nil { return err } for _, doc := range docsList { // 1. create the apidoc object - _, err = Create(siteid, doc.Title, doc.Description, strconv.FormatBool(doc.Published), + respBody, err = Create(siteid, doc.Title, doc.Description, strconv.FormatBool(doc.Published), strconv.FormatBool(doc.AnonAllowed), doc.ApiProductName, strconv.FormatBool(doc.RequireCallbackUrl), doc.ImageUrl, doc.CategoryIDs) if err != nil { @@ -294,28 +304,30 @@ func Import(siteid string, folder string) (err error) { continue } + + // get the new doc.ID from the created apidoc + response := apidocResponse{} + err = json.Unmarshal(respBody, &response) + if err != nil { + return err + } + // 2. find the documentation associated with this site - documentationFileName := fmt.Sprintf("apidocs_%s_%s", siteid, doc.ID) + documentationFileName := path.Join(folder, "apidocs_"+siteid+"_"+doc.ID+".json") apidocument, err := readAPIDocumentationFile(documentationFileName) if err != nil { errs = append(errs, err.Error()) continue } if apidocument.Data.GraphqlDocumentation != nil { - schema, err := base64.StdEncoding.DecodeString(apidocument.Data.GraphqlDocumentation.Schema.Contents) - if err != nil { - errs = append(errs, err.Error()) - continue - } - _, err = UpdateDocumentation(siteid, doc.ID, apidocument.Data.OasDocumentation.Spec.DisplayName, "", - string(schema), apidocument.Data.GraphqlDocumentation.EndpointUri) + _, err = UpdateDocumentation(siteid, response.Data.ID, + apidocument.Data.GraphqlDocumentation.Schema.DisplayName, "", + apidocument.Data.GraphqlDocumentation.Schema.Contents, + apidocument.Data.GraphqlDocumentation.EndpointUri) } else { - oasdoc, err := base64.StdEncoding.DecodeString(apidocument.Data.OasDocumentation.Spec.Contents) - if err != nil { - errs = append(errs, err.Error()) - continue - } - _, err = UpdateDocumentation(siteid, doc.ID, apidocument.Data.OasDocumentation.Spec.DisplayName, string(oasdoc), + _, err = UpdateDocumentation(siteid, response.Data.ID, + apidocument.Data.OasDocumentation.Spec.DisplayName, + apidocument.Data.OasDocumentation.Spec.Contents, "", "") } if err != nil { From 3113a870f28ef80908d86208ddca0d61683b1e4b Mon Sep 17 00:00:00 2001 From: Kurt Kanaskie Date: Tue, 12 Dec 2023 15:51:17 -0500 Subject: [PATCH 9/9] Alternate approach to creating JSON payload --- internal/client/apidocs/apidocs.go | 34 +++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/internal/client/apidocs/apidocs.go b/internal/client/apidocs/apidocs.go index 6ad009809..f59e26366 100644 --- a/internal/client/apidocs/apidocs.go +++ b/internal/client/apidocs/apidocs.go @@ -15,7 +15,6 @@ package apidocs import ( - // "encoding/base64" "encoding/json" "errors" "fmt" @@ -215,22 +214,41 @@ func Delete(siteid string, id string) (respBody []byte, err error) { func UpdateDocumentation(siteid string, id string, displayName string, openAPIDoc string, graphQLDoc string, endpointUri string, ) (respBody []byte, err error) { - var payload string + var data map[string]interface{} + // var payload string if openAPIDoc != "" { - payload = "{\"oasDocumentation\":{\"spec\":{\"displayName\":\"" + - displayName + "\",\"contents\":\"" + openAPIDoc + "\"}}}" + data = map[string]interface{}{ + "oasDocumentation": map[string]interface{}{ + "spec": map[string]interface{}{ + "displayName":displayName, + "contents":openAPIDoc, + }, + }, + } } if graphQLDoc != "" { - payload = "{\"graphqlDocumentation\":{\"endpointUri\":\"" + endpointUri + - "\",\"schema\":{\"displayName\":\"" + displayName + - "\",\"contents\":\"" + graphQLDoc + "\"}}}" + data = map[string]interface{}{ + "graphqlDocumentation": map[string]interface{}{ + "endpointUri":endpointUri, + "schema": map[string]interface{}{ + "displayName":displayName, + "contents":graphQLDoc, + }, + }, + } + } + + jsonData, err := json.Marshal(data) + if err != nil { + fmt.Printf("could not marshal json: %s\n", err) + return } + payload := string(jsonData); u, _ := url.Parse(apiclient.BaseURL) u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apidocs", id, "documentation") - fmt.Printf("Kurt: update %s\n%s\n", u.String(), payload) respBody, err = apiclient.HttpClient(u.String(), payload, "PATCH") return nil, nil