diff --git a/.gitignore b/.gitignore index f78d454de..7a54dd572 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ apiproxy/ #keys for envoy remote-service.* + +#json files +*.json diff --git a/cmd/apidocs/apidocs.go b/cmd/apidocs/apidocs.go index 4e2826b41..11dc44b34 100644 --- a/cmd/apidocs/apidocs.go +++ b/cmd/apidocs/apidocs.go @@ -39,6 +39,8 @@ func init() { Cmd.AddCommand(DocCmd) Cmd.AddCommand(CreateCmd) Cmd.AddCommand(UpdateCmd) + Cmd.AddCommand(ExpCmd) + Cmd.AddCommand(ImpCmd) _ = Cmd.MarkFlagRequired("org") _ = Cmd.MarkFlagRequired("siteid") diff --git a/cmd/apidocs/export.go b/cmd/apidocs/export.go new file mode 100644 index 000000000..60212cfc6 --- /dev/null +++ b/cmd/apidocs/export.go @@ -0,0 +1,52 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apidocs + +import ( + "os" + + "internal/apiclient" + + "internal/client/apidocs" + + "github.com/spf13/cobra" +) + +// ExpCmd to export apidocs +var ExpCmd = &cobra.Command{ + Use: "export", + Short: "Export API Docs to a file", + Long: "Export API Docs to a file", + Args: func(cmd *cobra.Command, args []string) (err error) { + return apiclient.SetApigeeOrg(org) + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + if folder == "" { + folder, _ = os.Getwd() + } + if err = apiclient.FolderExists(folder); err != nil { + return err + } + apiclient.DisableCmdPrintHttpResponse() + return apidocs.Export(folder) + }, +} + +var folder string + +func init() { + ExpCmd.Flags().StringVarP(&folder, "folder", "f", + "", "folder to export API Docs") +} diff --git a/cmd/apidocs/import.go b/cmd/apidocs/import.go new file mode 100644 index 000000000..6e6aab53e --- /dev/null +++ b/cmd/apidocs/import.go @@ -0,0 +1,47 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apidocs + +import ( + "fmt" + "internal/apiclient" + + "internal/client/apidocs" + + "github.com/spf13/cobra" +) + +// ImpCmd to import products +var ImpCmd = &cobra.Command{ + Use: "import", + Short: "Import from a folder containing apidocs", + Long: "Import from a folder containing apidocs", + Args: func(cmd *cobra.Command, args []string) (err error) { + return apiclient.SetApigeeOrg(org) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if siteid == "" { + return fmt.Errorf("siteid is a mandatory parameter") + } + return apidocs.Import(siteid, folder) + }, +} + +func init() { + ImpCmd.Flags().StringVarP(&folder, "folder", "f", + "", "Folder containing site_.json and apidocs__.json files") + + _ = ImpCmd.MarkFlagRequired("file") +} diff --git a/cmd/apidocs/list.go b/cmd/apidocs/list.go index 38a7f5284..2cc187982 100644 --- a/cmd/apidocs/list.go +++ b/cmd/apidocs/list.go @@ -30,7 +30,19 @@ var ListCmd = &cobra.Command{ return apiclient.SetApigeeOrg(org) }, RunE: func(cmd *cobra.Command, args []string) (err error) { - _, err = apidocs.List(siteid) + _, err = apidocs.List(siteid, pageSize, pageToken) return }, } + +var ( + pageSize int + pageToken string +) + +func init() { + ListCmd.Flags().IntVarP(&pageSize, "page-size", "", + -1, "The maximum number of versions to return") + ListCmd.Flags().StringVarP(&pageToken, "page-token", "", + "", "A page token, received from a previous call") +} diff --git a/cmd/org/export.go b/cmd/org/export.go index 450735eb0..2c278ab5d 100644 --- a/cmd/org/export.go +++ b/cmd/org/export.go @@ -26,6 +26,7 @@ import ( "internal/clilog" + "internal/client/apidocs" "internal/client/apis" "internal/client/appgroups" "internal/client/apps" @@ -180,6 +181,11 @@ var ExportCmd = &cobra.Command{ } } + clilog.Info.Println("Exporting API Portal apidocs Configuration...") + if err = apidocs.Export(portalsFolderName); proceedOnError(err) != nil { + return err + } + if runtimeType == "HYBRID" { clilog.Info.Println("Exporting Sync Authorization Identities...") if respBody, err = sync.Get(); err != nil { @@ -307,8 +313,13 @@ func createFolders() (err error) { if err = os.Mkdir(proxiesFolderName, 0o755); err != nil { return err } - err = os.Mkdir(sharedFlowsFolderName, 0o755) - return err + if err = os.Mkdir(sharedFlowsFolderName, 0o755); err != nil { + return err + } + if err = os.Mkdir(portalsFolderName, 0o755); err != nil { + return err + } + return nil } func exportKVMEntries(scope string, env string, listKVMBytes []byte) (err error) { diff --git a/cmd/org/variables.go b/cmd/org/variables.go index dd377ec6c..175fa499a 100644 --- a/cmd/org/variables.go +++ b/cmd/org/variables.go @@ -32,6 +32,7 @@ const ( proxiesFolderName = "proxies" sharedFlowsFolderName = "sharedflows" + portalsFolderName = "portals" ) var conn int diff --git a/cmd/root.go b/cmd/root.go index d1f126c17..38e0b296f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -54,6 +54,7 @@ import ( res "github.com/apigee/apigeecli/cmd/res" "github.com/apigee/apigeecli/cmd/securityprofiles" "github.com/apigee/apigeecli/cmd/sharedflows" + "github.com/apigee/apigeecli/cmd/sites" "github.com/apigee/apigeecli/cmd/sync" targetservers "github.com/apigee/apigeecli/cmd/targetservers" "github.com/apigee/apigeecli/cmd/token" @@ -180,6 +181,7 @@ func init() { RootCmd.AddCommand(apicategories.Cmd) RootCmd.AddCommand(datastores.Cmd) RootCmd.AddCommand(securityprofiles.Cmd) + RootCmd.AddCommand(sites.Cmd) } func initConfig() { diff --git a/cmd/sites/list.go b/cmd/sites/list.go new file mode 100644 index 000000000..7ea7b1924 --- /dev/null +++ b/cmd/sites/list.go @@ -0,0 +1,36 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sites + +import ( + "internal/apiclient" + "internal/client/sites" + + "github.com/spf13/cobra" +) + +// ListCmd to list catalog items +var ListCmd = &cobra.Command{ + Use: "list", + Short: "Returns the API Portals associated with the org", + Long: "Returns the API Portals associated with the org", + Args: func(cmd *cobra.Command, args []string) (err error) { + return apiclient.SetApigeeOrg(org) + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + _, err = sites.List() + return + }, +} diff --git a/cmd/sites/sites.go b/cmd/sites/sites.go new file mode 100644 index 000000000..2774ffd9a --- /dev/null +++ b/cmd/sites/sites.go @@ -0,0 +1,37 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sites + +import ( + "github.com/spf13/cobra" +) + +// Cmd to manage +var Cmd = &cobra.Command{ + Use: "sites", + Short: "Manage Apigee API Portals", + Long: "Manage Apigee API Portals", +} + +var org string + +func init() { + Cmd.PersistentFlags().StringVarP(&org, "org", "o", + "", "Apigee organization name") + + Cmd.AddCommand(ListCmd) + + _ = Cmd.MarkFlagRequired("org") +} diff --git a/internal/client/apidocs/apidocs.go b/internal/client/apidocs/apidocs.go index affc67ff3..f59e26366 100644 --- a/internal/client/apidocs/apidocs.go +++ b/internal/client/apidocs/apidocs.go @@ -15,11 +15,18 @@ package apidocs import ( + "encoding/json" + "errors" + "fmt" + "io" "net/url" + "os" "path" + "strconv" "strings" "internal/apiclient" + "internal/client/sites" ) type Action uint8 @@ -29,6 +36,72 @@ const ( UPDATE ) +const maxPageSize = 100 + +type listapidocs struct { + Status string `json:"status,omitempty"` + Message string `json:"message,omitempty"` + RequestID string `json:"requestId,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + Data []data `json:"data,omitempty"` + NextPageToken string `json:"nextPageToken,omitempty"` +} + +type apidocsdata struct { + Status string `json:"status,omitempty"` + Message string `json:"message,omitempty"` + RequestID string `json:"requestId,omitempty"` + Data documentation `json:"data,omitempty"` +} + +type documentation struct { + OasDocumentation *oasDocumentation `json:"oasDocumentation,omitempty"` + GraphqlDocumentation *graphqlDocumentation `json:"graphqlDocumentation,omitempty"` +} + +type oasDocumentation struct { + Spec spec `json:"spec,omitempty"` + Format string `json:"format,omitempty"` +} + +type spec struct { + DisplayName string `json:"displayName,omitempty"` + Contents string `json:"contents,omitempty"` +} + +type graphqlDocumentation struct { + Schema spec `json:"schema,omitempty"` + EndpointUri string `json:"endpointUri,omitempty"` +} + +type data struct { + SiteID string `json:"siteId,omitempty"` + ID string `json:"id,omitempty"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + Published bool `json:"published,omitempty"` + AnonAllowed bool `json:"anonAllowed,omitempty"` + ApiProductName string `json:"apiProductName,omitempty"` + RequireCallbackUrl bool `json:"requireCallbackUrl,omitempty"` + ImageUrl string `json:"imageUrl,omitempty"` + CategoryIDs []string `json:"categoryIds,omitempty"` + Modified string `json:"modified,omitempty"` + Visibility bool `json:"visibility,omitempty"` + EdgeAPIProductName string `json:"edgeAPIProductName,omitempty"` + SpecID string `json:"specId,omitempty"` + GraphqlSchema string `json:"graphqlSchema,omitempty"` + GraphqlEndpointUrl string `json:"graphqlEndpointUrl,omitempty"` + GraphqlSchemaDisplayName string `json:"graphqlSchemaDisplayName,omitempty"` +} + +type apidocResponse struct { + Status string `json:"status,omitempty"` + Message string `json:"message,omitempty"` + RequestID string `json:"requestId,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + Data data `json:"data,omitempty"` +} + // Create func Create(siteid string, title string, description string, published string, anonAllowed string, apiProductName string, requireCallbackUrl string, imageUrl string, @@ -113,9 +186,18 @@ func Get(siteid string, id string) (respBody []byte, err error) { } // List -func List(siteid string) (respBody []byte, err error) { +func List(siteid string, pageSize int, pageToken string) (respBody []byte, err error) { u, _ := url.Parse(apiclient.BaseURL) u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apidocs") + q := u.Query() + if pageSize != -1 { + q.Set("pageSize", strconv.Itoa(pageSize)) + } + if pageToken != "" { + q.Set("pageToken", pageToken) + } + + u.RawQuery = q.Encode() respBody, err = apiclient.HttpClient(u.String()) return respBody, err } @@ -129,19 +211,41 @@ func Delete(siteid string, id string) (respBody []byte, err error) { } // UpdateDocumentation -func UpdateDocumentation(siteid string, id string, displayName string, openAPIDoc string, graphQLDoc string, endpointUri string) (respBody []byte, err error) { - var payload string +func UpdateDocumentation(siteid string, id string, displayName string, + openAPIDoc string, graphQLDoc string, endpointUri string, +) (respBody []byte, err error) { + var data map[string]interface{} + // var payload string if openAPIDoc != "" { - payload = "{\"oasDocumentation\":{\"spec\":{\"displayName\":\"" + - displayName + "\",\"contents\":" + openAPIDoc + "}}}" + data = map[string]interface{}{ + "oasDocumentation": map[string]interface{}{ + "spec": map[string]interface{}{ + "displayName":displayName, + "contents":openAPIDoc, + }, + }, + } } if graphQLDoc != "" { - payload = "{\"graphqlDocumentation\":{\"endpointUri\":\"" + endpointUri + - "\",\"schema\":{\"displayName\":\"" + displayName + - "\",\"contents\":" + graphQLDoc + "}}}" + data = map[string]interface{}{ + "graphqlDocumentation": map[string]interface{}{ + "endpointUri":endpointUri, + "schema": map[string]interface{}{ + "displayName":displayName, + "contents":graphQLDoc, + }, + }, + } + } + + jsonData, err := json.Marshal(data) + if err != nil { + fmt.Printf("could not marshal json: %s\n", err) + return } + payload := string(jsonData); u, _ := url.Parse(apiclient.BaseURL) u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apidocs", id, "documentation") @@ -150,8 +254,151 @@ func UpdateDocumentation(siteid string, id string, displayName string, openAPIDo return nil, nil } +// Export +func Export(folder string) (err error) { + apiclient.ClientPrintHttpResponse.Set(false) + defer apiclient.ClientPrintHttpResponse.Set(apiclient.GetCmdPrintHttpResponseSetting()) + + siteids, err := sites.GetSiteIDs() + if err != nil { + return err + } + + pageToken := "" + + for _, siteid := range siteids { + l := listapidocs{} + for { + listRespBytes, err := List(siteid, maxPageSize, pageToken) + if err != nil { + return fmt.Errorf("failed to fetch apidocs: %w", err) + } + err = json.Unmarshal(listRespBytes, &l) + if err != nil { + return fmt.Errorf("failed to unmarshall: %w", err) + } + pageToken = l.NextPageToken + // write apidocs Documentation + for _, data := range l.Data { + respDocsBody, err := GetDocumentation(siteid, data.ID) + if err != nil { + return err + } + docFileName := fmt.Sprintf("apidocs_%s_%s.json", siteid, data.ID) + if err = apiclient.WriteByteArrayToFile(path.Join(folder, docFileName), false, respDocsBody); err != nil { + return err + } + } + if l.NextPageToken == "" { + break + } + } + respBody, err := json.Marshal(l.Data) + if err != nil { + return err + } + respBody, _ = apiclient.PrettifyJSON(respBody) + if err = apiclient.WriteByteArrayToFile(path.Join(folder, "site_"+siteid+".json"), false, respBody); err != nil { + return err + } + } + return nil +} + +func Import(siteid string, folder string) (err error) { + var errs []string + var respBody []byte + docsList, err := readAPIDocsDataFile(path.Join(folder, "site_"+siteid+".json")) + if err != nil { + return err + } + for _, doc := range docsList { + // 1. create the apidoc object + respBody, err = Create(siteid, doc.Title, doc.Description, strconv.FormatBool(doc.Published), + strconv.FormatBool(doc.AnonAllowed), doc.ApiProductName, + strconv.FormatBool(doc.RequireCallbackUrl), doc.ImageUrl, doc.CategoryIDs) + if err != nil { + errs = append(errs, err.Error()) + continue + + } + + // get the new doc.ID from the created apidoc + response := apidocResponse{} + err = json.Unmarshal(respBody, &response) + if err != nil { + return err + } + + // 2. find the documentation associated with this site + documentationFileName := path.Join(folder, "apidocs_"+siteid+"_"+doc.ID+".json") + apidocument, err := readAPIDocumentationFile(documentationFileName) + if err != nil { + errs = append(errs, err.Error()) + continue + } + if apidocument.Data.GraphqlDocumentation != nil { + _, err = UpdateDocumentation(siteid, response.Data.ID, + apidocument.Data.GraphqlDocumentation.Schema.DisplayName, "", + apidocument.Data.GraphqlDocumentation.Schema.Contents, + apidocument.Data.GraphqlDocumentation.EndpointUri) + } else { + _, err = UpdateDocumentation(siteid, response.Data.ID, + apidocument.Data.OasDocumentation.Spec.DisplayName, + apidocument.Data.OasDocumentation.Spec.Contents, + "", "") + } + if err != nil { + errs = append(errs, err.Error()) + continue + } + } + if len(errs) > 0 { + return errors.New(strings.Join(errs, "\n")) + } + return nil +} + func getArrayStr(str []string) string { tmp := strings.Join(str, ",") tmp = strings.ReplaceAll(tmp, ",", "\",\"") return tmp } + +func readAPIDocumentationFile(fileName string) (a apidocsdata, err error) { + jsonFile, err := os.Open(fileName) + if err != nil { + return a, err + } + + defer jsonFile.Close() + + content, err := io.ReadAll(jsonFile) + if err != nil { + return a, err + } + err = json.Unmarshal(content, &a) + if err != nil { + return a, err + } + return a, nil +} + +func readAPIDocsDataFile(fileName string) (d []data, err error) { + jsonFile, err := os.Open(fileName) + if err != nil { + return d, err + } + + defer jsonFile.Close() + + content, err := io.ReadAll(jsonFile) + if err != nil { + return d, err + } + err = json.Unmarshal(content, &d) + if err != nil { + return d, err + } + return d, nil +} diff --git a/internal/client/securityprofiles/securityprofiles.go b/internal/client/securityprofiles/securityprofiles.go index 6f080ea45..f75ae15a1 100644 --- a/internal/client/securityprofiles/securityprofiles.go +++ b/internal/client/securityprofiles/securityprofiles.go @@ -179,6 +179,7 @@ func Export(conn int, folder string, allRevisions bool) (err error) { return fmt.Errorf("failed to unmarshall: %w", err) } listsecprofiles.SecurityProfiles = append(listsecprofiles.SecurityProfiles, l.SecurityProfiles...) + pageToken = l.NextPageToken if l.NextPageToken == "" { break } diff --git a/internal/client/sites/sites.go b/internal/client/sites/sites.go new file mode 100644 index 000000000..07a158d79 --- /dev/null +++ b/internal/client/sites/sites.go @@ -0,0 +1,49 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sites + +import ( + "net/url" + "path" + + "internal/apiclient" + + "github.com/thedevsaddam/gojsonq" +) + +// List +func List() (respBody []byte, err error) { + u, _ := url.Parse(apiclient.BaseURL) + u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites") + respBody, err = apiclient.HttpClient(u.String()) + return respBody, err +} + +// GetSiteIDs +func GetSiteIDs() (siteIDs []string, err error) { + apiclient.ClientPrintHttpResponse.Set(false) + defer apiclient.ClientPrintHttpResponse.Set(apiclient.GetCmdPrintHttpResponseSetting()) + respBody, err := List() + if err != nil { + return nil, err + } + jq := gojsonq.New().JSONString(string(respBody)) + ids := jq.From("data").Pluck("id").([]interface{}) + siteIDs = make([]string, len(ids)) + for k, v := range ids { + siteIDs[k] = v.(string) + } + return siteIDs, nil +}