From da8842641e32ebd0896e6bbeaff7710f1ae1fb0b Mon Sep 17 00:00:00 2001 From: anaik91 Date: Tue, 22 Aug 2023 23:09:42 +0530 Subject: [PATCH 01/14] feat: Added wrappers for target-server-validator --- CODEOWNERS | 1 + README.md | 2 + tools/target-server-validator/README.md | 90 +++++++ tools/target-server-validator/apigee.py | 239 ++++++++++++++++++ .../apiproxy/policies/JC1.xml | 19 ++ .../apiproxy/policies/set-json-response.xml | 23 ++ .../apiproxy/proxies/default.xml | 36 +++ .../apiproxy/target_server_validator.xml | 30 +++ .../callout/build_setup.sh | 48 ++++ tools/target-server-validator/callout/pom.xml | 159 ++++++++++++ .../callout/src/main/java/PortOpenCheck.java | 71 ++++++ .../target-server-validator/input.properties | 24 ++ tools/target-server-validator/main.py | 234 +++++++++++++++++ tools/target-server-validator/pipeline.sh | 76 ++++++ tools/target-server-validator/report.md | 8 + .../target-server-validator/requirements.txt | 20 ++ tools/target-server-validator/utils.py | 214 ++++++++++++++++ 17 files changed, 1294 insertions(+) create mode 100644 tools/target-server-validator/README.md create mode 100644 tools/target-server-validator/apigee.py create mode 100644 tools/target-server-validator/apiproxy/policies/JC1.xml create mode 100644 tools/target-server-validator/apiproxy/policies/set-json-response.xml create mode 100644 tools/target-server-validator/apiproxy/proxies/default.xml create mode 100644 tools/target-server-validator/apiproxy/target_server_validator.xml create mode 100644 tools/target-server-validator/callout/build_setup.sh create mode 100644 tools/target-server-validator/callout/pom.xml create mode 100644 tools/target-server-validator/callout/src/main/java/PortOpenCheck.java create mode 100644 tools/target-server-validator/input.properties create mode 100644 tools/target-server-validator/main.py create mode 100755 tools/target-server-validator/pipeline.sh create mode 100644 tools/target-server-validator/report.md create mode 100644 tools/target-server-validator/requirements.txt create mode 100644 tools/target-server-validator/utils.py diff --git a/CODEOWNERS b/CODEOWNERS index 8e5385587..ef3d3cc1b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -38,3 +38,4 @@ /tools/pipeline-runner @seymen @danistrebel /tools/sf-dependency-list @yuriylesyuk /tools/proxy-endpoint-unifier @anaik91 +tools/target-server-validator @anaik91 diff --git a/README.md b/README.md index 4c707765c..ee662dee6 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,8 @@ Apigee products. A tool to set up the sample deployments of Apigee Envoy. - [Apigee API Proxy Endpoint Unifier](tools/proxy-endpoint-unifier) - A tool to unify/split proxy endpoints based on API basepath. +- [Apigee Target Server Validator](tools/target-server-validator) - + A tool to validate all targets in Target Servers & Apigee API Proxy Bundles. ## Labs diff --git a/tools/target-server-validator/README.md b/tools/target-server-validator/README.md new file mode 100644 index 000000000..a6bb541ca --- /dev/null +++ b/tools/target-server-validator/README.md @@ -0,0 +1,90 @@ +# Apigee Target Server Validator + +The objective of this tool to validate targets in Target Servers & Apigee API Proxy Bundles exported from Apigee OPDK/X/Hybrid. +Validation is done by deploying a sample proxy which check if HOST & PORT is open from Apigee OPDK/X/Hybrid. + +> **NOTE**: Discovery of Targets in API Proxy & Sharedflows is limited to only parsing URL from `TargetEndpoint` & `ServiceCallout` Policy. + +> **NOTE**: Dynamic targets are **NOT** supported, Ex : `https://host.{request.formparam.region}.example.com}` + +## Disclaimer +This is not an Officially Supported Google Product! + +## Pre-Requisites +* python3.x +* Please Install required Python Libs + +``` + python3 -m pip install requirements.txt +``` +* Please fill in `input.properties` + +``` +[source] +baseurl=http://34.131.144.184:8080/v1 # Apigee OPDK/Edge/X/Hybrid Base URL +org=xxx-xxxx-xxx-xxxxx # Apigee OPDK/Edge/X/Hybrid Org +auth_type=basic # API Auth type basic | oauth + +[target] +baseurl=https://apigee.googleapis.com/v1 # Apigee OPDK/Edge/X/Hybrid Base URL +org=xxx-xxxx-xxx-xxxxx # Apigee OPDK/Edge/X/Hybrid Org Id +auth_type=oauth # API Auth type basic | oauth + +[csv] +file=input.csv # Path to input CSV. Note: CSV needs HOST & PORT columns +default_port=443 # default port if port is not provided in CSV + +[validation] +check_csv=true # 'true' to validate Targets in input csv +check_proxies=true # 'true' to validate Proxy Targets else 'false' +skip_proxy_list=mock1,stream # Comma sperated list of proxies to skip validation; +proxy_export_dir=export # Export directory needed when check_proxies='true' +api_env=dev # Target Environment to deploy Validation API Proxy +api_name=target_server_validator # Target API Name of Validation API Proxy +vhost_domain_name=devgroup # Target VHost or EnvGroup +vhost_ip= # IP address corresponding to vhost_domain_name. Use if DNS record doesnt exist +report_format=csv # Report Format. Choose csv or md (Markdown) +``` + +* Sample input CSV with target servers +> **NOTE:** You need to set `check_csv=true` in the `validation` section of `input.properties` + +> **NOTE:** You need to set `file=` in the `csv` section of `input.properties` + +``` +HOST,PORT +httpbin.org +mocktarget.apigee.net,80 +smtp.gmail.com,465 +``` + + +* Please run below command to authenticate against Apigee X/Hybrid APIS + +``` + export APIGEE_OPDK_ACCESS_TOKEN=$(echo -n ":" | base64) # Access token for Apigee OPDK + export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token) # Access token for Apigee X +``` + +## Highlevel Working +* Export Target Server Details +* Export Proxy Bundle +* Parse Each Proxy Bundle for Target +* Run Validate API against each Target +* Generate CSV Report + +## Usage + +Run the Script as below +``` +python3 main.py +``` + +## Report +Validation Report : `report.md` OR `report.csv` can be accessed in same localtion as script. + +Please check a [Sample report](report.md) + +## Copyright + +Copyright 2023 Google LLC. This software is provided as-is, without warranty or representation for any use or purpose. Your use of it is subject to your agreement with Google. diff --git a/tools/target-server-validator/apigee.py b/tools/target-server-validator/apigee.py new file mode 100644 index 000000000..727b61d4f --- /dev/null +++ b/tools/target-server-validator/apigee.py @@ -0,0 +1,239 @@ +#!/usr/bin/python + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import requests +import os +import sys +import shutil +from time import sleep + + +class Apigee: + + def __init__( + self, + apigee_type="x", + base_url="https://apigee.googleapis.com/v1", + auth_type="oauth", + org="validate"): + self.org = org + self.baseurl = f"{base_url}/organizations/{org}" + self.apigee_type = apigee_type + self.auth_type = auth_type + access_token = self.get_access_token() + self.auth_header = { + 'Authorization': 'Bearer {}'.format(access_token) if self.auth_type == 'oauth' else 'Basic {}'.format(access_token) # noqa + } + + def is_token_valid(self, token): + url = f"https://www.googleapis.com/oauth2/v1/tokeninfo?access_token={token}" # noqa + r = requests.get(url) + if r.status_code == 200: + print(f"Token Validated for user {r.json()['email']}") + return True + return False + + def get_access_token(self): + token = os.getenv('APIGEE_ACCESS_TOKEN' if self.apigee_type == 'x' else 'APIGEE_OPDK_ACCESS_TOKEN') # noqa + if token is not None: + if self.apigee_type == 'x': + if self.is_token_valid(token): + return token + else: + print('please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ') # noqa + sys.exit(1) + else: + return token + else: + if self.apigee_type == 'x': + print('please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ') # noqa + else: + print('please export APIGEE_OPDK_ACCESS_TOKEN') + sys.exit(1) + + def set_auth_header(self): + access_token = self.get_access_token() + self.auth_header = { + 'Authorization': 'Bearer {}'.format(access_token) if self.auth_type == 'oauth' else 'Basic {}'.format(access_token) # noqa + } + + def list_environments(self): + url = f"{self.baseurl}/environments" + headers = self.auth_header.copy() + response = requests.request("GET", url, headers=headers) + if response.status_code == 200: + return response.json() + else: + return [] + + def list_target_servers(self, env): + url = f"{self.baseurl}/environments/{env}/targetservers" + headers = self.auth_header.copy() + response = requests.request("GET", url, headers=headers) + if response.status_code == 200: + return response.json() + else: + return [] + + def get_target_server(self, env, target_server): + url = f"{self.baseurl}/environments/{env}/targetservers/{target_server}" # noqa + headers = self.auth_header.copy() + response = requests.request("GET", url, headers=headers) + if response.status_code == 200: + return response.json() + else: + return [] + + def get_api(self, api_name): + url = f"{self.baseurl}/apis/{api_name}" + headers = self.auth_header.copy() + response = requests.request("GET", url, headers=headers) + if response.status_code == 200: + return True + else: + return False + + def create_api(self, api_name, proxy_bundle_path): + url = f"{self.baseurl}/apis?action=import&name={api_name}&validate=true" # noqa + proxy_bundle_name = os.path.basename(proxy_bundle_path) + files = [ + ('data',(proxy_bundle_name,open(proxy_bundle_path,'rb'),'application/zip')) # noqa + ] + headers = self.auth_header.copy() + response = requests.request("POST", url, headers=headers, data={}, files=files) # noqa + if response.status_code == 200: + return True + else: + print(response.json()) + return False + + def get_api_revisions_deployment(self, env, api_name, api_rev): # noqa + url = url = f"{self.baseurl}/environments/{env}/apis/{api_name}/revisions/{api_rev}/deployments" # noqa + headers = self.auth_header.copy() + response = requests.request("GET", url, headers=headers, data={}) + if response.status_code == 200: + resp = response.json() + api_deployment_status = resp.get('state', '') + if self.apigee_type == 'x': + if api_deployment_status == 'READY': + return True + if self.apigee_type == 'opdk': + if api_deployment_status == 'deployed': + return True + print(f"API {api_name} is in Status: {api_deployment_status} !") # noqa + return False + else: + return False + + def deploy_api(self, env, api_name, api_rev): + url = url = f"{self.baseurl}/environments/{env}/apis/{api_name}/revisions/{api_rev}/deployments?override=true" # noqa + headers = self.auth_header.copy() + response = requests.request("POST", url, headers=headers, data={}) + if response.status_code == 200: + return True + else: + resp = response.json() + if 'already deployed' in resp['error']['message']: + print('Proxy {} is already Deployed'.format(api_name)) + return True + return False + + def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_rev=1): # noqa + api_deployment_retry = 60 + api_deployment_sleep = 5 + api_deployment_retry_count = 0 + api_exists = False + if self.get_api(api_name): + print(f'Proxy with name {api_name} already exists in Apigee Org {self.org}') # noqa + api_exists = True + else: + if self.create_api(api_name, proxy_bundle_path): + print(f'Proxy has been imported with name {api_name} in Apigee Org {self.org}') # noqa + api_exists = True + else: + print(f'ERROR : Proxy {api_name} import failed !!! ') + return False + if api_exists: + if self.deploy_api(env, api_name, api_rev): + print(f'Proxy with name {api_name} has been deployed to {env} in Apigee Org {self.org}') # noqa + while api_deployment_retry_count < api_deployment_retry: + if self.get_api_revisions_deployment(env, api_name, api_rev): # noqa + print(f'Proxy {api_name} active in runtime after {api_deployment_retry_count*api_deployment_sleep} seconds ') # noqa + return True + else: + print(f"Checking API deployment status in {api_deployment_sleep} seconds") # noqa + sleep(api_deployment_sleep) + api_deployment_retry_count += 1 + else: + print(f'ERROR : Proxy deployment to {env} in Apigee Org {self.org} Failed !!') # noqa + return False + + def get_api_vhost(self, vhost_name, env): + if self.apigee_type == 'opdk': + url = f"{self.baseurl}/environments/{env}/virtualhosts/{vhost_name}" # noqa + else: + url = f"{self.baseurl}/envgroups/{vhost_name}" + headers = self.auth_header.copy() + response = requests.request("GET", url, headers=headers) + if response.status_code == 200: + if self.apigee_type == 'opdk': + hosts = response.json()['hostAliases'] + else: + hosts = response.json()['hostnames'] + if len(hosts) == 0: + print(f'ERROR: Vhost/Env Group {vhost_name} contains no domains') # noqa + return None + return hosts + else: + print(f'ERROR: Vhost/Env Group {vhost_name} contains no domains') # noqa + return None + + def list_apis(self, api_type): + url = f"{self.baseurl}/{api_type}" + headers = self.auth_header.copy() + r = requests.get(url, headers=headers) + if r.status_code == 200: + if self.apigee_type == 'x': + if len(r.json()) == 0: + return [] + return [ p['name'] for p in r.json()['proxies' if api_type == 'apis' else 'sharedFlows']] # noqa + return r.json() + else: + return [] + + def list_api_revisions(self, api_type, api_name): + url = f"{self.baseurl}/{api_type}/{api_name}/revisions" + headers = self.auth_header.copy() + r = requests.get(url, headers=headers) + if r.status_code == 200: + return r.json() + else: + return [] + + def fetch_api_revision(self, api_type, api_name, revision, export_dir): # noqa + url = f"{self.baseurl}/{api_type}/{api_name}/revisions/{revision}?format=bundle" # noqa + headers = self.auth_header.copy() + r = requests.get(url, headers=headers, stream=True) + if r.status_code == 200: + self.write_proxy_bundle(export_dir, api_name, r.raw) + return True + return False + + def write_proxy_bundle(self, export_dir, file_name, data): + file_path = f"./{export_dir}/{file_name}.zip" + with open(file_path, 'wb') as fl: + shutil.copyfileobj(data, fl) diff --git a/tools/target-server-validator/apiproxy/policies/JC1.xml b/tools/target-server-validator/apiproxy/policies/JC1.xml new file mode 100644 index 000000000..d3b8e5544 --- /dev/null +++ b/tools/target-server-validator/apiproxy/policies/JC1.xml @@ -0,0 +1,19 @@ + + + + JC1 + + com.apigeesample.PortOpenCheck + java://edge-custom-policy-java-hello.jar + \ No newline at end of file diff --git a/tools/target-server-validator/apiproxy/policies/set-json-response.xml b/tools/target-server-validator/apiproxy/policies/set-json-response.xml new file mode 100644 index 000000000..7a7da0e47 --- /dev/null +++ b/tools/target-server-validator/apiproxy/policies/set-json-response.xml @@ -0,0 +1,23 @@ + + + + + { + "host":"{request.header.host_name}", + "port": "{request.header.port_number}", + "status":"{REACHABLE_STATUS}" +} + + + \ No newline at end of file diff --git a/tools/target-server-validator/apiproxy/proxies/default.xml b/tools/target-server-validator/apiproxy/proxies/default.xml new file mode 100644 index 000000000..539ee0d85 --- /dev/null +++ b/tools/target-server-validator/apiproxy/proxies/default.xml @@ -0,0 +1,36 @@ + + + + + + + JC1 + + + + + set-json-response + + + + + + + + + + /validate_target_server + + + \ No newline at end of file diff --git a/tools/target-server-validator/apiproxy/target_server_validator.xml b/tools/target-server-validator/apiproxy/target_server_validator.xml new file mode 100644 index 000000000..ff411df92 --- /dev/null +++ b/tools/target-server-validator/apiproxy/target_server_validator.xml @@ -0,0 +1,30 @@ + + + + + + 1682421435781 + 1682421435781 + /validate_target_server + + JC1 + set-json-response + + + default + + + java://edge-custom-policy-java-hello.jar + + \ No newline at end of file diff --git a/tools/target-server-validator/callout/build_setup.sh b/tools/target-server-validator/callout/build_setup.sh new file mode 100644 index 000000000..9684fef6f --- /dev/null +++ b/tools/target-server-validator/callout/build_setup.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +echo +echo "This script downloads JAR files and installs them into the local Maven repo." +echo + +curl -O https://raw.githubusercontent.com/apigee/api-platform-samples/master/doc-samples/java-cookbook/lib/expressions-1.0.0.jar + + mvn install:install-file \ + -Dfile=expressions-1.0.0.jar \ + -DgroupId=com.apigee.edge \ + -DartifactId=expressions \ + -Dversion=1.0.0 \ + -Dpackaging=jar \ + -DgeneratePom=true + +rm expressions-1.0.0.jar + +curl -O https://raw.githubusercontent.com/apigee/api-platform-samples/master/doc-samples/java-cookbook/lib/message-flow-1.0.0.jar + + mvn install:install-file \ + -Dfile=message-flow-1.0.0.jar \ + -DgroupId=com.apigee.edge \ + -DartifactId=message-flow \ + -Dversion=1.0.0 \ + -Dpackaging=jar \ + -DgeneratePom=true + +rm message-flow-1.0.0.jar + +echo +echo done. +echo \ No newline at end of file diff --git a/tools/target-server-validator/callout/pom.xml b/tools/target-server-validator/callout/pom.xml new file mode 100644 index 000000000..797024c9a --- /dev/null +++ b/tools/target-server-validator/callout/pom.xml @@ -0,0 +1,159 @@ + + + + 4.0.0 + com.apigee.callout + edge-custom-policy-java-hello + 1.0-SNAPSHOT + EdgeCustomJavaHello + http://maven.apache.org + jar + + UTF-8 + UTF-8 + 1.7 + ../apiproxy/resources/java + 6.8.7 + 1.7 + 1.6.1 + + + + + + + + com.apigee.edge + message-flow + 1.0.0 + + + com.apigee.edge + expressions + 1.0.0 + + + + + + ${project.artifactId} + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + prepare-package + + copy-dependencies + + + ${project.build.directory}/lib + false + false + true + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + ${java.version} + ${java.version} + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.6 + + + package + + jar + + + + + + + + false + + + + + + + maven-antrun-plugin + + + package + + + + + + + + + + + + + + + run + + + + + + + + diff --git a/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java b/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java new file mode 100644 index 000000000..7271de7ac --- /dev/null +++ b/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java @@ -0,0 +1,71 @@ +// Copyright 2023 Google LLC + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package com.apigeesample; + +import com.apigee.flow.execution.ExecutionContext; +import com.apigee.flow.execution.ExecutionResult; +import com.apigee.flow.execution.spi.Execution; +import com.apigee.flow.message.MessageContext; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; + + +public class PortOpenCheck implements Execution { + + private static String available(String host,int port) { + Socket socket = new Socket(); + try { + socket.connect(new InetSocketAddress(host, port), 1000); + return "REACHABLE"; + } catch (SocketTimeoutException e) { + return "NOT_REACHABLE"; + } catch (UnknownHostException e) { + return "UNKNOWN_HOST"; + } + catch (IOException e) { + return "NOT_REACHABLE"; + } finally { + if (socket != null) { + try { + socket.close(); + } catch (IOException e) { + throw new RuntimeException("You should handle this error.", e); + } + } + } + } + + public ExecutionResult execute(MessageContext messageContext, ExecutionContext executionContext) { + + try { + String host_name = messageContext.getMessage().getHeader("host_name"); + String port = messageContext.getMessage().getHeader("port_number"); + int port_number = Integer.parseInt(port); + String Status = available(host_name,port_number); + // messageContext.getMessage().setContent(Status); + messageContext.setVariable("REACHABLE_STATUS", Status); + return ExecutionResult.SUCCESS; + + } catch (Exception e) { + return ExecutionResult.ABORT; + } + } + +} diff --git a/tools/target-server-validator/input.properties b/tools/target-server-validator/input.properties new file mode 100644 index 000000000..0570fb794 --- /dev/null +++ b/tools/target-server-validator/input.properties @@ -0,0 +1,24 @@ +[source] +baseurl=https://apigee.googleapis.com/v1 +org=apigee-hybrid-378710 +auth_type=oauth + +[target] +baseurl=https://apigee.googleapis.com/v1 +org=apigee-hybrid-378710 +auth_type=oauth + +[csv] +file=input.csv +default_port=443 + +[validation] +check_csv=true +check_proxies=true +proxy_export_dir=export +skip_proxy_list=mock1,stream +api_env=dev +api_name=target_server_validator +vhost_domain_name=example.apigee.com +vhost_ip=34.134.171.41 +report_format=md diff --git a/tools/target-server-validator/main.py b/tools/target-server-validator/main.py new file mode 100644 index 000000000..2c145c603 --- /dev/null +++ b/tools/target-server-validator/main.py @@ -0,0 +1,234 @@ +#!/usr/bin/python + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import sys +from utils import (parse_config, + create_proxy_bundle, + run_validator_proxy, + delete_file, + read_csv, + write_csv_report, + write_md_report, + create_dir, + unzip_file, + parse_proxy_hosts, + has_templating, + get_tes, + get_row_host_port) +from apigee import Apigee + + +def main(): + # Parse Inputs + cfg = parse_config('input.properties') + check_proxies = cfg['validation'].getboolean('check_proxies') + proxy_export_dir = cfg['validation']['proxy_export_dir'] + report_format = cfg['validation']['report_format'] + if report_format not in ['csv', 'md']: + report_format = 'md' + + # Intialize Source & Target Apigee + SourceApigee = Apigee( + 'x' if 'apigee.googleapis.com' in cfg['source']['baseurl'] else 'opdk', + cfg['source']['baseurl'], + cfg['source']['auth_type'], + cfg['source']['org'] + ) + + TargetApigee = Apigee( + 'x' if 'apigee.googleapis.com' in cfg['source']['baseurl'] else 'opdk', + cfg['target']['baseurl'], + cfg['target']['auth_type'], + cfg['target']['org'] + ) + + environments = SourceApigee.list_environments() + all_target_servers = [] + # Fetch Target Servers from Source Apigee@ + print('INFO: exporting Target Servers !') + for each_env in environments: + target_servers = SourceApigee.list_target_servers(each_env) + for each_ts in target_servers: + ts_info = SourceApigee.get_target_server(each_env, each_ts) + ts_info['env'] = each_env + all_target_servers.append(ts_info) + + # Fetch Targets in APIs & Shared Flows from Source Apigee + proxy_hosts = {} + proxy_targets = {} + if check_proxies: + skip_proxy_list = cfg['validation'].get('skip_proxy_list', '').split(',') # noqa + print('INFO: exporting proxies to be analyzed ! this may take a while !') # noqa + api_types = ['apis', 'sharedflows'] + api_revision_map = {} + for each_api_type in api_types: + api_revision_map[each_api_type] = {} + api_revision_map[each_api_type]['proxies'] = {} + api_revision_map[each_api_type]['export_dir'] = proxy_export_dir+f'/{each_api_type}' # noqa + create_dir(proxy_export_dir+f'/{each_api_type}') + + for each_api in SourceApigee.list_apis(each_api_type): + if each_api not in skip_proxy_list: + api_revision_map[each_api_type]['proxies'][each_api] = SourceApigee.list_api_revisions(each_api_type,each_api)[-1] # noqa + else: + print(f"INFO : Skipping API {each_api}") + for each_api_type, each_api_type_data in api_revision_map.items(): + proxy_hosts[each_api_type] = {} + for each_api, each_api_rev in each_api_type_data['proxies'].items(): # noqa + print(f"Exporting API : {each_api} with revision : {each_api_rev} ") # noqa + SourceApigee.fetch_api_revision(each_api_type, + each_api, + each_api_rev, + api_revision_map[each_api_type]['export_dir']) # noqa + print(f"Unzipping API : {each_api} with revision : {each_api_rev} ") # noqa + unzip_file( + f"{api_revision_map[each_api_type]['export_dir']}/{each_api}.zip", # noqa + f"{api_revision_map[each_api_type]['export_dir']}/{each_api}" # noqa + ) + parsed_proxy_hosts = parse_proxy_hosts( + f"{api_revision_map[each_api_type]['export_dir']}/{each_api}/apiproxy" # noqa + ) + proxy_hosts[each_api_type][each_api] = parsed_proxy_hosts + proxy_tes = get_tes(parsed_proxy_hosts) + for each_te in proxy_tes: + if each_te in proxy_targets: + proxy_targets[each_te].append(f"{each_api_type} - {each_api}") # noqa + else: + proxy_targets[each_te] = [f"{each_api_type} - {each_api}"] # noqa + # Validate Targets against Target Apigee + + bundle_path = os.path.dirname(os.path.abspath(__file__)) + + # Create Validation Proxy Bundle + print('INFO: Creating proxy bundle !') + create_proxy_bundle( + bundle_path, + cfg['validation']['api_name'], + 'apiproxy' + ) + + # Deploy Validation Proxy Bundle + print('INFO: Deploying proxy bundle !') + if not TargetApigee.deploy_api_bundle( + cfg['validation']['api_env'], + cfg['validation']['api_name'], + f"{bundle_path}/{cfg['validation']['api_name']}.zip" + ): + print(f"Proxy: {cfg['validation']['api_name']} deployment failed.") + sys.exit(1) + # CleanUp Validation Proxy Bundle + print('INFO: Cleaning Up local proxy bundle !') + delete_file(f"{bundle_path}/{cfg['validation']['api_name']}.zip") + + # Fetch API Northbound Endpoint + print(f"INFO: Fetching VHost with name {cfg['validation']['vhost_domain_name']} !") # noqa + vhost_domain_name = cfg['validation']['vhost_domain_name'] + vhost_ip = cfg['validation'].get('vhost_ip', '').strip() + api_url = f"https://{vhost_domain_name}/validate_target_server" + final_report = [] + _cached_hosts = {} + + # Run Target Server Validation + print('INFO: Running validation against All Target Servers') + for each_ts in all_target_servers: + status = run_validator_proxy( + api_url, + vhost_domain_name, + vhost_ip, + each_ts['host'], + each_ts['port'] + ) + final_report.append([ + each_ts['name'], + 'TargetServer', + each_ts['host'], + str(each_ts['port']), + each_ts['env'], + status, + " & ".join(list(set(proxy_targets[each_ts['name']]))) if each_ts['name'] in proxy_targets else "No References in any API" # noqa + ]) + + # Run Validation on Targets configured in Proxies + print('INFO: Running validation against All Targets discovered in Proxies') + for each_api_type, apis in proxy_hosts.items(): + for each_api, each_targets in apis.items(): + for each_target in each_targets: + if not has_templating(each_target['host']) and not each_target['target_server']: # noqa + if f"{each_target['host']}:{each_target['port']}" in _cached_hosts: # noqa + print('INFO: Fetching validation status from cached hosts') # noqa + status = _cached_hosts[f"{each_target['host']}:{each_target['port']}"] # noqa + else: + status = run_validator_proxy( + api_url, + vhost_domain_name, + vhost_ip, + each_target['host'], + each_target['port'] + ) + _cached_hosts[f"{each_target['host']}:{each_target['port']}"] = status # noqa + final_report.append([ + each_api, + "APIProxy" if each_api_type == 'apis' else "SharedFlow", # noqa + each_target['host'], + str(each_target['port']), + "_ORG_API_", + status, + each_target['source'] + ]) + if cfg['validation'].getboolean('check_csv'): + csv_file = cfg['csv']['file'] + default_port = cfg['csv']['default_port'] + csv_rows = read_csv(csv_file) + for each_row in csv_rows: + each_host, each_port = get_row_host_port(each_row, default_port) + if f"{each_host}:{each_port}" in _cached_hosts: + print('INFO: Fetching validation status from cached hosts') + status = _cached_hosts[f"{each_host}:{each_port}"] + else: + status = run_validator_proxy( + api_url, + vhost_domain_name, + vhost_ip, + each_host, + each_port + ) + _cached_hosts[f"{each_host}:{each_port}"] = status + final_report.append([ + each_host, + 'Input CSV', + each_host, + each_port, + "_NA_", + status, + "_NA_", + ]) + + # Write CSV Report + if report_format == 'csv': + report_file = 'report.csv' + print(f'INFO: Dumping report to file {report_file}') + write_csv_report(report_file, final_report) + + if report_format == 'md': + report_file = 'report.md' + print(f'INFO: Dumping report to file {report_file}') + write_md_report(report_file, final_report) + + +if __name__ == '__main__': + main() diff --git a/tools/target-server-validator/pipeline.sh b/tools/target-server-validator/pipeline.sh new file mode 100755 index 000000000..05842d420 --- /dev/null +++ b/tools/target-server-validator/pipeline.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SCRIPTPATH="$( cd "$(dirname "$0")" || exit >/dev/null 2>&1 ; pwd -P )" + +# Clean up previously generated files +rm -rf "$SCRIPTPATH/input.properties" +rm -rf "$SCRIPTPATH/export" +rm -rf "$SCRIPTPATH/report*" + +# Generate input file +cat > "$SCRIPTPATH/input.properties" << EOF +[source] +baseurl=https://apigee.googleapis.com/v1 +org=$APIGEE_X_ORG +auth_type=oauth + +[target] +baseurl=https://apigee.googleapis.com/v1 +org=$APIGEE_X_ORG +auth_type=oauth + +[csv] +file=input.csv +default_port=443 + +[validation] +check_csv=true +check_proxies=true +proxy_export_dir=export +skip_proxy_list= +api_env=$APIGEE_X_ENV +api_name=target_server_validator +vhost_domain_name=$APIGEE_X_HOSTNAME +vhost_ip= +report_format=md +EOF + +# Generate optional input csv file +cat > "$SCRIPTPATH/input.csv" << EOF +HOST,PORT +httpbin.org +httpbin.org,443 +mocktarget.apigee.tom +smtp.gmail.com,465 +EOF + +# Install Dependencies +python3 -m pip install -r "$SCRIPTPATH/requirements.txt" + +# Generate Gcloud Acccess Token +APIGEE_ACCESS_TOKEN="$(gcloud config config-helper --force-auth-refresh --format json | jq -r '.credential.access_token')" +export APIGEE_ACCESS_TOKEN + +# Running the Target Server Validator +cd "$SCRIPTPATH" + +python3 main.py + +# Display Report +cat "$SCRIPTPATH/report.md" diff --git a/tools/target-server-validator/report.md b/tools/target-server-validator/report.md new file mode 100644 index 000000000..eabb59ea0 --- /dev/null +++ b/tools/target-server-validator/report.md @@ -0,0 +1,8 @@ + +# Apigee Target Server Health Report + +NAME | TARGET_SOURCE | HOST | PORT | ENV | STATUS | INFO +--- | --- | --- | --- | --- | --- | --- +httpbin | APIProxy | httpbin.org | 443 | _ORG_API_ | REACHABLE | TargetEndpoint : default +mock | APIProxy | mocktarget.apigee.net | 443 | _ORG_API_ | REACHABLE | TargetEndpoint : default +mock | APIProxy | httpbin.org | 443 | _ORG_API_ | REACHABLE | TargetEndpoint : tec \ No newline at end of file diff --git a/tools/target-server-validator/requirements.txt b/tools/target-server-validator/requirements.txt new file mode 100644 index 000000000..52f3e381b --- /dev/null +++ b/tools/target-server-validator/requirements.txt @@ -0,0 +1,20 @@ +#!/usr/bin/python + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +xmltodict==0.13.0 +requests==2.28.1 +forcediphttpsadapter==1.0.2 diff --git a/tools/target-server-validator/utils.py b/tools/target-server-validator/utils.py new file mode 100644 index 000000000..147a99efe --- /dev/null +++ b/tools/target-server-validator/utils.py @@ -0,0 +1,214 @@ +#!/usr/bin/python + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import sys +import configparser +import zipfile +import requests +import csv +import xmltodict +from urllib.parse import urlparse +import urllib3 +from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +def parse_config(config_file): + config = configparser.ConfigParser() + config.read(config_file) + return config + + +def zipdir(path, ziph): + # ziph is zipfile handle + for root, _, files in os.walk(path): + for file in files: + ziph.write(os.path.join(root, file), + os.path.relpath(os.path.join(root, file), # noqa + os.path.join(path, '..'))) # noqa + +def create_proxy_bundle(proxy_bundle_directory, api_name, target_dir): # noqa + with zipfile.ZipFile(f"{proxy_bundle_directory}/{api_name}.zip", 'w', zipfile.ZIP_DEFLATED) as zipf: # noqa + zipdir(target_dir, zipf) + + +def run_validator_proxy(url, dns_host, vhost_ip, target_host, target_port='443'): # noqa + headers = { + 'host_name': target_host, + 'port_number': str(target_port), + 'Host': dns_host + } + session = requests.Session() + if len(vhost_ip) > 0: + session.mount(f"https://{dns_host}", ForcedIPHTTPSAdapter(dest_ip=vhost_ip)) # noqa + r = session.get(url, headers=headers, verify=False) + if r.status_code == 200: + return r.json()['status'] + return 'STATUS_UNKNOWN' + + +def delete_file(file_name): + try: + os.remove(file_name) + except FileNotFoundError: + print(f'File {file_name} doesnt exist') + + +def write_csv_report( + file_name, + rows, + header=['NAME','TARGET_SOURCE','HOST','PORT','ENV','STATUS','INFO']): # noqa + with open(file_name, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(header) + for each_row in rows: + writer.writerow(each_row) + + +def read_csv(file_name): + read_rows = [] + try: + with open(file_name) as file: + rows = csv.reader(file) + for each_row in rows: + read_rows.append(each_row) + except FileNotFoundError: + print(f'WARN: {file_name} not found ! ') + return read_rows + + +def write_md_report( + file_name, + rows, + header=['NAME', 'TARGET_SOURCE', 'HOST', 'PORT', 'ENV', 'STATUS', 'INFO']): # noqa + mded_rows = [] + for each_row in rows: + mded_rows.append(" | ".join(each_row)) + mded_rows = "\n".join(mded_rows) + data = f""" +# Apigee Target Server Health Report + +{" | ".join(header)} +{" | ".join(['---' for i in range(len(header))])} +{mded_rows} + """ + with open(file_name, 'w') as file: + file.write(data) + + +def create_dir(dir): + try: + os.makedirs(dir) + except FileExistsError: + print(f"INFO: {dir} already exists") + + +def list_dir(dir, soft=False): + try: + return os.listdir(dir) + except FileNotFoundError: + if soft: + return [] + print(f"ERROR: Directory \"{dir}\" not found") + sys.exit(1) + + +def unzip_file(path_to_zip_file, directory_to_extract_to): + with zipfile.ZipFile(path_to_zip_file, 'r') as zip_ref: + zip_ref.extractall(directory_to_extract_to) + + +def parse_xml(file): + try: + with open(file) as fl: + doc = xmltodict.parse(fl.read()) + return doc + except FileNotFoundError: + print(f"ERROR: File \"{file}\" not found") + return {} + + +def parse_http_target_connection(http_placement, http_placement_data): + hosts = None + if ('HTTPTargetConnection' in http_placement_data[http_placement] and # noqa + 'URL' in http_placement_data[http_placement]['HTTPTargetConnection']): # noqa + url_data = urlparse(http_placement_data[http_placement]['HTTPTargetConnection']['URL']) # noqa + hosts = { + 'host': url_data.hostname, + 'port': str(url_data.port) if url_data.port is not None else ( '443' if url_data.scheme == 'https' else '80' ), # noqa + 'source': f"{http_placement} : {http_placement_data[http_placement]['@name']}", # noqa + 'target_server': False + } + if ('HTTPTargetConnection' in http_placement_data[http_placement] and # noqa + 'LoadBalancer' in http_placement_data[http_placement]['HTTPTargetConnection']): # noqa + servers = http_placement_data[http_placement]['HTTPTargetConnection']['LoadBalancer']['Server'] # noqa + servers_list = servers if type(servers) is list else [servers] # noqa + target_servers = [ ts['@name'] for ts in servers_list ] # noqa + hosts = { + 'host': target_servers, + 'port': '', + 'source': f"{http_placement} : {http_placement_data[http_placement]['@name']}", # noqa + 'target_server': True + } + return hosts + + +def parse_proxy_hosts(proxy_path): + policies_path = f"{proxy_path}/policies" + targets_path = f"{proxy_path}/targets" + policies = [ i for i in list_dir(policies_path,True) if i.endswith('.xml')] # noqa + targets = [ i for i in list_dir(targets_path,True) if i.endswith('.xml')] # noqa + hosts = [] + for each_policy in policies: + each_policy_info = parse_xml(f"{policies_path}/{each_policy}") # noqa + if 'ServiceCallout' in each_policy_info: + host_data = parse_http_target_connection('ServiceCallout',each_policy_info) # noqa + if host_data is not None: + hosts.append(host_data) + for each_target in targets: + each_target_info = parse_xml(f"{targets_path}/{each_target}") # noqa + host_data = parse_http_target_connection('TargetEndpoint',each_target_info) # noqa + if host_data is not None: + hosts.append(host_data) + return hosts + + +def has_templating(data): + if '{' in data and '}' in data: + return True + else: + return False + + +def get_tes(data): + tes = [] + for each_host in data: + if each_host['target_server']: + tes.extend(each_host['host']) + return tes + + +def get_row_host_port(row, default_port=443): + host, port = None, None + if len(row) == 0: + print('WARN: Input row has no host ') + if len(row) == 1: + host, port = row[0], default_port + if len(row) > 1: + host, port = row[0], row[1] + return host, port From 6d177168406a4a3e573301d329b0e11102e6af57 Mon Sep 17 00:00:00 2001 From: anaik91 Date: Wed, 23 Aug 2023 00:12:46 +0530 Subject: [PATCH 02/14] fix: fixed pyline issues --- tools/target-server-validator/apigee.py | 30 ++++++++++++------------- tools/target-server-validator/report.md | 5 +++-- tools/target-server-validator/utils.py | 4 ++-- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tools/target-server-validator/apigee.py b/tools/target-server-validator/apigee.py index 727b61d4f..28be92605 100644 --- a/tools/target-server-validator/apigee.py +++ b/tools/target-server-validator/apigee.py @@ -15,9 +15,9 @@ # limitations under the License. -import requests import os import sys +import requests import shutil from time import sleep @@ -41,9 +41,9 @@ def __init__( def is_token_valid(self, token): url = f"https://www.googleapis.com/oauth2/v1/tokeninfo?access_token={token}" # noqa - r = requests.get(url) - if r.status_code == 200: - print(f"Token Validated for user {r.json()['email']}") + response = requests.get(url) + if response.status_code == 200: + print(f"Token Validated for user {response.json()['email']}") return True return False @@ -205,31 +205,31 @@ def get_api_vhost(self, vhost_name, env): def list_apis(self, api_type): url = f"{self.baseurl}/{api_type}" headers = self.auth_header.copy() - r = requests.get(url, headers=headers) - if r.status_code == 200: + response = requests.get(url, headers=headers) + if response.status_code == 200: if self.apigee_type == 'x': - if len(r.json()) == 0: + if len(response.json()) == 0: return [] - return [ p['name'] for p in r.json()['proxies' if api_type == 'apis' else 'sharedFlows']] # noqa - return r.json() + return [ p['name'] for p in response.json()['proxies' if api_type == 'apis' else 'sharedFlows']] # noqa + return response.json() else: return [] def list_api_revisions(self, api_type, api_name): url = f"{self.baseurl}/{api_type}/{api_name}/revisions" headers = self.auth_header.copy() - r = requests.get(url, headers=headers) - if r.status_code == 200: - return r.json() + response = requests.get(url, headers=headers) + if response.status_code == 200: + return response.json() else: return [] def fetch_api_revision(self, api_type, api_name, revision, export_dir): # noqa url = f"{self.baseurl}/{api_type}/{api_name}/revisions/{revision}?format=bundle" # noqa headers = self.auth_header.copy() - r = requests.get(url, headers=headers, stream=True) - if r.status_code == 200: - self.write_proxy_bundle(export_dir, api_name, r.raw) + response = requests.get(url, headers=headers, stream=True) + if response.status_code == 200: + self.write_proxy_bundle(export_dir, api_name, response.raw) return True return False diff --git a/tools/target-server-validator/report.md b/tools/target-server-validator/report.md index eabb59ea0..6ff20241c 100644 --- a/tools/target-server-validator/report.md +++ b/tools/target-server-validator/report.md @@ -3,6 +3,7 @@ NAME | TARGET_SOURCE | HOST | PORT | ENV | STATUS | INFO --- | --- | --- | --- | --- | --- | --- -httpbin | APIProxy | httpbin.org | 443 | _ORG_API_ | REACHABLE | TargetEndpoint : default +git | APIProxy | api.github.com | 443 | _ORG_API_ | REACHABLE | TargetEndpoint : default mock | APIProxy | mocktarget.apigee.net | 443 | _ORG_API_ | REACHABLE | TargetEndpoint : default -mock | APIProxy | httpbin.org | 443 | _ORG_API_ | REACHABLE | TargetEndpoint : tec \ No newline at end of file +mock_base | APIProxy | mocktarget.apigee.net | 443 | _ORG_API_ | REACHABLE | TargetEndpoint : default + \ No newline at end of file diff --git a/tools/target-server-validator/utils.py b/tools/target-server-validator/utils.py index 147a99efe..de86d8af1 100644 --- a/tools/target-server-validator/utils.py +++ b/tools/target-server-validator/utils.py @@ -19,10 +19,10 @@ import sys import configparser import zipfile -import requests import csv -import xmltodict from urllib.parse import urlparse +import requests +import xmltodict import urllib3 from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) From 541877dd4555f6b3566ae92a5c882a76c9d1766d Mon Sep 17 00:00:00 2001 From: anaik91 Date: Wed, 23 Aug 2023 14:10:43 +0530 Subject: [PATCH 03/14] fix: fixed java , python lint issues --- tools/target-server-validator/apigee.py | 128 ++++++--- .../callout/src/main/java/PortOpenCheck.java | 91 +++--- .../callout/src/main/java/package-info.java | 22 ++ tools/target-server-validator/main.py | 269 ++++++++++-------- tools/target-server-validator/utils.py | 126 ++++---- 5 files changed, 388 insertions(+), 248 deletions(-) create mode 100644 tools/target-server-validator/callout/src/main/java/package-info.java diff --git a/tools/target-server-validator/apigee.py b/tools/target-server-validator/apigee.py index 28be92605..3516b29e2 100644 --- a/tools/target-server-validator/apigee.py +++ b/tools/target-server-validator/apigee.py @@ -23,20 +23,22 @@ class Apigee: - def __init__( - self, - apigee_type="x", - base_url="https://apigee.googleapis.com/v1", - auth_type="oauth", - org="validate"): + self, + apigee_type="x", + base_url="https://apigee.googleapis.com/v1", + auth_type="oauth", + org="validate", + ): self.org = org self.baseurl = f"{base_url}/organizations/{org}" self.apigee_type = apigee_type self.auth_type = auth_type access_token = self.get_access_token() self.auth_header = { - 'Authorization': 'Bearer {}'.format(access_token) if self.auth_type == 'oauth' else 'Basic {}'.format(access_token) # noqa + "Authorization": "Bearer {}".format(access_token) + if self.auth_type == "oauth" + else "Basic {}".format(access_token) # noqa } def is_token_valid(self, token): @@ -48,27 +50,37 @@ def is_token_valid(self, token): return False def get_access_token(self): - token = os.getenv('APIGEE_ACCESS_TOKEN' if self.apigee_type == 'x' else 'APIGEE_OPDK_ACCESS_TOKEN') # noqa + token = os.getenv( + "APIGEE_ACCESS_TOKEN" + if self.apigee_type == "x" + else "APIGEE_OPDK_ACCESS_TOKEN" + ) if token is not None: - if self.apigee_type == 'x': + if self.apigee_type == "x": if self.is_token_valid(token): return token else: - print('please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ') # noqa + print( + 'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ' # noqa type: ignore + ) sys.exit(1) else: return token else: - if self.apigee_type == 'x': - print('please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ') # noqa + if self.apigee_type == "x": + print( + 'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ' # noqa + ) else: - print('please export APIGEE_OPDK_ACCESS_TOKEN') + print("please export APIGEE_OPDK_ACCESS_TOKEN") sys.exit(1) def set_auth_header(self): access_token = self.get_access_token() self.auth_header = { - 'Authorization': 'Bearer {}'.format(access_token) if self.auth_type == 'oauth' else 'Basic {}'.format(access_token) # noqa + "Authorization": "Bearer {}".format(access_token) + if self.auth_type == "oauth" + else "Basic {}".format(access_token) } def list_environments(self): @@ -111,10 +123,15 @@ def create_api(self, api_name, proxy_bundle_path): url = f"{self.baseurl}/apis?action=import&name={api_name}&validate=true" # noqa proxy_bundle_name = os.path.basename(proxy_bundle_path) files = [ - ('data',(proxy_bundle_name,open(proxy_bundle_path,'rb'),'application/zip')) # noqa + ( + "data", + (proxy_bundle_name, open(proxy_bundle_path, "rb"), "application/zip"), # noqa + ) ] headers = self.auth_header.copy() - response = requests.request("POST", url, headers=headers, data={}, files=files) # noqa + response = requests.request( + "POST", url, headers=headers, data={}, files=files + ) if response.status_code == 200: return True else: @@ -122,17 +139,19 @@ def create_api(self, api_name, proxy_bundle_path): return False def get_api_revisions_deployment(self, env, api_name, api_rev): # noqa - url = url = f"{self.baseurl}/environments/{env}/apis/{api_name}/revisions/{api_rev}/deployments" # noqa + url = ( + url + ) = f"{self.baseurl}/environments/{env}/apis/{api_name}/revisions/{api_rev}/deployments" # noqa headers = self.auth_header.copy() response = requests.request("GET", url, headers=headers, data={}) if response.status_code == 200: resp = response.json() - api_deployment_status = resp.get('state', '') - if self.apigee_type == 'x': - if api_deployment_status == 'READY': + api_deployment_status = resp.get("state", "") + if self.apigee_type == "x": + if api_deployment_status == "READY": return True - if self.apigee_type == 'opdk': - if api_deployment_status == 'deployed': + if self.apigee_type == "opdk": + if api_deployment_status == "deployed": return True print(f"API {api_name} is in Status: {api_deployment_status} !") # noqa return False @@ -140,15 +159,17 @@ def get_api_revisions_deployment(self, env, api_name, api_rev): # noqa return False def deploy_api(self, env, api_name, api_rev): - url = url = f"{self.baseurl}/environments/{env}/apis/{api_name}/revisions/{api_rev}/deployments?override=true" # noqa + url = ( + url + ) = f"{self.baseurl}/environments/{env}/apis/{api_name}/revisions/{api_rev}/deployments?override=true" # noqa headers = self.auth_header.copy() response = requests.request("POST", url, headers=headers, data={}) if response.status_code == 200: return True else: resp = response.json() - if 'already deployed' in resp['error']['message']: - print('Proxy {} is already Deployed'.format(api_name)) + if "already deployed" in resp["error"]["message"]: + print("Proxy {} is already Deployed".format(api_name)) return True return False @@ -158,48 +179,64 @@ def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_rev=1): # noq api_deployment_retry_count = 0 api_exists = False if self.get_api(api_name): - print(f'Proxy with name {api_name} already exists in Apigee Org {self.org}') # noqa - api_exists = True + print( + f"Proxy with name {api_name} already exists in Apigee Org {self.org}" # noqa + ) + api_exists = True else: if self.create_api(api_name, proxy_bundle_path): - print(f'Proxy has been imported with name {api_name} in Apigee Org {self.org}') # noqa + print( + f"Proxy has been imported with name {api_name} in Apigee Org {self.org}" # noqa + ) api_exists = True else: - print(f'ERROR : Proxy {api_name} import failed !!! ') + print(f"ERROR : Proxy {api_name} import failed !!! ") return False if api_exists: if self.deploy_api(env, api_name, api_rev): - print(f'Proxy with name {api_name} has been deployed to {env} in Apigee Org {self.org}') # noqa + print( + f"Proxy with name {api_name} has been deployed to {env} in Apigee Org {self.org}" # noqa + ) while api_deployment_retry_count < api_deployment_retry: - if self.get_api_revisions_deployment(env, api_name, api_rev): # noqa - print(f'Proxy {api_name} active in runtime after {api_deployment_retry_count*api_deployment_sleep} seconds ') # noqa + if self.get_api_revisions_deployment( + env, api_name, api_rev + ): + print( + f"Proxy {api_name} active in runtime after {api_deployment_retry_count*api_deployment_sleep} seconds " # noqa + ) return True else: - print(f"Checking API deployment status in {api_deployment_sleep} seconds") # noqa + print( + f"Checking API deployment status in {api_deployment_sleep} seconds" # noqa + ) sleep(api_deployment_sleep) api_deployment_retry_count += 1 else: - print(f'ERROR : Proxy deployment to {env} in Apigee Org {self.org} Failed !!') # noqa + print( + f"ERROR : Proxy deployment to {env} in Apigee Org {self.org} Failed !!" # noqa + ) return False def get_api_vhost(self, vhost_name, env): - if self.apigee_type == 'opdk': + if self.apigee_type == "opdk": url = f"{self.baseurl}/environments/{env}/virtualhosts/{vhost_name}" # noqa else: url = f"{self.baseurl}/envgroups/{vhost_name}" headers = self.auth_header.copy() response = requests.request("GET", url, headers=headers) if response.status_code == 200: - if self.apigee_type == 'opdk': - hosts = response.json()['hostAliases'] + if self.apigee_type == "opdk": + hosts = response.json()["hostAliases"] else: - hosts = response.json()['hostnames'] + hosts = response.json()["hostnames"] if len(hosts) == 0: - print(f'ERROR: Vhost/Env Group {vhost_name} contains no domains') # noqa + print( + f"ERROR: Vhost/Env Group {vhost_name} contains no domains" # noqa + ) return None return hosts else: - print(f'ERROR: Vhost/Env Group {vhost_name} contains no domains') # noqa + print(f"ERROR: Vhost/Env Group {vhost_name} contains no domains") # noqa return None def list_apis(self, api_type): @@ -207,10 +244,15 @@ def list_apis(self, api_type): headers = self.auth_header.copy() response = requests.get(url, headers=headers) if response.status_code == 200: - if self.apigee_type == 'x': + if self.apigee_type == "x": if len(response.json()) == 0: return [] - return [ p['name'] for p in response.json()['proxies' if api_type == 'apis' else 'sharedFlows']] # noqa + return [ + p["name"] + for p in response.json()[ + "proxies" if api_type == "apis" else "sharedFlows" + ] + ] # noqa return response.json() else: return [] @@ -235,5 +277,5 @@ def fetch_api_revision(self, api_type, api_name, revision, export_dir): # noqa def write_proxy_bundle(self, export_dir, file_name, data): file_path = f"./{export_dir}/{file_name}.zip" - with open(file_path, 'wb') as fl: + with open(file_path, "wb") as fl: shutil.copyfileobj(data, fl) diff --git a/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java b/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java index 7271de7ac..f8a564cb0 100644 --- a/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java +++ b/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java @@ -19,53 +19,66 @@ import com.apigee.flow.execution.ExecutionResult; import com.apigee.flow.execution.spi.Execution; import com.apigee.flow.message.MessageContext; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketTimeoutException; import java.net.UnknownHostException; - +/** + * A callout that checks if a particular port is open on a specified host. + */ public class PortOpenCheck implements Execution { - private static String available(String host,int port) { - Socket socket = new Socket(); - try { - socket.connect(new InetSocketAddress(host, port), 1000); - return "REACHABLE"; - } catch (SocketTimeoutException e) { - return "NOT_REACHABLE"; - } catch (UnknownHostException e) { - return "UNKNOWN_HOST"; - } - catch (IOException e) { - return "NOT_REACHABLE"; - } finally { - if (socket != null) { - try { - socket.close(); - } catch (IOException e) { - throw new RuntimeException("You should handle this error.", e); - } - } - } - } - - public ExecutionResult execute(MessageContext messageContext, ExecutionContext executionContext) { - - try { - String host_name = messageContext.getMessage().getHeader("host_name"); - String port = messageContext.getMessage().getHeader("port_number"); - int port_number = Integer.parseInt(port); - String Status = available(host_name,port_number); - // messageContext.getMessage().setContent(Status); - messageContext.setVariable("REACHABLE_STATUS", Status); - return ExecutionResult.SUCCESS; - - } catch (Exception e) { - return ExecutionResult.ABORT; - } + /** + * Checks if the specified host and port are available. + * + * @param host The host name or IP address to check. + * @param port The port number to check. + * @return A string indicating whether the host and port are available + */ + private static String available(final String host, final int port) { + Socket socket = new Socket(); + final int sockettimeout = 1000; + try { + socket.connect(new InetSocketAddress(host, port), sockettimeout); + return "REACHABLE"; + } catch (SocketTimeoutException e) { + return "NOT_REACHABLE"; + } catch (UnknownHostException e) { + return "UNKNOWN_HOST"; + } catch (IOException e) { + return "NOT_REACHABLE"; + } finally { + if (socket != null) { + try { + socket.close(); + } catch (IOException e) { + throw new RuntimeException("You should handle this error.", e); } + } + } + } + /** + * Executes the callout. + * + * @param messageContext The message context. + * @param executionContext The execution context. + * @return The execution result. + */ + public ExecutionResult execute(final MessageContext messageContext, + final ExecutionContext executionContext) { + try { + String hostname = messageContext.getMessage().getHeader("host_name"); + String port = messageContext.getMessage().getHeader("port_number"); + int portnumber = Integer.parseInt(port); + String status = available(hostname, portnumber); + // messageContext.getMessage().setContent(Status); + messageContext.setVariable("REACHABLE_STATUS", status); + return ExecutionResult.SUCCESS; + } catch (Exception e) { + return ExecutionResult.ABORT; + } + } } diff --git a/tools/target-server-validator/callout/src/main/java/package-info.java b/tools/target-server-validator/callout/src/main/java/package-info.java new file mode 100644 index 000000000..019106599 --- /dev/null +++ b/tools/target-server-validator/callout/src/main/java/package-info.java @@ -0,0 +1,22 @@ +// Copyright 2023 Google LLC + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +/** + * A callout that checks if a particular port is open on a specified host. + * + * @author anaik91 + * @version .01 + */ +package com.apigeesample; diff --git a/tools/target-server-validator/main.py b/tools/target-server-validator/main.py index 2c145c603..2a8cce651 100644 --- a/tools/target-server-validator/main.py +++ b/tools/target-server-validator/main.py @@ -17,218 +17,251 @@ import os import sys -from utils import (parse_config, - create_proxy_bundle, - run_validator_proxy, - delete_file, - read_csv, - write_csv_report, - write_md_report, - create_dir, - unzip_file, - parse_proxy_hosts, - has_templating, - get_tes, - get_row_host_port) +from utils import ( + parse_config, + create_proxy_bundle, + run_validator_proxy, + delete_file, + read_csv, + write_csv_report, + write_md_report, + create_dir, + unzip_file, + parse_proxy_hosts, + has_templating, + get_tes, + get_row_host_port, +) from apigee import Apigee def main(): # Parse Inputs - cfg = parse_config('input.properties') - check_proxies = cfg['validation'].getboolean('check_proxies') - proxy_export_dir = cfg['validation']['proxy_export_dir'] - report_format = cfg['validation']['report_format'] - if report_format not in ['csv', 'md']: - report_format = 'md' + cfg = parse_config("input.properties") + check_proxies = cfg["validation"].getboolean("check_proxies") + proxy_export_dir = cfg["validation"]["proxy_export_dir"] + report_format = cfg["validation"]["report_format"] + if report_format not in ["csv", "md"]: + report_format = "md" # Intialize Source & Target Apigee SourceApigee = Apigee( - 'x' if 'apigee.googleapis.com' in cfg['source']['baseurl'] else 'opdk', - cfg['source']['baseurl'], - cfg['source']['auth_type'], - cfg['source']['org'] + "x" if "apigee.googleapis.com" in cfg["source"]["baseurl"] else "opdk", + cfg["source"]["baseurl"], + cfg["source"]["auth_type"], + cfg["source"]["org"], ) TargetApigee = Apigee( - 'x' if 'apigee.googleapis.com' in cfg['source']['baseurl'] else 'opdk', - cfg['target']['baseurl'], - cfg['target']['auth_type'], - cfg['target']['org'] + "x" if "apigee.googleapis.com" in cfg["source"]["baseurl"] else "opdk", + cfg["target"]["baseurl"], + cfg["target"]["auth_type"], + cfg["target"]["org"], ) environments = SourceApigee.list_environments() all_target_servers = [] # Fetch Target Servers from Source Apigee@ - print('INFO: exporting Target Servers !') + print("INFO: exporting Target Servers !") for each_env in environments: target_servers = SourceApigee.list_target_servers(each_env) for each_ts in target_servers: ts_info = SourceApigee.get_target_server(each_env, each_ts) - ts_info['env'] = each_env + ts_info["env"] = each_env all_target_servers.append(ts_info) # Fetch Targets in APIs & Shared Flows from Source Apigee proxy_hosts = {} proxy_targets = {} if check_proxies: - skip_proxy_list = cfg['validation'].get('skip_proxy_list', '').split(',') # noqa - print('INFO: exporting proxies to be analyzed ! this may take a while !') # noqa - api_types = ['apis', 'sharedflows'] + skip_proxy_list = ( + cfg["validation"].get("skip_proxy_list", "").split(",") + ) + print( + "INFO: exporting proxies to be analyzed ! this may take a while !" + ) + api_types = ["apis", "sharedflows"] api_revision_map = {} for each_api_type in api_types: api_revision_map[each_api_type] = {} - api_revision_map[each_api_type]['proxies'] = {} - api_revision_map[each_api_type]['export_dir'] = proxy_export_dir+f'/{each_api_type}' # noqa - create_dir(proxy_export_dir+f'/{each_api_type}') + api_revision_map[each_api_type]["proxies"] = {} + api_revision_map[each_api_type]["export_dir"] = ( + proxy_export_dir + f"/{each_api_type}" + ) + create_dir(proxy_export_dir + f"/{each_api_type}") for each_api in SourceApigee.list_apis(each_api_type): if each_api not in skip_proxy_list: - api_revision_map[each_api_type]['proxies'][each_api] = SourceApigee.list_api_revisions(each_api_type,each_api)[-1] # noqa + api_revision_map[each_api_type]["proxies"][ + each_api + ] = SourceApigee.list_api_revisions(each_api_type, each_api)[ # noqa + -1 + ] else: print(f"INFO : Skipping API {each_api}") for each_api_type, each_api_type_data in api_revision_map.items(): proxy_hosts[each_api_type] = {} - for each_api, each_api_rev in each_api_type_data['proxies'].items(): # noqa - print(f"Exporting API : {each_api} with revision : {each_api_rev} ") # noqa - SourceApigee.fetch_api_revision(each_api_type, - each_api, - each_api_rev, - api_revision_map[each_api_type]['export_dir']) # noqa - print(f"Unzipping API : {each_api} with revision : {each_api_rev} ") # noqa + for each_api, each_api_rev in each_api_type_data["proxies"].items(): # noqa + print( + f"Exporting API : {each_api} with revision : {each_api_rev} " # noqa + ) + SourceApigee.fetch_api_revision( + each_api_type, + each_api, + each_api_rev, + api_revision_map[each_api_type]["export_dir"], + ) + print( + f"Unzipping API : {each_api} with revision : {each_api_rev} " # noqa + ) unzip_file( f"{api_revision_map[each_api_type]['export_dir']}/{each_api}.zip", # noqa - f"{api_revision_map[each_api_type]['export_dir']}/{each_api}" # noqa + f"{api_revision_map[each_api_type]['export_dir']}/{each_api}", # noqa ) parsed_proxy_hosts = parse_proxy_hosts( - f"{api_revision_map[each_api_type]['export_dir']}/{each_api}/apiproxy" # noqa + f"{api_revision_map[each_api_type]['export_dir']}/{each_api}/apiproxy" # noqa ) proxy_hosts[each_api_type][each_api] = parsed_proxy_hosts proxy_tes = get_tes(parsed_proxy_hosts) for each_te in proxy_tes: if each_te in proxy_targets: - proxy_targets[each_te].append(f"{each_api_type} - {each_api}") # noqa + proxy_targets[each_te].append( + f"{each_api_type} - {each_api}" + ) else: - proxy_targets[each_te] = [f"{each_api_type} - {each_api}"] # noqa + proxy_targets[each_te] = [ + f"{each_api_type} - {each_api}" + ] # Validate Targets against Target Apigee bundle_path = os.path.dirname(os.path.abspath(__file__)) # Create Validation Proxy Bundle - print('INFO: Creating proxy bundle !') - create_proxy_bundle( - bundle_path, - cfg['validation']['api_name'], - 'apiproxy' - ) + print("INFO: Creating proxy bundle !") + create_proxy_bundle(bundle_path, cfg["validation"]["api_name"], "apiproxy") # Deploy Validation Proxy Bundle - print('INFO: Deploying proxy bundle !') + print("INFO: Deploying proxy bundle !") if not TargetApigee.deploy_api_bundle( - cfg['validation']['api_env'], - cfg['validation']['api_name'], - f"{bundle_path}/{cfg['validation']['api_name']}.zip" + cfg["validation"]["api_env"], + cfg["validation"]["api_name"], + f"{bundle_path}/{cfg['validation']['api_name']}.zip", ): print(f"Proxy: {cfg['validation']['api_name']} deployment failed.") sys.exit(1) # CleanUp Validation Proxy Bundle - print('INFO: Cleaning Up local proxy bundle !') + print("INFO: Cleaning Up local proxy bundle !") delete_file(f"{bundle_path}/{cfg['validation']['api_name']}.zip") # Fetch API Northbound Endpoint - print(f"INFO: Fetching VHost with name {cfg['validation']['vhost_domain_name']} !") # noqa - vhost_domain_name = cfg['validation']['vhost_domain_name'] - vhost_ip = cfg['validation'].get('vhost_ip', '').strip() + print( + f"INFO: Fetching VHost with name {cfg['validation']['vhost_domain_name']} !" # noqa + ) + vhost_domain_name = cfg["validation"]["vhost_domain_name"] + vhost_ip = cfg["validation"].get("vhost_ip", "").strip() api_url = f"https://{vhost_domain_name}/validate_target_server" final_report = [] _cached_hosts = {} # Run Target Server Validation - print('INFO: Running validation against All Target Servers') + print("INFO: Running validation against All Target Servers") for each_ts in all_target_servers: status = run_validator_proxy( - api_url, - vhost_domain_name, - vhost_ip, - each_ts['host'], - each_ts['port'] + api_url, vhost_domain_name, vhost_ip, each_ts["host"], each_ts["port"] # noqa + ) + final_report.append( + [ + each_ts["name"], + "TargetServer", + each_ts["host"], + str(each_ts["port"]), + each_ts["env"], + status, + " & ".join(list(set(proxy_targets[each_ts["name"]]))) + if each_ts["name"] in proxy_targets + else "No References in any API", + ] ) - final_report.append([ - each_ts['name'], - 'TargetServer', - each_ts['host'], - str(each_ts['port']), - each_ts['env'], - status, - " & ".join(list(set(proxy_targets[each_ts['name']]))) if each_ts['name'] in proxy_targets else "No References in any API" # noqa - ]) # Run Validation on Targets configured in Proxies - print('INFO: Running validation against All Targets discovered in Proxies') + print("INFO: Running validation against All Targets discovered in Proxies") for each_api_type, apis in proxy_hosts.items(): for each_api, each_targets in apis.items(): for each_target in each_targets: - if not has_templating(each_target['host']) and not each_target['target_server']: # noqa - if f"{each_target['host']}:{each_target['port']}" in _cached_hosts: # noqa - print('INFO: Fetching validation status from cached hosts') # noqa - status = _cached_hosts[f"{each_target['host']}:{each_target['port']}"] # noqa + if ( + not has_templating(each_target["host"]) + and not each_target["target_server"] + ): + if ( + f"{each_target['host']}:{each_target['port']}" in _cached_hosts # noqa + ): + print( + "INFO: Fetching validation status from cached hosts" # noqa + ) + status = _cached_hosts[ + f"{each_target['host']}:{each_target['port']}" # noqa + ] else: status = run_validator_proxy( api_url, vhost_domain_name, vhost_ip, - each_target['host'], - each_target['port'] + each_target["host"], + each_target["port"], ) - _cached_hosts[f"{each_target['host']}:{each_target['port']}"] = status # noqa - final_report.append([ - each_api, - "APIProxy" if each_api_type == 'apis' else "SharedFlow", # noqa - each_target['host'], - str(each_target['port']), - "_ORG_API_", - status, - each_target['source'] - ]) - if cfg['validation'].getboolean('check_csv'): - csv_file = cfg['csv']['file'] - default_port = cfg['csv']['default_port'] + _cached_hosts[ + f"{each_target['host']}:{each_target['port']}" + ] = status + final_report.append( + [ + each_api, + "APIProxy" + if each_api_type == "apis" + else "SharedFlow", + each_target["host"], + str(each_target["port"]), + "_ORG_API_", + status, + each_target["source"], + ] + ) + if cfg["validation"].getboolean("check_csv"): + csv_file = cfg["csv"]["file"] + default_port = cfg["csv"]["default_port"] csv_rows = read_csv(csv_file) for each_row in csv_rows: each_host, each_port = get_row_host_port(each_row, default_port) if f"{each_host}:{each_port}" in _cached_hosts: - print('INFO: Fetching validation status from cached hosts') + print("INFO: Fetching validation status from cached hosts") status = _cached_hosts[f"{each_host}:{each_port}"] else: status = run_validator_proxy( - api_url, - vhost_domain_name, - vhost_ip, - each_host, - each_port + api_url, vhost_domain_name, vhost_ip, each_host, each_port ) _cached_hosts[f"{each_host}:{each_port}"] = status - final_report.append([ - each_host, - 'Input CSV', - each_host, - each_port, - "_NA_", - status, - "_NA_", - ]) + final_report.append( + [ + each_host, + "Input CSV", + each_host, + each_port, + "_NA_", + status, + "_NA_", + ] + ) # Write CSV Report - if report_format == 'csv': - report_file = 'report.csv' - print(f'INFO: Dumping report to file {report_file}') + if report_format == "csv": + report_file = "report.csv" + print(f"INFO: Dumping report to file {report_file}") write_csv_report(report_file, final_report) - if report_format == 'md': - report_file = 'report.md' - print(f'INFO: Dumping report to file {report_file}') + if report_format == "md": + report_file = "report.md" + print(f"INFO: Dumping report to file {report_file}") write_md_report(report_file, final_report) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/target-server-validator/utils.py b/tools/target-server-validator/utils.py index de86d8af1..6309736c7 100644 --- a/tools/target-server-validator/utils.py +++ b/tools/target-server-validator/utils.py @@ -25,6 +25,7 @@ import xmltodict import urllib3 from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -38,42 +39,53 @@ def zipdir(path, ziph): # ziph is zipfile handle for root, _, files in os.walk(path): for file in files: - ziph.write(os.path.join(root, file), - os.path.relpath(os.path.join(root, file), # noqa - os.path.join(path, '..'))) # noqa + ziph.write( + os.path.join(root, file), + os.path.relpath( + os.path.join(root, file), os.path.join(path, "..") # noqa + ), + ) # noqa + def create_proxy_bundle(proxy_bundle_directory, api_name, target_dir): # noqa - with zipfile.ZipFile(f"{proxy_bundle_directory}/{api_name}.zip", 'w', zipfile.ZIP_DEFLATED) as zipf: # noqa + with zipfile.ZipFile( + f"{proxy_bundle_directory}/{api_name}.zip", "w", zipfile.ZIP_DEFLATED + ) as zipf: # noqa zipdir(target_dir, zipf) -def run_validator_proxy(url, dns_host, vhost_ip, target_host, target_port='443'): # noqa +def run_validator_proxy( + url, dns_host, vhost_ip, target_host, target_port="443" +): # noqa headers = { - 'host_name': target_host, - 'port_number': str(target_port), - 'Host': dns_host + "host_name": target_host, + "port_number": str(target_port), + "Host": dns_host, } session = requests.Session() if len(vhost_ip) > 0: - session.mount(f"https://{dns_host}", ForcedIPHTTPSAdapter(dest_ip=vhost_ip)) # noqa + session.mount( + f"https://{dns_host}", ForcedIPHTTPSAdapter(dest_ip=vhost_ip) + ) # noqa r = session.get(url, headers=headers, verify=False) if r.status_code == 200: - return r.json()['status'] - return 'STATUS_UNKNOWN' + return r.json()["status"] + return "STATUS_UNKNOWN" def delete_file(file_name): try: os.remove(file_name) except FileNotFoundError: - print(f'File {file_name} doesnt exist') + print(f"File {file_name} doesnt exist") def write_csv_report( - file_name, - rows, - header=['NAME','TARGET_SOURCE','HOST','PORT','ENV','STATUS','INFO']): # noqa - with open(file_name, 'w', newline='') as file: + file_name, + rows, + header=["NAME", "TARGET_SOURCE", "HOST", "PORT", "ENV", "STATUS", "INFO"], +): # noqa + with open(file_name, "w", newline="") as file: writer = csv.writer(file) writer.writerow(header) for each_row in rows: @@ -88,14 +100,15 @@ def read_csv(file_name): for each_row in rows: read_rows.append(each_row) except FileNotFoundError: - print(f'WARN: {file_name} not found ! ') + print(f"WARN: {file_name} not found ! ") return read_rows def write_md_report( - file_name, - rows, - header=['NAME', 'TARGET_SOURCE', 'HOST', 'PORT', 'ENV', 'STATUS', 'INFO']): # noqa + file_name, + rows, + header=["NAME", "TARGET_SOURCE", "HOST", "PORT", "ENV", "STATUS", "INFO"], +): # noqa mded_rows = [] for each_row in rows: mded_rows.append(" | ".join(each_row)) @@ -107,7 +120,7 @@ def write_md_report( {" | ".join(['---' for i in range(len(header))])} {mded_rows} """ - with open(file_name, 'w') as file: + with open(file_name, "w") as file: file.write(data) @@ -124,12 +137,12 @@ def list_dir(dir, soft=False): except FileNotFoundError: if soft: return [] - print(f"ERROR: Directory \"{dir}\" not found") + print(f'ERROR: Directory "{dir}" not found') sys.exit(1) def unzip_file(path_to_zip_file, directory_to_extract_to): - with zipfile.ZipFile(path_to_zip_file, 'r') as zip_ref: + with zipfile.ZipFile(path_to_zip_file, "r") as zip_ref: zip_ref.extractall(directory_to_extract_to) @@ -139,31 +152,44 @@ def parse_xml(file): doc = xmltodict.parse(fl.read()) return doc except FileNotFoundError: - print(f"ERROR: File \"{file}\" not found") + print(f'ERROR: File "{file}" not found') return {} def parse_http_target_connection(http_placement, http_placement_data): hosts = None - if ('HTTPTargetConnection' in http_placement_data[http_placement] and # noqa - 'URL' in http_placement_data[http_placement]['HTTPTargetConnection']): # noqa - url_data = urlparse(http_placement_data[http_placement]['HTTPTargetConnection']['URL']) # noqa + if ( + "HTTPTargetConnection" in http_placement_data[http_placement] + and "URL" in http_placement_data[http_placement]["HTTPTargetConnection"] # noqa + ): # noqa + url_data = urlparse( + http_placement_data[http_placement]["HTTPTargetConnection"]["URL"] + ) # noqa hosts = { - 'host': url_data.hostname, - 'port': str(url_data.port) if url_data.port is not None else ( '443' if url_data.scheme == 'https' else '80' ), # noqa - 'source': f"{http_placement} : {http_placement_data[http_placement]['@name']}", # noqa - 'target_server': False + "host": url_data.hostname, + "port": str(url_data.port) + if url_data.port is not None + else ("443" if url_data.scheme == "https" else "80"), # noqa + "source": f"{http_placement} : {http_placement_data[http_placement]['@name']}", # noqa + "target_server": False, } - if ('HTTPTargetConnection' in http_placement_data[http_placement] and # noqa - 'LoadBalancer' in http_placement_data[http_placement]['HTTPTargetConnection']): # noqa - servers = http_placement_data[http_placement]['HTTPTargetConnection']['LoadBalancer']['Server'] # noqa + if ( + "HTTPTargetConnection" in http_placement_data[http_placement] + and "LoadBalancer" # noqa + in http_placement_data[http_placement]["HTTPTargetConnection"] + ): # noqa + servers = http_placement_data[http_placement]["HTTPTargetConnection"][ + "LoadBalancer" + ][ + "Server" + ] # noqa servers_list = servers if type(servers) is list else [servers] # noqa - target_servers = [ ts['@name'] for ts in servers_list ] # noqa + target_servers = [ts["@name"] for ts in servers_list] # noqa hosts = { - 'host': target_servers, - 'port': '', - 'source': f"{http_placement} : {http_placement_data[http_placement]['@name']}", # noqa - 'target_server': True + "host": target_servers, + "port": "", + "source": f"{http_placement} : {http_placement_data[http_placement]['@name']}", # noqa + "target_server": True, } return hosts @@ -171,25 +197,29 @@ def parse_http_target_connection(http_placement, http_placement_data): def parse_proxy_hosts(proxy_path): policies_path = f"{proxy_path}/policies" targets_path = f"{proxy_path}/targets" - policies = [ i for i in list_dir(policies_path,True) if i.endswith('.xml')] # noqa - targets = [ i for i in list_dir(targets_path,True) if i.endswith('.xml')] # noqa + policies = [i for i in list_dir(policies_path, True) if i.endswith(".xml")] # noqa + targets = [i for i in list_dir(targets_path, True) if i.endswith(".xml")] # noqa hosts = [] for each_policy in policies: each_policy_info = parse_xml(f"{policies_path}/{each_policy}") # noqa - if 'ServiceCallout' in each_policy_info: - host_data = parse_http_target_connection('ServiceCallout',each_policy_info) # noqa + if "ServiceCallout" in each_policy_info: + host_data = parse_http_target_connection( + "ServiceCallout", each_policy_info + ) # noqa if host_data is not None: hosts.append(host_data) for each_target in targets: each_target_info = parse_xml(f"{targets_path}/{each_target}") # noqa - host_data = parse_http_target_connection('TargetEndpoint',each_target_info) # noqa + host_data = parse_http_target_connection( + "TargetEndpoint", each_target_info + ) # noqa if host_data is not None: hosts.append(host_data) return hosts def has_templating(data): - if '{' in data and '}' in data: + if "{" in data and "}" in data: return True else: return False @@ -198,15 +228,15 @@ def has_templating(data): def get_tes(data): tes = [] for each_host in data: - if each_host['target_server']: - tes.extend(each_host['host']) + if each_host["target_server"]: + tes.extend(each_host["host"]) return tes def get_row_host_port(row, default_port=443): host, port = None, None if len(row) == 0: - print('WARN: Input row has no host ') + print("WARN: Input row has no host ") if len(row) == 1: host, port = row[0], default_port if len(row) > 1: From f05d807f532782c221e295a768e7a3b79219ee42 Mon Sep 17 00:00:00 2001 From: anaik91 Date: Thu, 24 Aug 2023 02:33:06 +0530 Subject: [PATCH 04/14] fix: fixed pylint errors --- tools/proxy-endpoint-unifier/main.py | 4 ++-- tools/target-server-validator/main.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/proxy-endpoint-unifier/main.py b/tools/proxy-endpoint-unifier/main.py index c03240151..447c7beb2 100644 --- a/tools/proxy-endpoint-unifier/main.py +++ b/tools/proxy-endpoint-unifier/main.py @@ -16,8 +16,8 @@ import os import sys -from apigee import Apigee -import utils +from apigee import Apigee # pylint: disable=import-error +import utils # pylint: disable=import-error def main(): diff --git a/tools/target-server-validator/main.py b/tools/target-server-validator/main.py index 2a8cce651..506457c56 100644 --- a/tools/target-server-validator/main.py +++ b/tools/target-server-validator/main.py @@ -17,7 +17,7 @@ import os import sys -from utils import ( +from utils import ( # pylint: disable=import-error parse_config, create_proxy_bundle, run_validator_proxy, @@ -32,7 +32,7 @@ get_tes, get_row_host_port, ) -from apigee import Apigee +from apigee import Apigee # pylint: disable=import-error def main(): From 4f6b06f6befe2796439a494403fb696e51c0c4de Mon Sep 17 00:00:00 2001 From: anaik91 Date: Tue, 29 Aug 2023 15:40:22 +0530 Subject: [PATCH 05/14] feat: renamed wrappers --- .../{apigee.py => apigee_utils.py} | 0 .../callout/target/classes/PortOpenCheck.class | Bin 0 -> 837 bytes .../callout/target/classes/package-info.class | Bin 0 -> 105 bytes tools/target-server-validator/main.py | 4 ++-- .../{utils.py => utilities.py} | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename tools/target-server-validator/{apigee.py => apigee_utils.py} (100%) create mode 100644 tools/target-server-validator/callout/target/classes/PortOpenCheck.class create mode 100644 tools/target-server-validator/callout/target/classes/package-info.class rename tools/target-server-validator/{utils.py => utilities.py} (100%) diff --git a/tools/target-server-validator/apigee.py b/tools/target-server-validator/apigee_utils.py similarity index 100% rename from tools/target-server-validator/apigee.py rename to tools/target-server-validator/apigee_utils.py diff --git a/tools/target-server-validator/callout/target/classes/PortOpenCheck.class b/tools/target-server-validator/callout/target/classes/PortOpenCheck.class new file mode 100644 index 0000000000000000000000000000000000000000..9680c39113e68de1df2ed2a82b57774584a17199 GIT binary patch literal 837 zcma)4O>fgc5Ph45I58;+f%1*y=F)^ieJCe_xJ0dl6eyyl632}*O}1To-SwvA%CABK z3GVzT#Mns*r4exOtY`Gzo0+%!{m1980Cw=$LrGx$jj?V&rKCTh=+r}5pm`$CWvFE` z4)=#A6gdwSfvw2Ip-k17C_L8YLr61;dn8cqn-O`q zF3?yC?^V?WClrh*($X>}mC>mjQ_yDc z_I#d9#;K-uFfx<{iE%+JT{H9JRz7HRbIy^K5HadglPzTPcWZcV#ZVXgH zucz^l?4jg21X=?VNqr=(%JW6B>?SJ9(U)B83Dnp^-lSk|t?2Qxd$n&B_XIZoX@`I} zx1NXk*ctqJEiq*o`xCyJpM8@!nz`N;-17Q Date: Tue, 29 Aug 2023 17:42:52 +0530 Subject: [PATCH 06/14] feat: disabled mypy check --- .github/workflows/devrel-static-checks.yml | 1 + tools/target-server-validator/input.properties | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/devrel-static-checks.yml b/.github/workflows/devrel-static-checks.yml index 5b308779f..7f2fe3be2 100644 --- a/.github/workflows/devrel-static-checks.yml +++ b/.github/workflows/devrel-static-checks.yml @@ -74,6 +74,7 @@ jobs: LINTER_RULES_PATH: "." GROOVY_NPM_GROOVY_LINT_FILTER_REGEX_EXCLUDE: "Jenkinsfile" MARKDOWN_MARKDOWN_LINK_CHECK_DISABLE_ERRORS: true + PYTHON_MYPY_DISABLE_ERRORS: true commit-messages: name: Conventional Commits Lint diff --git a/tools/target-server-validator/input.properties b/tools/target-server-validator/input.properties index 0570fb794..2105b7411 100644 --- a/tools/target-server-validator/input.properties +++ b/tools/target-server-validator/input.properties @@ -1,11 +1,11 @@ [source] baseurl=https://apigee.googleapis.com/v1 -org=apigee-hybrid-378710 +org=xxx-xxx-xxx-xxx auth_type=oauth [target] baseurl=https://apigee.googleapis.com/v1 -org=apigee-hybrid-378710 +org=xxx-xxx-xxx-xxx auth_type=oauth [csv] @@ -20,5 +20,5 @@ skip_proxy_list=mock1,stream api_env=dev api_name=target_server_validator vhost_domain_name=example.apigee.com -vhost_ip=34.134.171.41 +vhost_ip=x.x.x.x report_format=md From a83d4b018fcdb2fa71945e68e811c6d4967e420b Mon Sep 17 00:00:00 2001 From: anaik91 Date: Tue, 29 Aug 2023 18:48:21 +0530 Subject: [PATCH 07/14] feat: updated in-solidarity.yml --- .github/in-solidarity.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/in-solidarity.yml b/.github/in-solidarity.yml index 6fc13d49a..0b4dc36a1 100644 --- a/.github/in-solidarity.yml +++ b/.github/in-solidarity.yml @@ -14,3 +14,4 @@ ignore: - "tools/hybrid-quickstart/steps.sh" # because the GKE cli uses 'master' + - "tools/target-server-validator/callout/build_setup.sh" # because https://github.com/apigee/api-platform-samples uses 'master' branch From 733a807788c39677bf93f83d204521c4f584fffc Mon Sep 17 00:00:00 2001 From: anaik91 Date: Tue, 29 Aug 2023 18:53:08 +0530 Subject: [PATCH 08/14] fix: Fixed CODEOWNERS file --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index ef3d3cc1b..fafbcd98b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -38,4 +38,4 @@ /tools/pipeline-runner @seymen @danistrebel /tools/sf-dependency-list @yuriylesyuk /tools/proxy-endpoint-unifier @anaik91 -tools/target-server-validator @anaik91 +/tools/target-server-validator @anaik91 From 143d27dc89e8b1ea635037196992d809eeed64f0 Mon Sep 17 00:00:00 2001 From: anaik91 Date: Tue, 29 Aug 2023 21:29:41 +0530 Subject: [PATCH 09/14] feat: added the java-callout resource for target server validator --- .gitignore | 1 + .../java/edge-custom-policy-java-hello.jar | Bin 0 -> 2106 bytes 2 files changed, 1 insertion(+) create mode 100644 tools/target-server-validator/apiproxy/resources/java/edge-custom-policy-java-hello.jar diff --git a/.gitignore b/.gitignore index 05607f38d..2f2719b8a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ .vscode .DS_Store *.jar +!edge-custom-policy-java-hello.jar \ No newline at end of file diff --git a/tools/target-server-validator/apiproxy/resources/java/edge-custom-policy-java-hello.jar b/tools/target-server-validator/apiproxy/resources/java/edge-custom-policy-java-hello.jar new file mode 100644 index 0000000000000000000000000000000000000000..3029ab84f63dfd5d2e36a3eb481af88161f03d33 GIT binary patch literal 2106 zcmZ{lc{p478po5-v_WkZ!q|pN)e_4nwYDf35>d6*PBr$Tq^%{2+G?5h-XIh~W2>f? zmUb)?v|=y_ssuwVMJ-9y5>>Qwlj)sXkLmrM=RCjnJm>p8zxVz9aX!{4u0uQk02lyB z1wkDF`^3$z(PnnWa0@Ghnl;-70f52af7l@VZD?aF3xt`iohll!f$r_GM8Q=D`A~2r zT5o@yjruFi>G?1fB&|yY1vdd7nJab35>;-Kk7z7Nw-H9!*kXxFRS-$uW0{&ZA~@k5 zn+-??OJ(zDWMctRGaT9WB(+0){$Au*Wa_;!!5N5KO;%7b+K9ZF5%+8@3<$7B9RXvB zS|wN64W+PePRm~=OF1yvA@*DEkSl);^=*gHe;r<-{=OJYxYw1?Ak4lqIQT*NZ`wn` zu;)WD!RP!i-hry#L0;kE*$ytY!oFfR36k2}yx4)V9VWgyeZ^sdL&HwYZj@Ml=DgCX@4-`U!IU(yIOV) zS2rkA(ROq0z+^!lPhgNrChMo0`|L7$r$9Wo*K-_;`&Y z&zPtC$P*CEj}Kw3n8RAl9*Pe_x{~XIrC>q zcbN+^Ir>g3t5C_lr^Q_zvk%)tuEv@q3q=PU`O9q-T{Z6|JLo9A)=@%Mq1a7z#`{5}J6v64`bEXD(j5 zKx!JBhbJEZ(;lPx+mbCF1L9X}WNQb8$C>U) zn*oddB(mXbH9cCA()sd6v&HCR5ied|p%97#ww_eQYB7<}yqXC7D2@Yo|K5h&N;)j^IPMX3YKpWtMBwqPe_Ft&6@sMq~ zb#tNv5F<|N^xDDYxffAmmSg2V$8DdRy?guYiO-4Ks|I6AH@K_ne^`Ug%2X=8?K>BH z3%BAcAHdff;LSySN3U(x7M0tgJ~t(ssds65wGwNaX(=9&>x$YF{-i*^*uJ(2IT~2b z`J~ZONHn^<2MA9zI%1&Z;3D<7EPiy*oDrcsIAy-snjNvF^+t#eRDV=wFXy&*8%E}T zb@u24PXu-vruT848I?+eo9mDT81~dj=FNDw_QLyah0sx{HGV8}X{&|m4qYg^zw19M z9#uGQ3-#h4-HNvaOCFbz0lDu)zP>_$}c72eVsp07ysnY;7y0n z!$1%Kkig5K0q{Er*Z(IS`wz~RAb@GiS@&^|6Z=ZO{ea<+?{AgHE3nJ|#rAP<|HcQo z!2!`1aW)=jmtU~-wbg?O4w$}(lh|XIoWuc4eQlW&-+u}mQ2njacxB+X;{U{!H41c? PlLE0X19oQduc!Y2^lnp- literal 0 HcmV?d00001 From 18dfdb688f5749d1200afb051e09807d6d08a355 Mon Sep 17 00:00:00 2001 From: anaik91 Date: Tue, 29 Aug 2023 23:51:10 +0530 Subject: [PATCH 10/14] fix: fixed CODEOWNERS, Readme --- .github/in-solidarity.yml | 2 +- CODEOWNERS | 2 +- tools/target-server-validator/README.md | 15 ++++----------- .../callout/target/classes/PortOpenCheck.class | Bin 837 -> 0 bytes .../callout/target/classes/package-info.class | Bin 105 -> 0 bytes 5 files changed, 6 insertions(+), 13 deletions(-) delete mode 100644 tools/target-server-validator/callout/target/classes/PortOpenCheck.class delete mode 100644 tools/target-server-validator/callout/target/classes/package-info.class diff --git a/.github/in-solidarity.yml b/.github/in-solidarity.yml index 0b4dc36a1..0d17276a2 100644 --- a/.github/in-solidarity.yml +++ b/.github/in-solidarity.yml @@ -14,4 +14,4 @@ ignore: - "tools/hybrid-quickstart/steps.sh" # because the GKE cli uses 'master' - - "tools/target-server-validator/callout/build_setup.sh" # because https://github.com/apigee/api-platform-samples uses 'master' branch + - "tools/target-server-validator/callout/build_setup.sh" # because github.com/apigee/api-platform-samples uses voliating branch name diff --git a/CODEOWNERS b/CODEOWNERS index fafbcd98b..8c37d5c10 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -38,4 +38,4 @@ /tools/pipeline-runner @seymen @danistrebel /tools/sf-dependency-list @yuriylesyuk /tools/proxy-endpoint-unifier @anaik91 -/tools/target-server-validator @anaik91 +/tools/target-server-validator @anaik91 diff --git a/tools/target-server-validator/README.md b/tools/target-server-validator/README.md index a6bb541ca..a5fbeac37 100644 --- a/tools/target-server-validator/README.md +++ b/tools/target-server-validator/README.md @@ -1,15 +1,12 @@ # Apigee Target Server Validator -The objective of this tool to validate targets in Target Servers & Apigee API Proxy Bundles exported from Apigee OPDK/X/Hybrid. +The objective of this tool to validate targets in Target Servers & Apigee API Proxy Bundles exported from Apigee. Validation is done by deploying a sample proxy which check if HOST & PORT is open from Apigee OPDK/X/Hybrid. > **NOTE**: Discovery of Targets in API Proxy & Sharedflows is limited to only parsing URL from `TargetEndpoint` & `ServiceCallout` Policy. > **NOTE**: Dynamic targets are **NOT** supported, Ex : `https://host.{request.formparam.region}.example.com}` -## Disclaimer -This is not an Officially Supported Google Product! - ## Pre-Requisites * python3.x * Please Install required Python Libs @@ -59,7 +56,7 @@ smtp.gmail.com,465 ``` -* Please run below command to authenticate against Apigee X/Hybrid APIS +* Please run below command to authenticate against Apigee X/Hybrid APIs ``` export APIGEE_OPDK_ACCESS_TOKEN=$(echo -n ":" | base64) # Access token for Apigee OPDK @@ -75,16 +72,12 @@ smtp.gmail.com,465 ## Usage -Run the Script as below +Run the script as below ``` python3 main.py ``` ## Report -Validation Report : `report.md` OR `report.csv` can be accessed in same localtion as script. +Validation Report: `report.md` OR `report.csv` can be found in the same directory as the script. Please check a [Sample report](report.md) - -## Copyright - -Copyright 2023 Google LLC. This software is provided as-is, without warranty or representation for any use or purpose. Your use of it is subject to your agreement with Google. diff --git a/tools/target-server-validator/callout/target/classes/PortOpenCheck.class b/tools/target-server-validator/callout/target/classes/PortOpenCheck.class deleted file mode 100644 index 9680c39113e68de1df2ed2a82b57774584a17199..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 837 zcma)4O>fgc5Ph45I58;+f%1*y=F)^ieJCe_xJ0dl6eyyl632}*O}1To-SwvA%CABK z3GVzT#Mns*r4exOtY`Gzo0+%!{m1980Cw=$LrGx$jj?V&rKCTh=+r}5pm`$CWvFE` z4)=#A6gdwSfvw2Ip-k17C_L8YLr61;dn8cqn-O`q zF3?yC?^V?WClrh*($X>}mC>mjQ_yDc z_I#d9#;K-uFfx<{iE%+JT{H9JRz7HRbIy^K5HadglPzTPcWZcV#ZVXgH zucz^l?4jg21X=?VNqr=(%JW6B>?SJ9(U)B83Dnp^-lSk|t?2Qxd$n&B_XIZoX@`I} zx1NXk*ctqJEiq*o`xCyJpM8@!nz`N;-17Q Date: Wed, 30 Aug 2023 00:57:08 +0530 Subject: [PATCH 11/14] feat: added javacallout build to pipeline --- tools/target-server-validator/README.md | 17 ++++++++++++---- tools/target-server-validator/apigee_utils.py | 19 +++++++++++------- .../java/edge-custom-policy-java-hello.jar | Bin 2106 -> 0 bytes .../{build_setup.sh => build_java_callout.sh} | 6 +++++- .../target-server-validator/input.properties | 7 ++++--- tools/target-server-validator/main.py | 2 ++ tools/target-server-validator/pipeline.sh | 3 +++ 7 files changed, 39 insertions(+), 15 deletions(-) delete mode 100644 tools/target-server-validator/apiproxy/resources/java/edge-custom-policy-java-hello.jar rename tools/target-server-validator/callout/{build_setup.sh => build_java_callout.sh} (91%) diff --git a/tools/target-server-validator/README.md b/tools/target-server-validator/README.md index a5fbeac37..0160adc3e 100644 --- a/tools/target-server-validator/README.md +++ b/tools/target-server-validator/README.md @@ -9,11 +9,19 @@ Validation is done by deploying a sample proxy which check if HOST & PORT is ope ## Pre-Requisites * python3.x -* Please Install required Python Libs +* Java +* mvn +* Please install required Python Libs ``` - python3 -m pip install requirements.txt +python3 -m pip install requirements.txt ``` +* Please build the java callout jar by running the below command + +``` +bash callout/build_java_callout.sh +``` + * Please fill in `input.properties` ``` @@ -38,6 +46,7 @@ skip_proxy_list=mock1,stream # Comma sperated list of proxies to proxy_export_dir=export # Export directory needed when check_proxies='true' api_env=dev # Target Environment to deploy Validation API Proxy api_name=target_server_validator # Target API Name of Validation API Proxy +api_force_redeploy=false # set 'true' to Re-deploy Target API Proxy vhost_domain_name=devgroup # Target VHost or EnvGroup vhost_ip= # IP address corresponding to vhost_domain_name. Use if DNS record doesnt exist report_format=csv # Report Format. Choose csv or md (Markdown) @@ -59,8 +68,8 @@ smtp.gmail.com,465 * Please run below command to authenticate against Apigee X/Hybrid APIs ``` - export APIGEE_OPDK_ACCESS_TOKEN=$(echo -n ":" | base64) # Access token for Apigee OPDK - export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token) # Access token for Apigee X +export APIGEE_OPDK_ACCESS_TOKEN=$(echo -n ":" | base64) # Access token for Apigee OPDK +export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token) # Access token for Apigee X ``` ## Highlevel Working diff --git a/tools/target-server-validator/apigee_utils.py b/tools/target-server-validator/apigee_utils.py index 3516b29e2..89edd3ebe 100644 --- a/tools/target-server-validator/apigee_utils.py +++ b/tools/target-server-validator/apigee_utils.py @@ -133,10 +133,10 @@ def create_api(self, api_name, proxy_bundle_path): "POST", url, headers=headers, data={}, files=files ) if response.status_code == 200: - return True - else: - print(response.json()) - return False + revision = response.json().get('revision', "1") + return True, revision + print(response.text) + return False, None def get_api_revisions_deployment(self, env, api_name, api_rev): # noqa url = ( @@ -156,6 +156,7 @@ def get_api_revisions_deployment(self, env, api_name, api_rev): # noqa print(f"API {api_name} is in Status: {api_deployment_status} !") # noqa return False else: + print(response.text) return False def deploy_api(self, env, api_name, api_rev): @@ -171,9 +172,10 @@ def deploy_api(self, env, api_name, api_rev): if "already deployed" in resp["error"]["message"]: print("Proxy {} is already Deployed".format(api_name)) return True + print(response.text) return False - def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_rev=1): # noqa + def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_rev=1, api_force_redeploy=False): # noqa api_deployment_retry = 60 api_deployment_sleep = 5 api_deployment_retry_count = 0 @@ -183,8 +185,11 @@ def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_rev=1): # noq f"Proxy with name {api_name} already exists in Apigee Org {self.org}" # noqa ) api_exists = True - else: - if self.create_api(api_name, proxy_bundle_path): + if api_force_redeploy: + api_exists = False + if not api_exists: + api_created, api_rev = self.create_api(api_name, proxy_bundle_path) + if api_created: print( f"Proxy has been imported with name {api_name} in Apigee Org {self.org}" # noqa ) diff --git a/tools/target-server-validator/apiproxy/resources/java/edge-custom-policy-java-hello.jar b/tools/target-server-validator/apiproxy/resources/java/edge-custom-policy-java-hello.jar deleted file mode 100644 index 3029ab84f63dfd5d2e36a3eb481af88161f03d33..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2106 zcmZ{lc{p478po5-v_WkZ!q|pN)e_4nwYDf35>d6*PBr$Tq^%{2+G?5h-XIh~W2>f? zmUb)?v|=y_ssuwVMJ-9y5>>Qwlj)sXkLmrM=RCjnJm>p8zxVz9aX!{4u0uQk02lyB z1wkDF`^3$z(PnnWa0@Ghnl;-70f52af7l@VZD?aF3xt`iohll!f$r_GM8Q=D`A~2r zT5o@yjruFi>G?1fB&|yY1vdd7nJab35>;-Kk7z7Nw-H9!*kXxFRS-$uW0{&ZA~@k5 zn+-??OJ(zDWMctRGaT9WB(+0){$Au*Wa_;!!5N5KO;%7b+K9ZF5%+8@3<$7B9RXvB zS|wN64W+PePRm~=OF1yvA@*DEkSl);^=*gHe;r<-{=OJYxYw1?Ak4lqIQT*NZ`wn` zu;)WD!RP!i-hry#L0;kE*$ytY!oFfR36k2}yx4)V9VWgyeZ^sdL&HwYZj@Ml=DgCX@4-`U!IU(yIOV) zS2rkA(ROq0z+^!lPhgNrChMo0`|L7$r$9Wo*K-_;`&Y z&zPtC$P*CEj}Kw3n8RAl9*Pe_x{~XIrC>q zcbN+^Ir>g3t5C_lr^Q_zvk%)tuEv@q3q=PU`O9q-T{Z6|JLo9A)=@%Mq1a7z#`{5}J6v64`bEXD(j5 zKx!JBhbJEZ(;lPx+mbCF1L9X}WNQb8$C>U) zn*oddB(mXbH9cCA()sd6v&HCR5ied|p%97#ww_eQYB7<}yqXC7D2@Yo|K5h&N;)j^IPMX3YKpWtMBwqPe_Ft&6@sMq~ zb#tNv5F<|N^xDDYxffAmmSg2V$8DdRy?guYiO-4Ks|I6AH@K_ne^`Ug%2X=8?K>BH z3%BAcAHdff;LSySN3U(x7M0tgJ~t(ssds65wGwNaX(=9&>x$YF{-i*^*uJ(2IT~2b z`J~ZONHn^<2MA9zI%1&Z;3D<7EPiy*oDrcsIAy-snjNvF^+t#eRDV=wFXy&*8%E}T zb@u24PXu-vruT848I?+eo9mDT81~dj=FNDw_QLyah0sx{HGV8}X{&|m4qYg^zw19M z9#uGQ3-#h4-HNvaOCFbz0lDu)zP>_$}c72eVsp07ysnY;7y0n z!$1%Kkig5K0q{Er*Z(IS`wz~RAb@GiS@&^|6Z=ZO{ea<+?{AgHE3nJ|#rAP<|HcQo z!2!`1aW)=jmtU~-wbg?O4w$}(lh|XIoWuc4eQlW&-+u}mQ2njacxB+X;{U{!H41c? PlLE0X19oQduc!Y2^lnp- diff --git a/tools/target-server-validator/callout/build_setup.sh b/tools/target-server-validator/callout/build_java_callout.sh similarity index 91% rename from tools/target-server-validator/callout/build_setup.sh rename to tools/target-server-validator/callout/build_java_callout.sh index 9684fef6f..4b8fb85e5 100644 --- a/tools/target-server-validator/callout/build_setup.sh +++ b/tools/target-server-validator/callout/build_java_callout.sh @@ -15,6 +15,7 @@ # limitations under the License. +JAVA_SRC_PATH="$( cd "$(dirname "$0")" || exit >/dev/null 2>&1 ; pwd -P )" echo echo "This script downloads JAR files and installs them into the local Maven repo." echo @@ -45,4 +46,7 @@ rm message-flow-1.0.0.jar echo echo done. -echo \ No newline at end of file +echo + +cd "$JAVA_SRC_PATH" || exit 1 +mvn clean install diff --git a/tools/target-server-validator/input.properties b/tools/target-server-validator/input.properties index 2105b7411..912e21973 100644 --- a/tools/target-server-validator/input.properties +++ b/tools/target-server-validator/input.properties @@ -1,11 +1,11 @@ [source] baseurl=https://apigee.googleapis.com/v1 -org=xxx-xxx-xxx-xxx +org=apigee-hybrid-378710 auth_type=oauth [target] baseurl=https://apigee.googleapis.com/v1 -org=xxx-xxx-xxx-xxx +org=apigee-hybrid-378710 auth_type=oauth [csv] @@ -19,6 +19,7 @@ proxy_export_dir=export skip_proxy_list=mock1,stream api_env=dev api_name=target_server_validator +api_force_redeploy=true vhost_domain_name=example.apigee.com -vhost_ip=x.x.x.x +vhost_ip=34.134.171.41 report_format=md diff --git a/tools/target-server-validator/main.py b/tools/target-server-validator/main.py index 096b18356..b954f24c9 100644 --- a/tools/target-server-validator/main.py +++ b/tools/target-server-validator/main.py @@ -146,6 +146,8 @@ def main(): cfg["validation"]["api_env"], cfg["validation"]["api_name"], f"{bundle_path}/{cfg['validation']['api_name']}.zip", + 1, + cfg["validation"].getboolean("api_force_redeploy", False) ): print(f"Proxy: {cfg['validation']['api_name']} deployment failed.") sys.exit(1) diff --git a/tools/target-server-validator/pipeline.sh b/tools/target-server-validator/pipeline.sh index 05842d420..d0f557002 100755 --- a/tools/target-server-validator/pipeline.sh +++ b/tools/target-server-validator/pipeline.sh @@ -18,6 +18,8 @@ set -e SCRIPTPATH="$( cd "$(dirname "$0")" || exit >/dev/null 2>&1 ; pwd -P )" +bash "$SCRIPTPATH/callout/build_java_callout.sh" + # Clean up previously generated files rm -rf "$SCRIPTPATH/input.properties" rm -rf "$SCRIPTPATH/export" @@ -46,6 +48,7 @@ proxy_export_dir=export skip_proxy_list= api_env=$APIGEE_X_ENV api_name=target_server_validator +api_force_redeploy=true vhost_domain_name=$APIGEE_X_HOSTNAME vhost_ip= report_format=md From a56f446be3e740dec9b26d74966c653946ba9993 Mon Sep 17 00:00:00 2001 From: anaik91 Date: Wed, 30 Aug 2023 19:38:39 +0530 Subject: [PATCH 12/14] fix: fixed policy naming & gitignore --- .gitignore | 1 - .../{set-json-response.xml => AM-Set-Json-Response.xml} | 2 +- .../apiproxy/policies/{JC1.xml => JC-Port-Open-Check.xml} | 4 ++-- tools/target-server-validator/apiproxy/proxies/default.xml | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) rename tools/target-server-validator/apiproxy/policies/{set-json-response.xml => AM-Set-Json-Response.xml} (95%) rename tools/target-server-validator/apiproxy/policies/{JC1.xml => JC-Port-Open-Check.xml} (86%) diff --git a/.gitignore b/.gitignore index 2f2719b8a..05607f38d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,3 @@ .vscode .DS_Store *.jar -!edge-custom-policy-java-hello.jar \ No newline at end of file diff --git a/tools/target-server-validator/apiproxy/policies/set-json-response.xml b/tools/target-server-validator/apiproxy/policies/AM-Set-Json-Response.xml similarity index 95% rename from tools/target-server-validator/apiproxy/policies/set-json-response.xml rename to tools/target-server-validator/apiproxy/policies/AM-Set-Json-Response.xml index 7a7da0e47..29a8f0c52 100644 --- a/tools/target-server-validator/apiproxy/policies/set-json-response.xml +++ b/tools/target-server-validator/apiproxy/policies/AM-Set-Json-Response.xml @@ -11,7 +11,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + { "host":"{request.header.host_name}", diff --git a/tools/target-server-validator/apiproxy/policies/JC1.xml b/tools/target-server-validator/apiproxy/policies/JC-Port-Open-Check.xml similarity index 86% rename from tools/target-server-validator/apiproxy/policies/JC1.xml rename to tools/target-server-validator/apiproxy/policies/JC-Port-Open-Check.xml index d3b8e5544..feb508004 100644 --- a/tools/target-server-validator/apiproxy/policies/JC1.xml +++ b/tools/target-server-validator/apiproxy/policies/JC-Port-Open-Check.xml @@ -11,8 +11,8 @@ See the License for the specific language governing permissions and limitations under the License. --> - - JC1 + + JC-Port-Open-Check com.apigeesample.PortOpenCheck java://edge-custom-policy-java-hello.jar diff --git a/tools/target-server-validator/apiproxy/proxies/default.xml b/tools/target-server-validator/apiproxy/proxies/default.xml index 539ee0d85..2e50181da 100644 --- a/tools/target-server-validator/apiproxy/proxies/default.xml +++ b/tools/target-server-validator/apiproxy/proxies/default.xml @@ -15,12 +15,12 @@ - JC1 + JC-Port-Open-Check - set-json-response + AM-Set-Json-Response From 2d9387af1f71b67b16e3c3aa8cd9301c22510ec3 Mon Sep 17 00:00:00 2001 From: anaik91 Date: Wed, 6 Sep 2023 21:28:02 +0530 Subject: [PATCH 13/14] fix: addressed PR comments --- tools/target-server-validator/README.md | 35 +++++++++---------- .../policies/AM-Set-Json-Response.xml | 2 +- .../apiproxy/policies/JC-Port-Open-Check.xml | 6 ++-- .../apiproxy/target_server_validator.xml | 18 +--------- tools/target-server-validator/callout/pom.xml | 4 +-- .../callout/src/main/java/PortOpenCheck.java | 18 +++++++--- .../callout/src/main/java/package-info.java | 2 +- .../target-server-validator/input.properties | 8 ++--- tools/target-server-validator/main.py | 7 ++-- 9 files changed, 46 insertions(+), 54 deletions(-) diff --git a/tools/target-server-validator/README.md b/tools/target-server-validator/README.md index 0160adc3e..8b7b24b0d 100644 --- a/tools/target-server-validator/README.md +++ b/tools/target-server-validator/README.md @@ -1,20 +1,19 @@ # Apigee Target Server Validator The objective of this tool to validate targets in Target Servers & Apigee API Proxy Bundles exported from Apigee. -Validation is done by deploying a sample proxy which check if HOST & PORT is open from Apigee OPDK/X/Hybrid. +Validation is done by deploying a sample proxy which check if HOST & PORT is open from Apigee. > **NOTE**: Discovery of Targets in API Proxy & Sharedflows is limited to only parsing URL from `TargetEndpoint` & `ServiceCallout` Policy. > **NOTE**: Dynamic targets are **NOT** supported, Ex : `https://host.{request.formparam.region}.example.com}` ## Pre-Requisites -* python3.x +* Python3.x * Java -* mvn -* Please install required Python Libs - +* Maven +* Please install the required Python dependencies ``` -python3 -m pip install requirements.txt + python3 -m pip install -r requirements.txt ``` * Please build the java callout jar by running the below command @@ -26,13 +25,13 @@ bash callout/build_java_callout.sh ``` [source] -baseurl=http://34.131.144.184:8080/v1 # Apigee OPDK/Edge/X/Hybrid Base URL -org=xxx-xxxx-xxx-xxxxx # Apigee OPDK/Edge/X/Hybrid Org +baseurl=https://x.x.x.x/v1 # Apigee Base URL. e.g http://management-api.apigee-opdk.corp:8080 +org=xxx-xxxx-xxx-xxxxx # Apigee Org ID auth_type=basic # API Auth type basic | oauth [target] -baseurl=https://apigee.googleapis.com/v1 # Apigee OPDK/Edge/X/Hybrid Base URL -org=xxx-xxxx-xxx-xxxxx # Apigee OPDK/Edge/X/Hybrid Org Id +baseurl=https://apigee.googleapis.com/v1 # Apigee Base URL +org=xxx-xxxx-xxx-xxxxx # Apigee Org ID auth_type=oauth # API Auth type basic | oauth [csv] @@ -47,16 +46,16 @@ proxy_export_dir=export # Export directory needed when check api_env=dev # Target Environment to deploy Validation API Proxy api_name=target_server_validator # Target API Name of Validation API Proxy api_force_redeploy=false # set 'true' to Re-deploy Target API Proxy -vhost_domain_name=devgroup # Target VHost or EnvGroup -vhost_ip= # IP address corresponding to vhost_domain_name. Use if DNS record doesnt exist -report_format=csv # Report Format. Choose csv or md (Markdown) +api_hostname=example.apigee.com # Target VirtualHost or EnvGroup Domain Name +api_ip= # IP address corresponding to api_hostname. Use if DNS record doesnt exist +report_format=csv # Report Format. Choose csv or md (defaults to md) ``` * Sample input CSV with target servers > **NOTE:** You need to set `check_csv=true` in the `validation` section of `input.properties` > **NOTE:** You need to set `file=` in the `csv` section of `input.properties` - +> If PORT is omitted from the csv, the value of default_port will be used from `input.properties`. ``` HOST,PORT httpbin.org @@ -65,19 +64,19 @@ smtp.gmail.com,465 ``` -* Please run below command to authenticate against Apigee X/Hybrid APIs +* Please run below commands to authenticate, based on the Apigee flavours you are using. ``` export APIGEE_OPDK_ACCESS_TOKEN=$(echo -n ":" | base64) # Access token for Apigee OPDK -export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token) # Access token for Apigee X +export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token) # Access token for Apigee X/Hybrid ``` ## Highlevel Working * Export Target Server Details * Export Proxy Bundle * Parse Each Proxy Bundle for Target -* Run Validate API against each Target -* Generate CSV Report +* Run Validate API against each Target (optional) +* Generate csv/md Report ## Usage diff --git a/tools/target-server-validator/apiproxy/policies/AM-Set-Json-Response.xml b/tools/target-server-validator/apiproxy/policies/AM-Set-Json-Response.xml index 29a8f0c52..df4a122b0 100644 --- a/tools/target-server-validator/apiproxy/policies/AM-Set-Json-Response.xml +++ b/tools/target-server-validator/apiproxy/policies/AM-Set-Json-Response.xml @@ -16,7 +16,7 @@ { "host":"{request.header.host_name}", "port": "{request.header.port_number}", - "status":"{REACHABLE_STATUS}" + "status":"{flow.reachableStatus}" } diff --git a/tools/target-server-validator/apiproxy/policies/JC-Port-Open-Check.xml b/tools/target-server-validator/apiproxy/policies/JC-Port-Open-Check.xml index feb508004..cd2061782 100644 --- a/tools/target-server-validator/apiproxy/policies/JC-Port-Open-Check.xml +++ b/tools/target-server-validator/apiproxy/policies/JC-Port-Open-Check.xml @@ -12,8 +12,6 @@ limitations under the License. --> - JC-Port-Open-Check - - com.apigeesample.PortOpenCheck - java://edge-custom-policy-java-hello.jar + com.apigee.devrel.apigee_target_server_validator.PortOpenCheck + java://jc-target-server-validator.jar \ No newline at end of file diff --git a/tools/target-server-validator/apiproxy/target_server_validator.xml b/tools/target-server-validator/apiproxy/target_server_validator.xml index ff411df92..a86e9f5ac 100644 --- a/tools/target-server-validator/apiproxy/target_server_validator.xml +++ b/tools/target-server-validator/apiproxy/target_server_validator.xml @@ -11,20 +11,4 @@ See the License for the specific language governing permissions and limitations under the License. --> - - - - 1682421435781 - 1682421435781 - /validate_target_server - - JC1 - set-json-response - - - default - - - java://edge-custom-policy-java-hello.jar - - \ No newline at end of file + diff --git a/tools/target-server-validator/callout/pom.xml b/tools/target-server-validator/callout/pom.xml index 797024c9a..90b665844 100644 --- a/tools/target-server-validator/callout/pom.xml +++ b/tools/target-server-validator/callout/pom.xml @@ -16,9 +16,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> 4.0.0 com.apigee.callout - edge-custom-policy-java-hello + jc-target-server-validator 1.0-SNAPSHOT - EdgeCustomJavaHello + JavaTargetServerValidator http://maven.apache.org jar diff --git a/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java b/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java index f8a564cb0..c04630ba5 100644 --- a/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java +++ b/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java @@ -13,7 +13,7 @@ // limitations under the License. -package com.apigeesample; +package com.apigee.devrel.apigee_target_server_validator; import com.apigee.flow.execution.ExecutionContext; import com.apigee.flow.execution.ExecutionResult; @@ -24,6 +24,8 @@ import java.net.Socket; import java.net.SocketTimeoutException; import java.net.UnknownHostException; +import com.apigee.flow.execution.Action; + /** * A callout that checks if a particular port is open on a specified host. @@ -54,7 +56,7 @@ private static String available(final String host, final int port) { try { socket.close(); } catch (IOException e) { - throw new RuntimeException("You should handle this error.", e); + throw new RuntimeException("Exception occured", e); } } } @@ -75,10 +77,18 @@ public ExecutionResult execute(final MessageContext messageContext, int portnumber = Integer.parseInt(port); String status = available(hostname, portnumber); // messageContext.getMessage().setContent(Status); - messageContext.setVariable("REACHABLE_STATUS", status); + messageContext.setVariable("flow.reachableStatus", status); return ExecutionResult.SUCCESS; } catch (Exception e) { - return ExecutionResult.ABORT; + ExecutionResult executionResult = new ExecutionResult(false, + Action.ABORT); + //--Returns custom error message and header + executionResult.setErrorResponse(e.getMessage()); + executionResult.addErrorResponseHeader("ExceptionClass", + e.getClass().getName()); + //--Set flow variables -- may be useful for debugging. + messageContext.setVariable("JAVA_ERROR", e.getMessage()); + return executionResult; } } } diff --git a/tools/target-server-validator/callout/src/main/java/package-info.java b/tools/target-server-validator/callout/src/main/java/package-info.java index 019106599..d1e2a1829 100644 --- a/tools/target-server-validator/callout/src/main/java/package-info.java +++ b/tools/target-server-validator/callout/src/main/java/package-info.java @@ -19,4 +19,4 @@ * @author anaik91 * @version .01 */ -package com.apigeesample; +package com.apigee.devrel.apigee_target_server_validator; diff --git a/tools/target-server-validator/input.properties b/tools/target-server-validator/input.properties index 912e21973..093b4fab8 100644 --- a/tools/target-server-validator/input.properties +++ b/tools/target-server-validator/input.properties @@ -1,11 +1,11 @@ [source] baseurl=https://apigee.googleapis.com/v1 -org=apigee-hybrid-378710 +org=xxx-xxx-xxx auth_type=oauth [target] baseurl=https://apigee.googleapis.com/v1 -org=apigee-hybrid-378710 +org=xxx-xxx-xxx auth_type=oauth [csv] @@ -20,6 +20,6 @@ skip_proxy_list=mock1,stream api_env=dev api_name=target_server_validator api_force_redeploy=true -vhost_domain_name=example.apigee.com -vhost_ip=34.134.171.41 +api_hostname=example.apigee.com +api_ip=x.x.x.x report_format=md diff --git a/tools/target-server-validator/main.py b/tools/target-server-validator/main.py index b954f24c9..5d061a7a8 100644 --- a/tools/target-server-validator/main.py +++ b/tools/target-server-validator/main.py @@ -157,10 +157,10 @@ def main(): # Fetch API Northbound Endpoint print( - f"INFO: Fetching VHost with name {cfg['validation']['vhost_domain_name']} !" # noqa + f"INFO: Fetching VHost with name {cfg['validation']['api_hostname']} !" # noqa ) - vhost_domain_name = cfg["validation"]["vhost_domain_name"] - vhost_ip = cfg["validation"].get("vhost_ip", "").strip() + vhost_domain_name = cfg["validation"]["api_hostname"] + vhost_ip = cfg["validation"].get("api_ip", "").strip() api_url = f"https://{vhost_domain_name}/validate_target_server" final_report = [] _cached_hosts = {} @@ -254,6 +254,7 @@ def main(): ) # Write CSV Report + # TODO: support relative report path if report_format == "csv": report_file = "report.csv" print(f"INFO: Dumping report to file {report_file}") From 8efa176bae176173f7d239939b2c3ceb554df17d Mon Sep 17 00:00:00 2001 From: anaik91 Date: Thu, 7 Sep 2023 19:34:59 +0530 Subject: [PATCH 14/14] fix: fixed PR comments --- .github/in-solidarity.yml | 2 +- .../apiproxy/proxies/default.xml | 2 +- ...erver_validator.xml => target-server-validator.xml} | 2 +- .../callout/src/main/java/PortOpenCheck.java | 1 - tools/target-server-validator/input.properties | 5 +++-- tools/target-server-validator/main.py | 6 ++++-- tools/target-server-validator/pipeline.sh | 7 ++++--- tools/target-server-validator/utilities.py | 10 +++++----- 8 files changed, 19 insertions(+), 16 deletions(-) rename tools/target-server-validator/apiproxy/{target_server_validator.xml => target-server-validator.xml} (93%) diff --git a/.github/in-solidarity.yml b/.github/in-solidarity.yml index 0d17276a2..fdbd72ad8 100644 --- a/.github/in-solidarity.yml +++ b/.github/in-solidarity.yml @@ -14,4 +14,4 @@ ignore: - "tools/hybrid-quickstart/steps.sh" # because the GKE cli uses 'master' - - "tools/target-server-validator/callout/build_setup.sh" # because github.com/apigee/api-platform-samples uses voliating branch name + - "tools/target-server-validator/callout/build_java_callout.sh" # because github.com/apigee/api-platform-samples uses voliating branch name diff --git a/tools/target-server-validator/apiproxy/proxies/default.xml b/tools/target-server-validator/apiproxy/proxies/default.xml index 2e50181da..10d49b91f 100644 --- a/tools/target-server-validator/apiproxy/proxies/default.xml +++ b/tools/target-server-validator/apiproxy/proxies/default.xml @@ -30,7 +30,7 @@ - /validate_target_server + /validate-target-server \ No newline at end of file diff --git a/tools/target-server-validator/apiproxy/target_server_validator.xml b/tools/target-server-validator/apiproxy/target-server-validator.xml similarity index 93% rename from tools/target-server-validator/apiproxy/target_server_validator.xml rename to tools/target-server-validator/apiproxy/target-server-validator.xml index a86e9f5ac..f7457d683 100644 --- a/tools/target-server-validator/apiproxy/target_server_validator.xml +++ b/tools/target-server-validator/apiproxy/target-server-validator.xml @@ -11,4 +11,4 @@ See the License for the specific language governing permissions and limitations under the License. --> - + diff --git a/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java b/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java index c04630ba5..a31b6da86 100644 --- a/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java +++ b/tools/target-server-validator/callout/src/main/java/PortOpenCheck.java @@ -76,7 +76,6 @@ public ExecutionResult execute(final MessageContext messageContext, String port = messageContext.getMessage().getHeader("port_number"); int portnumber = Integer.parseInt(port); String status = available(hostname, portnumber); - // messageContext.getMessage().setContent(Status); messageContext.setVariable("flow.reachableStatus", status); return ExecutionResult.SUCCESS; } catch (Exception e) { diff --git a/tools/target-server-validator/input.properties b/tools/target-server-validator/input.properties index 093b4fab8..e15ba1439 100644 --- a/tools/target-server-validator/input.properties +++ b/tools/target-server-validator/input.properties @@ -18,8 +18,9 @@ check_proxies=true proxy_export_dir=export skip_proxy_list=mock1,stream api_env=dev -api_name=target_server_validator +api_name=target-server-validator api_force_redeploy=true api_hostname=example.apigee.com -api_ip=x.x.x.x +api_ip= report_format=md +allow_insecure=false \ No newline at end of file diff --git a/tools/target-server-validator/main.py b/tools/target-server-validator/main.py index 5d061a7a8..72410441d 100644 --- a/tools/target-server-validator/main.py +++ b/tools/target-server-validator/main.py @@ -41,6 +41,7 @@ def main(): check_proxies = cfg["validation"].getboolean("check_proxies") proxy_export_dir = cfg["validation"]["proxy_export_dir"] report_format = cfg["validation"]["report_format"] + allow_insecure = cfg["validation"].getboolean("allow_insecure") if report_format not in ["csv", "md"]: report_format = "md" @@ -169,7 +170,7 @@ def main(): print("INFO: Running validation against All Target Servers") for each_ts in all_target_servers: status = run_validator_proxy( - api_url, vhost_domain_name, vhost_ip, each_ts["host"], each_ts["port"] # noqa + api_url, vhost_domain_name, vhost_ip, each_ts["host"], each_ts["port"], allow_insecure # noqa ) final_report.append( [ @@ -210,6 +211,7 @@ def main(): vhost_ip, each_target["host"], each_target["port"], + allow_insecure, ) _cached_hosts[ f"{each_target['host']}:{each_target['port']}" @@ -238,7 +240,7 @@ def main(): status = _cached_hosts[f"{each_host}:{each_port}"] else: status = run_validator_proxy( - api_url, vhost_domain_name, vhost_ip, each_host, each_port + api_url, vhost_domain_name, vhost_ip, each_host, each_port, allow_insecure # noqa ) _cached_hosts[f"{each_host}:{each_port}"] = status final_report.append( diff --git a/tools/target-server-validator/pipeline.sh b/tools/target-server-validator/pipeline.sh index d0f557002..2e1fe7923 100755 --- a/tools/target-server-validator/pipeline.sh +++ b/tools/target-server-validator/pipeline.sh @@ -49,9 +49,10 @@ skip_proxy_list= api_env=$APIGEE_X_ENV api_name=target_server_validator api_force_redeploy=true -vhost_domain_name=$APIGEE_X_HOSTNAME -vhost_ip= +api_hostname=$APIGEE_X_HOSTNAME +api_ip= report_format=md +allow_insecure=false EOF # Generate optional input csv file @@ -59,7 +60,7 @@ cat > "$SCRIPTPATH/input.csv" << EOF HOST,PORT httpbin.org httpbin.org,443 -mocktarget.apigee.tom +domaindoesntexist.apigee.tom smtp.gmail.com,465 EOF diff --git a/tools/target-server-validator/utilities.py b/tools/target-server-validator/utilities.py index 6309736c7..c96feb169 100644 --- a/tools/target-server-validator/utilities.py +++ b/tools/target-server-validator/utilities.py @@ -26,8 +26,6 @@ import urllib3 from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - def parse_config(config_file): config = configparser.ConfigParser() @@ -55,19 +53,21 @@ def create_proxy_bundle(proxy_bundle_directory, api_name, target_dir): # noqa def run_validator_proxy( - url, dns_host, vhost_ip, target_host, target_port="443" -): # noqa + url, dns_host, vhost_ip, target_host, target_port="443", allow_insecure=False): # noqa headers = { "host_name": target_host, "port_number": str(target_port), "Host": dns_host, } + if allow_insecure: + print("INFO: Skipping Certificate Verification & disabling warnings because 'allow_insecure' is set to true") # noqa + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) session = requests.Session() if len(vhost_ip) > 0: session.mount( f"https://{dns_host}", ForcedIPHTTPSAdapter(dest_ip=vhost_ip) ) # noqa - r = session.get(url, headers=headers, verify=False) + r = session.get(url, headers=headers, verify=(not allow_insecure)) if r.status_code == 200: return r.json()["status"] return "STATUS_UNKNOWN"