Merge branch 'main' into dependabot/npm_and_yarn/references/cicd-shar…

…edflow-pipeline/crypto-js-4.2.0

tools/proxy-endpoint-unifier/main.py

@@ -32,14 +32,13 @@ def main():
     utils.create_dir(proxy_bundle_directory)
     proxy_endpoint_count = utils.get_proxy_endpoint_count(cfg)
     proxies = utils.list_dir(proxy_dir)
-
     final_dict = {}
     processed_dict = {}
 
     for each_dir in proxies:
         each_proxy_dict = utils.read_proxy_artifacts(
-                        f"{proxy_dir}/{each_dir}",
-                        utils.parse_proxy_root(f"{proxy_dir}/{each_dir}")
+                        f"{proxy_dir}/{each_dir}/apiproxy",
+                        utils.parse_proxy_root(f"{proxy_dir}/{each_dir}/apiproxy")
                             )
         if len(each_proxy_dict) > 0:
             each_proxy_rel = utils.get_proxy_objects_relationships(
@@ -91,7 +90,7 @@ def main():
     for each_api, grouped_api in bundled_group.items():
         for index, each_group in enumerate(grouped_api):
             utils.clone_proxies(
-                    f"{proxy_dir}/{each_api}",
+                    f"{proxy_dir}/{each_api}/apiproxy",
                     f"{proxy_dest_dir}/{each_api}_{index}",
                     merged_objects[f"{each_api}_{index}"],
                     merged_pes,

tools/proxy-endpoint-unifier/pipeline.sh

@@ -38,15 +38,19 @@ gcp_project_id=$APIGEE_X_ORG
 EOF
 
 # Install Dependencies
-python3 -m pip install -r "$SCRIPTPATH/requirements.txt"
+VENV_PATH="$SCRIPTPATH/venv"
+python3 -m venv "$VENV_PATH"
+# shellcheck source=/dev/null
+. "$VENV_PATH/bin/activate"
+pip install -r "$SCRIPTPATH/requirements.txt"
 
 # Generate Gcloud Acccess Token
 APIGEE_ACCESS_TOKEN="$(gcloud config config-helper --force-auth-refresh --format json | jq -r '.credential.access_token')"
 export APIGEE_ACCESS_TOKEN
 
 # Building API Proxy Bundle for Proxy containing more than 5 Proxy Endpoints
-cd "$SCRIPTPATH/test/api_bundles"
-rm -rf "$SCRIPTPATH/test/api_bundles/test.zip"
+cd "$SCRIPTPATH/test/api_bundles/test-proxy"
+rm -rf "$SCRIPTPATH/test/api_bundles/test-proxy/test.zip"
 echo "Building original proxy bundle"
 zip -q -r test.zip apiproxy/
 cd "$SCRIPTPATH"
@@ -57,9 +61,13 @@ python3 -c "import os, sys ,json; \
             from apigee import Apigee; \
             x = Apigee(os.getenv('APIGEE_X_ORG')); \
             x.set_auth_header(os.getenv('APIGEE_ACCESS_TOKEN')); \
-            r=x.validate_api('apis','test/api_bundles/test.zip'); \
+            r=x.validate_api('apis','test/api_bundles/test-proxy/test.zip'); \
             print(json.dumps(r,indent=2))"
 rm -rf "$SCRIPTPATH/test/api_bundles/test.zip"
 
 # Running and Validating API Proxy Bundle after splitting the proxies
 python3 "$SCRIPTPATH/main.py"
+
+# deactivate venv & cleanup
+deactivate
+rm -rf "$VENV_PATH"

tools/target-server-validator/README.md

@@ -85,6 +85,39 @@ Run the script as below
 python3 main.py
 ```
 
+This script deploys an API proxy to validate if the target servers are reachable or not. To use the API proxy, make sure your payloads adhere to the following format:
+
+```json
+[
+    {
+        "host": "example.com",
+        "port": 443
+    },
+    {
+        "host": "example2.com",
+        "port": 443
+    },
+    // Add up to 8 more host-port combinations as needed
+]
+```
+
+The response will look like this - 
+```json
+[
+    {
+        "host": "example.com",
+        "port": 443,
+        "status" : "REACHABLE"
+    },
+    {
+        "host": "example2.com",
+        "port": 443,
+         "status" : "UNKNOWN_HOST"
+    },
+    // and so on 
+]
+```
+
 ## Report
 Validation Report: `report.md` OR `report.csv` can be found in the same directory as the script.
 

tools/target-server-validator/apigee_utils.py

@@ -20,6 +20,13 @@
 import requests
 import shutil
 from time import sleep
+from utilities import (  # pylint: disable=import-error
+    run_validator_proxy,
+    unzip_file,
+    parse_proxy_hosts,
+    get_tes,
+)
+from base_logger import logger
 
 
 class Apigee:
@@ -36,16 +43,16 @@ def __init__(
         self.auth_type = auth_type
         access_token = self.get_access_token()
         self.auth_header = {
-            "Authorization": "Bearer {}".format(access_token)
+            "Authorization": f"Bearer {access_token}"
             if self.auth_type == "oauth"
-            else "Basic {}".format(access_token)  # noqa
+            else f"Basic {access_token}"  # noqa
         }
 
     def is_token_valid(self, token):
         url = f"https://www.googleapis.com/oauth2/v1/tokeninfo?access_token={token}"  # noqa
         response = requests.get(url)
         if response.status_code == 200:
-            print(f"Token Validated for user {response.json()['email']}")
+            logger.info(f"Token Validated for user {response.json()['email']}")
             return True
         return False
 
@@ -60,19 +67,15 @@ def get_access_token(self):
                 if self.is_token_valid(token):
                     return token
                 else:
-                    print(
-                        'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! '  # noqa type: ignore
-                    )
+                    logger.error('please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ')  # noqa
                     sys.exit(1)
             else:
                 return token
         else:
             if self.apigee_type == "x":
-                print(
-                    'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! '  # noqa
-                )
+                logger.error('please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ')  # noqa
             else:
-                print("please export APIGEE_OPDK_ACCESS_TOKEN")
+                logger.error('please export APIGEE_OPDK_ACCESS_TOKEN')
             sys.exit(1)
 
     def set_auth_header(self):
@@ -136,7 +139,7 @@ def create_api(self, api_name, proxy_bundle_path):
         if response.status_code == 200:
             revision = response.json().get('revision', "1")
             return True, revision
-        print(response.text)
+        logger.debug(response.text)
         return False, None
 
     def get_api_revisions_deployment(self, env, api_name, api_rev):  # noqa
@@ -154,10 +157,10 @@ def get_api_revisions_deployment(self, env, api_name, api_rev):  # noqa
             if self.apigee_type == "opdk":
                 if api_deployment_status == "deployed":
                     return True
-            print(f"API {api_name} is in Status: {api_deployment_status} !")  # noqa
+            logger.debug(f"API {api_name} is in Status: {api_deployment_status} !")  # noqa
             return False
         else:
-            print(response.text)
+            logger.debug(response.text)
             return False
 
     def deploy_api(self, env, api_name, api_rev):
@@ -171,9 +174,9 @@ def deploy_api(self, env, api_name, api_rev):
         else:
             resp = response.json()
             if "already deployed" in resp["error"]["message"]:
-                print("Proxy {} is already Deployed".format(api_name))
+                logger.info(f"Proxy {api_name} is already Deployed")
                 return True
-            print(response.text)
+            logger.debug(f"{response.text}")
             return False
 
     def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_force_redeploy=False):  # noqa
@@ -185,50 +188,38 @@ def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_force_redeploy
         if get_api_status:
             api_exists = True
             api_rev = api_revs[-1]
-            print(
-                f"Proxy with name {api_name} with revision {api_rev} already exists in Apigee Org {self.org}"  # noqa
-            )
+            logger.warning(f"Proxy with name {api_name} with revision {api_rev} already exists in Apigee Org {self.org}")  # noqa
             if api_force_redeploy:
                 api_exists = False
         if not api_exists:
             api_created, api_rev = self.create_api(api_name, proxy_bundle_path)
             if api_created:
-                print(
-                    f"Proxy has been imported with name {api_name} in Apigee Org {self.org}"  # noqa
-                )
+                logger.info(f"Proxy has been imported with name {api_name} in Apigee Org {self.org}")  # noqa
                 api_exists = True
             else:
-                print(f"ERROR : Proxy {api_name} import failed !!! ")
+                logger.error(f"ERROR : Proxy {api_name} import failed !!! ")
                 return False
         if api_exists:
             if self.get_api_revisions_deployment(
                         env, api_name, api_rev
                     ):
-                print(f"INFO : Proxy {api_name} already active in to {env} in Apigee Org {self.org} !")  # noqa
+                logger.info(f"Proxy {api_name} already active in to {env} in Apigee Org {self.org} !")  # noqa
                 return True
             else:
                 if self.deploy_api(env, api_name, api_rev):
-                    print(
-                        f"Proxy with name {api_name} has been deployed  to {env} in Apigee Org {self.org}"  # noqa
-                    )
+                    logger.info(f"Proxy with name {api_name} has been deployed  to {env} in Apigee Org {self.org}")  # noqa
                     while api_deployment_retry_count < api_deployment_retry:
                         if self.get_api_revisions_deployment(
                             env, api_name, api_rev
                         ):
-                            print(
-                                f"Proxy {api_name} active in runtime after {api_deployment_retry_count*api_deployment_sleep} seconds "  # noqa
-                            )
+                            logger.debug(f"Proxy {api_name} active in runtime after {api_deployment_retry_count*api_deployment_sleep} seconds ")  # noqa
                             return True
                         else:
-                            print(
-                                f"Checking API deployment status in {api_deployment_sleep} seconds"  # noqa
-                            )
+                            logger.debug(f"Checking API deployment status in {api_deployment_sleep} seconds")  # noqa
                             sleep(api_deployment_sleep)
                             api_deployment_retry_count += 1
                 else:
-                    print(
-                        f"ERROR : Proxy deployment  to {env} in Apigee Org {self.org} Failed !!"  # noqa
-                    )
+                    logger.error(f"ERROR : Proxy deployment  to {env} in Apigee Org {self.org} Failed !!")  # noqa
                     return False
 
     def get_api_vhost(self, vhost_name, env):
@@ -244,13 +235,11 @@ def get_api_vhost(self, vhost_name, env):
             else:
                 hosts = response.json()["hostnames"]
             if len(hosts) == 0:
-                print(
-                    f"ERROR: Vhost/Env Group {vhost_name} contains no domains"  # noqa
-                )
+                logger.error(f"Vhost/Env Group {vhost_name} contains no domains")  # noqa
                 return None
             return hosts
         else:
-            print(f"ERROR: Vhost/Env Group {vhost_name} contains no domains")  # noqa
+            logger.error(f"Vhost/Env Group {vhost_name} contains no domains")
             return None
 
     def list_apis(self, api_type):
@@ -289,6 +278,43 @@ def fetch_api_revision(self, api_type, api_name, revision, export_dir):  # noqa
             return True
         return False
 
+    def fetch_api_proxy_ts_parallel(self, arg_tuple):
+        self.fetch_api_revision(arg_tuple[0], arg_tuple[1], arg_tuple[2], arg_tuple[3])  # noqa
+        unzip_file(
+                    f"{arg_tuple[3]}/{arg_tuple[1]}.zip",  # noqa
+                    f"{arg_tuple[3]}/{arg_tuple[1]}",  # noqa
+                )
+        parsed_proxy_hosts = parse_proxy_hosts(f"{arg_tuple[3]}/{arg_tuple[1]}/apiproxy")  # noqa
+        proxy_tes = get_tes(parsed_proxy_hosts)
+        return arg_tuple[0], arg_tuple[1], parsed_proxy_hosts, proxy_tes
+
+    def fetch_env_target_servers_parallel(self, arg_tuple):
+        ts_info = self.get_target_server(arg_tuple[0], arg_tuple[1])
+        return arg_tuple[1], ts_info
+
+    def call_validator_proxy_parallel(self, arg_tuple):
+        response = run_validator_proxy(arg_tuple[0], arg_tuple[1], arg_tuple[2], arg_tuple[3], arg_tuple[4])  # noqa
+        if isinstance(response, list):
+            report = []
+            for output in response:
+                report.append(
+                    [
+                        output["name"],
+                        output["extracted_from"],
+                        output["host"],
+                        output["port"],
+                        output["env"],
+                        output["status"],
+                        output["info"] if output.get("info") else " & ".join(
+                            list(set(arg_tuple[5][output["name"]])))
+                        if output["name"] in arg_tuple[5]
+                        else "No References in any API",
+                    ]
+                )
+            return report
+        else:
+            return {"error": f"Error while calling the validator proxy - {response.get('error','unknown error')} with payload {arg_tuple[3]}"}  # noqa
+
     def write_proxy_bundle(self, export_dir, file_name, data):
         file_path = f"./{export_dir}/{file_name}.zip"
         with open(file_path, "wb") as fl:

tools/target-server-validator/apiproxy/policies/AM-Set-Json-Response.xml

@@ -13,11 +13,6 @@
 -->
 <AssignMessage name="AM-Set-Json-Response">
     <Set>
-        <Payload contentType="application/json">{
-    "host":"{request.header.host_name}", 
-    "port": "{request.header.port_number}", 
-    "status":"{flow.reachableStatus}"
-}
-</Payload>
+        <Payload contentType="application/json">{flow.result}</Payload>
     </Set>
 </AssignMessage>

tools/target-server-validator/apiproxy/policies/JSON-Threat-Protection.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+    Copyright 2024 Google LLC
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+        http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<JSONThreatProtection continueOnError="false" enabled="true" name="JSON-Threat-Protection">
+    <DisplayName>JSON Threat Protection</DisplayName>
+    <Properties/>
+    <ArrayElementCount>5</ArrayElementCount>
+    <Source>request</Source>
+</JSONThreatProtection>

tools/target-server-validator/apiproxy/proxies/default.xml

@@ -14,6 +14,9 @@
 <ProxyEndpoint name="default">
     <PreFlow name="PreFlow">
         <Request>
+            <Step>
+                <Name>JSON-Threat-Protection</Name>
+            </Step>
             <Step>
                 <Name>JC-Port-Open-Check</Name>
             </Step>