From 0145d5ee9ebd8cce34247668370762a3e6a9a124 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Wed, 18 Jan 2023 15:54:43 -0600 Subject: [PATCH 1/3] Create Institution Endpoint (SOFTWARE-5443) Add Institution Endpoint Add Tests for Institution Endpoint --- src/app.py | 17 ++++++++++++- src/tests/test_api.py | 55 ++++++++++++++++++++++++++++++++++++++++++- src/webapp/common.py | 43 ++++++++++++++++++++++++++++----- 3 files changed, 107 insertions(+), 8 deletions(-) diff --git a/src/app.py b/src/app.py index 352407cba..aa1145fd4 100755 --- a/src/app.py +++ b/src/app.py @@ -14,7 +14,7 @@ import urllib.parse from webapp import default_config -from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, escape +from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, escape, create_accepted_response from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm from webapp.models import GlobalData @@ -224,6 +224,21 @@ def contacts(): app.log_exception(sys.exc_info()) return Response("Error getting users", status=503) # well, it's better than crashing +@app.route('/api/institutions') +def institutions(): + + resource_facilities = set(global_data.get_topology().facilities.keys()) + project_facilities = set(x['Organization'] for x in global_data.get_projects()['Projects']['Project']) + + facilities = project_facilities.union(resource_facilities) + + facility_data = [["Institution Name", "Has Resource(s)", "Has Project(s)"]] + for facility in sorted(facilities): + facility_data.append([facility, facility in resource_facilities, facility in project_facilities]) + + return create_accepted_response(facility_data, request.headers, default="text/csv") + + @app.route('/miscproject/xml') def miscproject_xml(): diff --git a/src/tests/test_api.py b/src/tests/test_api.py index bb16ec01d..dbb3cb7c4 100644 --- a/src/tests/test_api.py +++ b/src/tests/test_api.py @@ -6,6 +6,8 @@ # Rewrites the path so the app can be imported like it normally is import os import sys +import csv +import io topdir = os.path.join(os.path.dirname(__file__), "..") sys.path.append(topdir) @@ -52,7 +54,8 @@ "/origin/Authfile", "/origin/Authfile-public", "/origin/scitokens.conf", - "/cache/scitokens.conf" + "/cache/scitokens.conf", + "/api/institutions" ] @@ -216,6 +219,19 @@ def validate_namespace_schema(ns): for cache in namespace["caches"]: validate_cache_schema(cache) + def test_institution_accept_type(self, client: flask.Flask): + """Checks both formats output the same content""" + + json_institutions = client.get("/api/institutions", headers={"Accept": "application/json"}).json + json_tuples = [tuple(map(str, x)) for x in sorted(json_institutions, key=lambda x: x[0])] + + csv_institutions = csv.reader(io.StringIO(client.get("/api/institutions").data.decode())) + csv_tuples = [tuple(x) for x in sorted(csv_institutions, key=lambda x: x[0])] + + assert len(csv_tuples) == len(json_tuples) + + assert tuple(json_tuples) == tuple(csv_tuples) + class TestEndpointContent: @@ -733,6 +749,43 @@ def test_facility_defaults(self, client: flask.Flask): # Check that the site contains the appropriate keys assert set(facilities.popitem()[1]).issuperset(["ID", "Name", "IsCCStar"]) + def test_institution_default(self, client: flask.Flask): + institutions = client.get("/api/institutions", headers={"Accept": "application/json"}).json + + assert len(institutions) > 0 + + # Check facilities exist and have the "have resources" bit flipped + assert [i for i in institutions if i[0] == "JINR"][0][1] + assert [i for i in institutions if i[0] == "Universidade de São Paulo - Laboratório de Computação Científica Avançada"][0][1] + + # Project Organizations exist and have "has project" bit flipped + assert [i for i in institutions if i[0] == "Iolani School"][0][2] + assert [i for i in institutions if i[0] == "University of California, San Diego"][0][2] + + # Both + assert [i for i in institutions if i[0] == "Harvard University"][0][1] and [i for i in institutions if i[0] == "Harvard University"][0][2] + + # Check Project only doesn't have resource bit + assert [i for i in institutions if i[0] == "National Research Council of Canada"][0][1] is False + + # Facility Tests + facilities = set(global_data.get_topology().facilities.keys()) + + # Check all facilities exist + assert set(i[0] for i in institutions).issuperset(facilities) + + # Check all facilities have their facilities bit flipped + assert all(x[1] for x in institutions if x[0] in institutions) + + # Project Tests + projects = set(x['Organization'] for x in global_data.get_projects()['Projects']['Project']) + + # Check all projects exist + assert set(i[0] for i in institutions).issuperset(projects) + + # Check all projects have the project bit flipped + assert all(x[2] for x in institutions if x[0] in projects) + if __name__ == '__main__': pytest.main() diff --git a/src/webapp/common.py b/src/webapp/common.py index eea23856a..afe6f6881 100644 --- a/src/webapp/common.py +++ b/src/webapp/common.py @@ -13,6 +13,10 @@ import xmltodict import yaml +import csv +from io import StringIO +from flask import Response + try: from yaml import CSafeLoader as SafeLoader except ImportError: @@ -53,6 +57,34 @@ def populate_voown_name(self, vo_id_to_name: Dict): self.voown_name = [vo_id_to_name.get(i, "") for i in self.voown_id] +def create_accepted_response(data: list, headers, default=None) -> Response: + """Provides CSV or JSON options for list of list(string)""" + + if not default: + default = "application/json" + + accepted_response_builders = { + "text/csv": lambda: Response(to_csv(data), mimetype="text/csv"), + "application/json": lambda: Response(to_json_bytes(data), mimetype="application/json"), + } + + requested_types = set(headers.get('Accept', 'default').replace(' ', '').split(",")) + accepted_and_requested = set(accepted_response_builders.keys()).intersection(requested_types) + + if accepted_and_requested: + return accepted_response_builders[accepted_and_requested.pop()]() + else: + return accepted_response_builders[default]() + + +def to_csv(data: list) -> str: + csv_string = StringIO() + writer = csv.writer(csv_string) + for row in data: + writer.writerow(row) + return csv_string.getvalue() + + def is_null(x, *keys) -> bool: for key in keys: if not key: continue @@ -101,7 +133,7 @@ def simplify_attr_list(data: Union[Dict, List], namekey: str, del_name: bool = T return new_data -def expand_attr_list_single(data: Dict, namekey:str, valuekey: str, name_first=True) -> List[OrderedDict]: +def expand_attr_list_single(data: Dict, namekey: str, valuekey: str, name_first=True) -> List[OrderedDict]: """ Expand {"name1": "val1", @@ -120,7 +152,8 @@ def expand_attr_list_single(data: Dict, namekey:str, valuekey: str, name_first=T return newdata -def expand_attr_list(data: Dict, namekey: str, ordering: Union[List, None]=None, ignore_missing=False) -> List[OrderedDict]: +def expand_attr_list(data: Dict, namekey: str, ordering: Union[List, None] = None, ignore_missing=False) -> List[ + OrderedDict]: """ Expand {"name1": {"attr1": "val1", ...}, @@ -263,6 +296,7 @@ def git_clone_or_pull(repo, dir, branch, ssh_key=None) -> bool: ok = ok and run_git_cmd(["checkout", branch], dir=dir) return ok + def git_clone_or_fetch_mirror(repo, git_dir, ssh_key=None) -> bool: if os.path.exists(git_dir): ok = run_git_cmd(["fetch", "origin"], git_dir=git_dir, ssh_key=ssh_key) @@ -270,7 +304,7 @@ def git_clone_or_fetch_mirror(repo, git_dir, ssh_key=None) -> bool: ok = run_git_cmd(["clone", "--mirror", repo, git_dir], ssh_key=ssh_key) # disable mirror push ok = ok and run_git_cmd(["config", "--unset", "remote.origin.mirror"], - git_dir=git_dir) + git_dir=git_dir) return ok @@ -315,17 +349,14 @@ def escape(pattern: str) -> str: unescaped_characters = ['!', '"', '%', "'", ',', '/', ':', ';', '<', '=', '>', '@', "`"] for unescaped_character in unescaped_characters: - escaped_string = re.sub(unescaped_character, f"\\{unescaped_character}", escaped_string) return escaped_string def support_cors(f): - @wraps(f) def wrapped(): - response = f() response.headers['Access-Control-Allow-Origin'] = '*' From e3b6af528e71c053bd04667c7d7e030e749350b1 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Thu, 19 Jan 2023 10:47:11 -0600 Subject: [PATCH 2/3] Create Institution Endpoint (SOFTWARE-5443) Move the create_accepted_response function --- src/app.py | 3 ++- src/webapp/common.py | 21 --------------------- src/webapp/flask_common.py | 22 ++++++++++++++++++++++ 3 files changed, 24 insertions(+), 22 deletions(-) create mode 100644 src/webapp/flask_common.py diff --git a/src/app.py b/src/app.py index aa1145fd4..90bfbe751 100755 --- a/src/app.py +++ b/src/app.py @@ -14,7 +14,8 @@ import urllib.parse from webapp import default_config -from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, escape, create_accepted_response +from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, escape +from webapp.flask_common import create_accepted_response from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm from webapp.models import GlobalData diff --git a/src/webapp/common.py b/src/webapp/common.py index afe6f6881..872391d4c 100644 --- a/src/webapp/common.py +++ b/src/webapp/common.py @@ -15,7 +15,6 @@ import yaml import csv from io import StringIO -from flask import Response try: from yaml import CSafeLoader as SafeLoader @@ -57,26 +56,6 @@ def populate_voown_name(self, vo_id_to_name: Dict): self.voown_name = [vo_id_to_name.get(i, "") for i in self.voown_id] -def create_accepted_response(data: list, headers, default=None) -> Response: - """Provides CSV or JSON options for list of list(string)""" - - if not default: - default = "application/json" - - accepted_response_builders = { - "text/csv": lambda: Response(to_csv(data), mimetype="text/csv"), - "application/json": lambda: Response(to_json_bytes(data), mimetype="application/json"), - } - - requested_types = set(headers.get('Accept', 'default').replace(' ', '').split(",")) - accepted_and_requested = set(accepted_response_builders.keys()).intersection(requested_types) - - if accepted_and_requested: - return accepted_response_builders[accepted_and_requested.pop()]() - else: - return accepted_response_builders[default]() - - def to_csv(data: list) -> str: csv_string = StringIO() writer = csv.writer(csv_string) diff --git a/src/webapp/flask_common.py b/src/webapp/flask_common.py new file mode 100644 index 000000000..a76a800c1 --- /dev/null +++ b/src/webapp/flask_common.py @@ -0,0 +1,22 @@ +from flask import Response +from .common import to_csv, to_json_bytes + + +def create_accepted_response(data: list, headers, default=None) -> Response: + """Provides CSV or JSON options for list of list(string)""" + + if not default: + default = "application/json" + + accepted_response_builders = { + "text/csv": lambda: Response(to_csv(data), mimetype="text/csv"), + "application/json": lambda: Response(to_json_bytes(data), mimetype="application/json"), + } + + requested_types = set(headers.get('Accept', 'default').replace(' ', '').split(",")) + accepted_and_requested = set(accepted_response_builders.keys()).intersection(requested_types) + + if accepted_and_requested: + return accepted_response_builders[accepted_and_requested.pop()]() + else: + return accepted_response_builders[default]() From 19288b62029df74c2176f9b26eb8340c2d61687e Mon Sep 17 00:00:00 2001 From: Cannon Lock <49032265+CannonLock@users.noreply.github.com> Date: Wed, 25 Jan 2023 09:39:40 -0600 Subject: [PATCH 3/3] Change out the type annotation for one that is compatible with 3.6 Co-authored-by: Matyas Selmeci --- src/webapp/flask_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webapp/flask_common.py b/src/webapp/flask_common.py index a76a800c1..a5bc662d1 100644 --- a/src/webapp/flask_common.py +++ b/src/webapp/flask_common.py @@ -1,8 +1,8 @@ from flask import Response from .common import to_csv, to_json_bytes +from typing import List - -def create_accepted_response(data: list, headers, default=None) -> Response: +def create_accepted_response(data: List, headers, default=None) -> Response: """Provides CSV or JSON options for list of list(string)""" if not default: