Merge pull request #2887 from CannonLock/SOFTWARE-5443

Create Institution Endpoint (SOFTWARE-5443)

src/app.py

@@ -18,6 +18,7 @@
 
 from webapp import default_config
 from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, escape, cache_control_private
+from webapp.flask_common import create_accepted_response
 from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
 from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm, GenerateProjectForm
 from webapp.models import GlobalData
@@ -264,6 +265,21 @@ def contacts():
         app.log_exception(sys.exc_info())
         return Response("Error getting users", status=503)  # well, it's better than crashing
 
+@app.route('/api/institutions')
+def institutions():
+
+    resource_facilities = set(global_data.get_topology().facilities.keys())
+    project_facilities = set(x['Organization'] for x in global_data.get_projects()['Projects']['Project'])
+
+    facilities = project_facilities.union(resource_facilities)
+
+    facility_data = [["Institution Name", "Has Resource(s)", "Has Project(s)"]]
+    for facility in sorted(facilities):
+        facility_data.append([facility, facility in resource_facilities, facility in project_facilities])
+
+    return create_accepted_response(facility_data, request.headers, default="text/csv")
+
+
 
 @app.route('/miscproject/xml')
 def miscproject_xml():

src/tests/test_api.py

@@ -7,6 +7,8 @@
 # Rewrites the path so the app can be imported like it normally is
 import os
 import sys
+import csv
+import io
 
 topdir = os.path.join(os.path.dirname(__file__), "..")
 sys.path.append(topdir)
@@ -57,8 +59,9 @@
     "/origin/Authfile-public",
     "/origin/scitokens.conf",
     "/cache/scitokens.conf",
+    "/api/institutions",
     "/cache/grid-mapfile",
-    "/origin/grid-mapfile",
+    "/origin/grid-mapfile"
 ]
 
 
@@ -239,6 +242,21 @@ def validate_namespace_schema(ns):
                     validate_cache_schema(cache)
         assert found_credgen, "At least one namespace with credential_generation"
 
+
+    def test_institution_accept_type(self, client: flask.Flask):
+        """Checks both formats output the same content"""
+
+        json_institutions = client.get("/api/institutions", headers={"Accept": "application/json"}).json
+        json_tuples = [tuple(map(str, x)) for x in sorted(json_institutions, key=lambda x: x[0])]
+
+        csv_institutions = csv.reader(io.StringIO(client.get("/api/institutions").data.decode()))
+        csv_tuples = [tuple(x) for x in sorted(csv_institutions, key=lambda x: x[0])]
+
+        assert len(csv_tuples) == len(json_tuples)
+
+        assert tuple(json_tuples) == tuple(csv_tuples)
+
+
     def test_origin_grid_mapfile(self, client: flask.Flask):
         TEST_ORIGIN = "origin-auth2001.chtc.wisc.edu"  # This origin serves protected data
         response = client.get("/origin/grid-mapfile")
@@ -835,6 +853,43 @@ def test_facility_defaults(self, client: flask.Flask):
         # Check that the site contains the appropriate keys
         assert set(facilities.popitem()[1]).issuperset(["ID", "Name", "IsCCStar"])
 
+    def test_institution_default(self, client: flask.Flask):
+        institutions = client.get("/api/institutions", headers={"Accept": "application/json"}).json
+
+        assert len(institutions) > 0
+
+        # Check facilities exist and have the "have resources" bit flipped
+        assert [i for i in institutions if i[0] == "JINR"][0][1]
+        assert [i for i in institutions if i[0] == "Universidade de São Paulo - Laboratório de Computação Científica Avançada"][0][1]
+
+        # Project Organizations exist and have "has project" bit flipped
+        assert [i for i in institutions if i[0] == "Iolani School"][0][2]
+        assert [i for i in institutions if i[0] == "University of California, San Diego"][0][2]
+
+        # Both
+        assert [i for i in institutions if i[0] == "Harvard University"][0][1] and [i for i in institutions if i[0] == "Harvard University"][0][2]
+
+        # Check Project only doesn't have resource bit
+        assert [i for i in institutions if i[0] == "National Research Council of Canada"][0][1] is False
+
+        # Facility Tests
+        facilities = set(global_data.get_topology().facilities.keys())
+
+        # Check all facilities exist
+        assert set(i[0] for i in institutions).issuperset(facilities)
+
+        # Check all facilities have their facilities bit flipped
+        assert all(x[1] for x in institutions if x[0] in institutions)
+
+        # Project Tests
+        projects = set(x['Organization'] for x in global_data.get_projects()['Projects']['Project'])
+
+        # Check all projects exist
+        assert set(i[0] for i in institutions).issuperset(projects)
+
+        # Check all projects have the project bit flipped
+        assert all(x[2] for x in institutions if x[0] in projects)
+
 
 if __name__ == '__main__':
     pytest.main()

src/webapp/common.py

@@ -13,6 +13,9 @@
 
 import xmltodict
 import yaml
+import csv
+from io import StringIO
+
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
@@ -53,6 +56,14 @@ def populate_voown_name(self, vo_id_to_name: Dict):
         self.voown_name = [vo_id_to_name.get(i, "") for i in self.voown_id]
 
 
+def to_csv(data: list) -> str:
+    csv_string = StringIO()
+    writer = csv.writer(csv_string)
+    for row in data:
+        writer.writerow(row)
+    return csv_string.getvalue()
+
+
 def is_null(x, *keys) -> bool:
     for key in keys:
         if not key: continue
@@ -101,7 +112,7 @@ def simplify_attr_list(data: Union[Dict, List], namekey: str, del_name: bool = T
     return new_data
 
 
-def expand_attr_list_single(data: Dict, namekey:str, valuekey: str, name_first=True) -> List[OrderedDict]:
+def expand_attr_list_single(data: Dict, namekey: str, valuekey: str, name_first=True) -> List[OrderedDict]:
     """
     Expand
         {"name1": "val1",
@@ -120,7 +131,8 @@ def expand_attr_list_single(data: Dict, namekey:str, valuekey: str, name_first=T
     return newdata
 
 
-def expand_attr_list(data: Dict, namekey: str, ordering: Union[List, None]=None, ignore_missing=False) -> List[OrderedDict]:
+def expand_attr_list(data: Dict, namekey: str, ordering: Union[List, None] = None, ignore_missing=False) -> List[
+    OrderedDict]:
     """
     Expand
         {"name1": {"attr1": "val1", ...},
@@ -263,14 +275,15 @@ def git_clone_or_pull(repo, dir, branch, ssh_key=None) -> bool:
         ok = ok and run_git_cmd(["checkout", branch], dir=dir)
     return ok
 
+
 def git_clone_or_fetch_mirror(repo, git_dir, ssh_key=None) -> bool:
     if os.path.exists(git_dir):
         ok = run_git_cmd(["fetch", "origin"], git_dir=git_dir, ssh_key=ssh_key)
     else:
         ok = run_git_cmd(["clone", "--mirror", repo, git_dir], ssh_key=ssh_key)
         # disable mirror push
         ok = ok and run_git_cmd(["config", "--unset", "remote.origin.mirror"],
-                                                              git_dir=git_dir)
+                                git_dir=git_dir)
     return ok
 
 
@@ -315,17 +328,14 @@ def escape(pattern: str) -> str:
 
     unescaped_characters = ['!', '"', '%', "'", ',', '/', ':', ';', '<', '=', '>', '@', "`"]
     for unescaped_character in unescaped_characters:
-
         escaped_string = re.sub(unescaped_character, f"\\{unescaped_character}", escaped_string)
 
     return escaped_string
 
 
 def support_cors(f):
-
     @wraps(f)
     def wrapped():
-
         response = f()
 
         response.headers['Access-Control-Allow-Origin'] = '*'

src/webapp/flask_common.py

@@ -0,0 +1,22 @@
+from flask import Response
+from .common import to_csv, to_json_bytes
+from typing import List
+
+def create_accepted_response(data: List, headers, default=None) -> Response:
+    """Provides CSV or JSON options for list of list(string)"""
+
+    if not default:
+        default = "application/json"
+
+    accepted_response_builders = {
+        "text/csv": lambda: Response(to_csv(data), mimetype="text/csv"),
+        "application/json": lambda: Response(to_json_bytes(data), mimetype="application/json"),
+    }
+
+    requested_types = set(headers.get('Accept', 'default').replace(' ', '').split(","))
+    accepted_and_requested = set(accepted_response_builders.keys()).intersection(requested_types)
+
+    if accepted_and_requested:
+        return accepted_response_builders[accepted_and_requested.pop()]()
+    else:
+        return accepted_response_builders[default]()