From d0c0b5d9d28bc1edd3a237faa019bc9bdd8519a2 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Mon, 20 Nov 2023 17:28:41 -0600 Subject: [PATCH 1/9] Move namespaces json tests to a separate class and make a fixture for it This will let me split up the test into smaller chunks. --- src/tests/test_api.py | 115 ++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 54 deletions(-) diff --git a/src/tests/test_api.py b/src/tests/test_api.py index c4aa1c743..3f8571f5c 100644 --- a/src/tests/test_api.py +++ b/src/tests/test_api.py @@ -1,6 +1,7 @@ import re import flask import pytest +from typing import Dict import urllib.parse from pytest_mock import MockerFixture @@ -71,6 +72,66 @@ def client(): yield client +class TestNamespaces: + @pytest.fixture(scope="class") + def namespaces_json(self, client) -> Dict: + response = client.get('/stashcache/namespaces') + assert response.status_code == 200 + return response.json + + @staticmethod + def validate_cache_schema(cc): + assert HOST_PORT_RE.match(cc["auth_endpoint"]) + assert HOST_PORT_RE.match(cc["endpoint"]) + assert cc["resource"] and isinstance(cc["resource"], str) + + @staticmethod + def validate_namespace_schema(ns): + assert isinstance(ns["caches"], list) # we do have a case where it's empty + assert ns["path"].startswith("/") # implies str + assert isinstance(ns["readhttps"], bool) + assert isinstance(ns["usetokenonread"], bool) + assert ns["dirlisthost"] is None or PROTOCOL_HOST_PORT_RE.match(ns["dirlisthost"]) + assert ns["writebackhost"] is None or PROTOCOL_HOST_PORT_RE.match(ns["writebackhost"]) + credgen = ns["credential_generation"] + if credgen is not None: + assert isinstance(credgen["max_scope_depth"], int) and credgen["max_scope_depth"] > -1 + assert credgen["strategy"] in CredentialGeneration.STRATEGIES + assert credgen["issuer"] + parsed_issuer = urllib.parse.urlparse(credgen["issuer"]) + assert parsed_issuer.netloc and parsed_issuer.scheme == "https" + if credgen["vault_server"]: + assert isinstance(credgen["vault_server"], str) + if credgen["vault_issuer"]: + assert isinstance(credgen["vault_issuer"], str) + if credgen["base_path"]: + assert isinstance(credgen["base_path"], str) + + def test_caches(self, namespaces_json): + assert "caches" in namespaces_json + caches = namespaces_json["caches"] + # Have a reasonable number of caches + assert len(caches) > 20 + for cache in caches: + self.validate_cache_schema(cache) + + def test_namespaces(self, namespaces_json): + assert "namespaces" in namespaces_json + namespaces = namespaces_json["namespaces"] + # Have a reasonable number of namespaces + assert len(namespaces) > 15 + + found_credgen = False + for namespace in namespaces: + if namespace["credential_generation"] is not None: + found_credgen = True + self.validate_namespace_schema(namespace) + if namespace["caches"]: + for cache in namespace["caches"]: + self.validate_cache_schema(cache) + assert found_credgen, "At least one namespace with credential_generation" + + class TestAPI: def test_sanity(self, client: flask.Flask): @@ -189,60 +250,6 @@ def test_stashcache_file(key, endpoint, fqdn, resource_stashcache_files): else: app.config["STASHCACHE_LEGACY_AUTH"] = old_legacy_auth - def test_stashcache_namespaces(self, client: flask.Flask): - def validate_cache_schema(cc): - assert HOST_PORT_RE.match(cc["auth_endpoint"]) - assert HOST_PORT_RE.match(cc["endpoint"]) - assert cc["resource"] and isinstance(cc["resource"], str) - - def validate_namespace_schema(ns): - assert isinstance(ns["caches"], list) # we do have a case where it's empty - assert ns["path"].startswith("/") # implies str - assert isinstance(ns["readhttps"], bool) - assert isinstance(ns["usetokenonread"], bool) - assert ns["dirlisthost"] is None or PROTOCOL_HOST_PORT_RE.match(ns["dirlisthost"]) - assert ns["writebackhost"] is None or PROTOCOL_HOST_PORT_RE.match(ns["writebackhost"]) - credgen = ns["credential_generation"] - if credgen is not None: - assert isinstance(credgen["max_scope_depth"], int) and credgen["max_scope_depth"] > -1 - assert credgen["strategy"] in CredentialGeneration.STRATEGIES - assert credgen["issuer"] - parsed_issuer = urllib.parse.urlparse(credgen["issuer"]) - assert parsed_issuer.netloc and parsed_issuer.scheme == "https" - if credgen["vault_server"]: - assert isinstance(credgen["vault_server"], str) - if credgen["vault_issuer"]: - assert isinstance(credgen["vault_issuer"], str) - if credgen["base_path"]: - assert isinstance(credgen["base_path"], str) - - response = client.get('/stashcache/namespaces') - assert response.status_code == 200 - namespaces_json = response.json - - assert "caches" in namespaces_json - caches = namespaces_json["caches"] - # Have a reasonable number of caches - assert len(caches) > 20 - for cache in caches: - validate_cache_schema(cache) - - assert "namespaces" in namespaces_json - namespaces = namespaces_json["namespaces"] - # Have a reasonable number of namespaces - assert len(namespaces) > 15 - - found_credgen = False - for namespace in namespaces: - if namespace["credential_generation"] is not None: - found_credgen = True - validate_namespace_schema(namespace) - if namespace["caches"]: - for cache in namespace["caches"]: - validate_cache_schema(cache) - assert found_credgen, "At least one namespace with credential_generation" - - def test_institution_accept_type(self, client: flask.Flask): """Checks both formats output the same content""" From a8873d4a747180e5a3955d80bf77320e74ab8213 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Mon, 20 Nov 2023 17:48:32 -0600 Subject: [PATCH 2/9] Add test for "scitokens" blocks in namespaces json (SOFTWARE-5760) --- src/tests/test_api.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/tests/test_api.py b/src/tests/test_api.py index 3f8571f5c..f69de09d6 100644 --- a/src/tests/test_api.py +++ b/src/tests/test_api.py @@ -1,7 +1,7 @@ import re import flask import pytest -from typing import Dict +from typing import Dict, List import urllib.parse from pytest_mock import MockerFixture @@ -79,6 +79,11 @@ def namespaces_json(self, client) -> Dict: assert response.status_code == 200 return response.json + @pytest.fixture(scope="class") + def namespaces(self, namespaces_json) -> List[Dict]: + assert "namespaces" in namespaces_json + return namespaces_json["namespaces"] + @staticmethod def validate_cache_schema(cc): assert HOST_PORT_RE.match(cc["auth_endpoint"]) @@ -115,9 +120,7 @@ def test_caches(self, namespaces_json): for cache in caches: self.validate_cache_schema(cache) - def test_namespaces(self, namespaces_json): - assert "namespaces" in namespaces_json - namespaces = namespaces_json["namespaces"] + def test_namespaces(self, namespaces): # Have a reasonable number of namespaces assert len(namespaces) > 15 @@ -131,6 +134,27 @@ def test_namespaces(self, namespaces_json): self.validate_cache_schema(cache) assert found_credgen, "At least one namespace with credential_generation" + @staticmethod + def validate_scitokens_block(sci): + assert sci["issuer"] + assert isinstance(sci["issuer"], str) + assert "://" in sci["issuer"] + assert isinstance(sci["basepath"], list) + assert sci["basepath"] # must have at least 1 + for bp in sci["basepath"]: + assert bp.startswith("/") # implies str + assert "," not in bp + assert isinstance(sci["restrictedpath"], list) + for rp in sci["restrictedpath"]: # may be empty + assert rp.startswith("/") # implies str + assert "," not in rp + + def test_issuers_in_namespaces(self, namespaces): + for namespace in namespaces: + assert isinstance(namespace["scitokens"], list) + for scitokens_block in namespace["scitokens"]: + self.validate_scitokens_block(scitokens_block) + class TestAPI: From 3fed3fad9a18c5e4b88493e0cba90cb3cad0ea84 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Mon, 20 Nov 2023 18:49:18 -0600 Subject: [PATCH 3/9] The fixture `client` is function scoped so `namespaces` and `namespaces_json` have to be function scoped too --- src/tests/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/test_api.py b/src/tests/test_api.py index f69de09d6..c85949a44 100644 --- a/src/tests/test_api.py +++ b/src/tests/test_api.py @@ -73,13 +73,13 @@ def client(): class TestNamespaces: - @pytest.fixture(scope="class") + @pytest.fixture def namespaces_json(self, client) -> Dict: response = client.get('/stashcache/namespaces') assert response.status_code == 200 return response.json - @pytest.fixture(scope="class") + @pytest.fixture def namespaces(self, namespaces_json) -> List[Dict]: assert "namespaces" in namespaces_json return namespaces_json["namespaces"] From ac65c3b5a0e263a1e4ada8faf7361eef4b57e8fe Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 21 Nov 2023 12:24:56 -0600 Subject: [PATCH 4/9] Add scitokens issuer info to the namespaces info in the namespaces JSON (SOFTWARE-5768) --- src/stashcache.py | 8 ++++++++ src/webapp/data_federation.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/stashcache.py b/src/stashcache.py index fc97be44e..3e1931936 100644 --- a/src/stashcache.py +++ b/src/stashcache.py @@ -522,6 +522,13 @@ def get_credential_generation_dict_for_namespace(ns: Namespace) -> Optional[Dict return info +def get_scitokens_list_for_namespace(ns: Namespace) -> List[Dict]: + """Return the list of scitokens issuer info for the .namespaces[*].scitokens attribute in the namespaces JSON""" + return list( + filter(None, (a.get_namespaces_scitokens_block() for a in ns.authz_list)) + ) + + def get_namespaces_info(global_data: GlobalData) -> PreJSON: """Return data for the /stashcache/namespaces JSON endpoint. @@ -564,6 +571,7 @@ def _namespace_dict(ns: Namespace): "caches": [], "origins": [], "credential_generation": get_credential_generation_dict_for_namespace(ns), + "scitokens": get_scitokens_list_for_namespace(ns), } for cache_name, cache_resource_obj in cache_resource_objs.items(): diff --git a/src/webapp/data_federation.py b/src/webapp/data_federation.py index 204c05540..5c252a958 100644 --- a/src/webapp/data_federation.py +++ b/src/webapp/data_federation.py @@ -1,3 +1,4 @@ +import re import urllib import urllib.parse from collections import OrderedDict @@ -25,6 +26,8 @@ def get_scitokens_conf_block(self, service_name: str): def get_grid_mapfile_line(self): return "" + def get_namespaces_scitokens_block(self): + return None class NullAuth(AuthMethod): pass @@ -100,6 +103,15 @@ def get_scitokens_conf_block(self, service_name: str): return block + def get_namespaces_scitokens_block(self): + basepath = re.split(r"\s*,\s*", self.base_path) + restrictedpath = re.split(r"\s*,\s*", self.restricted_path) if self.restricted_path else [] + return { + "issuer": self.issuer, + "basepath": basepath, + "restrictedpath": restrictedpath, + } + # TODO Use a dataclass (https://docs.python.org/3.9/library/dataclasses.html) # once we can ditch Python 3.6; the webapp no longer supports 3.6 but some of From bf5a9a88d3ee95005f8567f89988c8c7d5d7503d Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 21 Nov 2023 13:00:36 -0600 Subject: [PATCH 5/9] Turn test_global_data into a fixture --- src/tests/test_stashcache.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/tests/test_stashcache.py b/src/tests/test_stashcache.py index 9e799fe69..d92206421 100644 --- a/src/tests/test_stashcache.py +++ b/src/tests/test_stashcache.py @@ -45,7 +45,8 @@ MOCK_DN_LIST = list(MOCK_DNS_AND_HASHES.keys()) -def get_test_global_data(global_data: models.GlobalData) -> models.GlobalData: +@pytest.fixture +def test_global_data() -> models.GlobalData: """Get a copy of the global data with some entries created for testing""" new_global_data = copy.deepcopy(global_data) @@ -105,8 +106,7 @@ def test_allowedVO_excludes_LIGO_and_ANY_for_ligo_inclusion(self, client: flask. assert spy.call_count == 0 - def test_scitokens_issuer_sections(self, client: flask.Flask): - test_global_data = get_test_global_data(global_data) + def test_scitokens_issuer_sections(self, test_global_data): origin_scitokens_conf = stashcache.generate_origin_scitokens( test_global_data, TEST_ITB_HELM_ORIGIN) assert origin_scitokens_conf.strip(), "Generated scitokens.conf empty" @@ -128,9 +128,7 @@ def test_scitokens_issuer_sections(self, client: flask.Flask): print(f"Generated origin scitokens.conf text:\n{origin_scitokens_conf}\n", file=sys.stderr) raise - def test_scitokens_issuer_public_read_auth_write_namespaces_info(self, client: flask.Flask): - test_global_data = get_test_global_data(global_data) - + def test_scitokens_issuer_public_read_auth_write_namespaces_info(self, test_global_data): namespaces_json = stashcache.get_namespaces_info(test_global_data) namespaces = namespaces_json["namespaces"] testvo_PUBLIC_namespace_list = [ @@ -145,9 +143,7 @@ def test_scitokens_issuer_public_read_auth_write_namespaces_info(self, client: f assert ns["writebackhost"] == f"https://{TEST_SC_ORIGIN}:1095", \ "writebackhost is wrong for namespace with auth write" - def test_scitokens_issuer_public_read_auth_write_scitokens_conf(self, client: flask.Flask): - test_global_data = get_test_global_data(global_data) - + def test_scitokens_issuer_public_read_auth_write_scitokens_conf(self, test_global_data): origin_scitokens_conf = stashcache.generate_origin_scitokens( test_global_data, TEST_SC_ORIGIN) assert origin_scitokens_conf.strip(), "Generated scitokens.conf empty" From f8bca6572f1f144aa72ad3043b0907a21c3e4432 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 21 Nov 2023 13:14:33 -0600 Subject: [PATCH 6/9] Move namespaces JSON test to test_stashcache.py so I can use the test global data keep a simple endpoint test in test_api.py --- src/tests/test_api.py | 97 +++--------------------------------- src/tests/test_stashcache.py | 88 ++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 89 deletions(-) diff --git a/src/tests/test_api.py b/src/tests/test_api.py index c85949a44..29f64e6f8 100644 --- a/src/tests/test_api.py +++ b/src/tests/test_api.py @@ -18,10 +18,6 @@ from app import app, global_data from webapp.topology import Facility, Site, Resource, ResourceGroup -from webapp.data_federation import CredentialGeneration - -HOST_PORT_RE = re.compile(r"[a-zA-Z0-9.-]{3,63}:[0-9]{2,5}") -PROTOCOL_HOST_PORT_RE = re.compile(r"[a-z]+://" + HOST_PORT_RE.pattern) INVALID_USER = dict( username="invalid", @@ -62,7 +58,9 @@ "/cache/scitokens.conf", "/api/institutions", "/cache/grid-mapfile", - "/origin/grid-mapfile" + "/origin/grid-mapfile", + "/osdf/namespaces", + "/stashcache/namespaces", ] @@ -72,90 +70,6 @@ def client(): yield client -class TestNamespaces: - @pytest.fixture - def namespaces_json(self, client) -> Dict: - response = client.get('/stashcache/namespaces') - assert response.status_code == 200 - return response.json - - @pytest.fixture - def namespaces(self, namespaces_json) -> List[Dict]: - assert "namespaces" in namespaces_json - return namespaces_json["namespaces"] - - @staticmethod - def validate_cache_schema(cc): - assert HOST_PORT_RE.match(cc["auth_endpoint"]) - assert HOST_PORT_RE.match(cc["endpoint"]) - assert cc["resource"] and isinstance(cc["resource"], str) - - @staticmethod - def validate_namespace_schema(ns): - assert isinstance(ns["caches"], list) # we do have a case where it's empty - assert ns["path"].startswith("/") # implies str - assert isinstance(ns["readhttps"], bool) - assert isinstance(ns["usetokenonread"], bool) - assert ns["dirlisthost"] is None or PROTOCOL_HOST_PORT_RE.match(ns["dirlisthost"]) - assert ns["writebackhost"] is None or PROTOCOL_HOST_PORT_RE.match(ns["writebackhost"]) - credgen = ns["credential_generation"] - if credgen is not None: - assert isinstance(credgen["max_scope_depth"], int) and credgen["max_scope_depth"] > -1 - assert credgen["strategy"] in CredentialGeneration.STRATEGIES - assert credgen["issuer"] - parsed_issuer = urllib.parse.urlparse(credgen["issuer"]) - assert parsed_issuer.netloc and parsed_issuer.scheme == "https" - if credgen["vault_server"]: - assert isinstance(credgen["vault_server"], str) - if credgen["vault_issuer"]: - assert isinstance(credgen["vault_issuer"], str) - if credgen["base_path"]: - assert isinstance(credgen["base_path"], str) - - def test_caches(self, namespaces_json): - assert "caches" in namespaces_json - caches = namespaces_json["caches"] - # Have a reasonable number of caches - assert len(caches) > 20 - for cache in caches: - self.validate_cache_schema(cache) - - def test_namespaces(self, namespaces): - # Have a reasonable number of namespaces - assert len(namespaces) > 15 - - found_credgen = False - for namespace in namespaces: - if namespace["credential_generation"] is not None: - found_credgen = True - self.validate_namespace_schema(namespace) - if namespace["caches"]: - for cache in namespace["caches"]: - self.validate_cache_schema(cache) - assert found_credgen, "At least one namespace with credential_generation" - - @staticmethod - def validate_scitokens_block(sci): - assert sci["issuer"] - assert isinstance(sci["issuer"], str) - assert "://" in sci["issuer"] - assert isinstance(sci["basepath"], list) - assert sci["basepath"] # must have at least 1 - for bp in sci["basepath"]: - assert bp.startswith("/") # implies str - assert "," not in bp - assert isinstance(sci["restrictedpath"], list) - for rp in sci["restrictedpath"]: # may be empty - assert rp.startswith("/") # implies str - assert "," not in rp - - def test_issuers_in_namespaces(self, namespaces): - for namespace in namespaces: - assert isinstance(namespace["scitokens"], list) - for scitokens_block in namespace["scitokens"]: - self.validate_scitokens_block(scitokens_block) - - class TestAPI: def test_sanity(self, client: flask.Flask): @@ -368,6 +282,11 @@ def test_cache_grid_mapfile(self, client: flask.Flask): hashes_not_in_authfile = mapfile_hashes - authfile_hashes assert not hashes_not_in_authfile, f"Hashes in mapfile but not in authfile: {hashes_not_in_authfile}" + def test_namespaces_json(self, client): + response = client.get('/osdf/namespaces') + assert response.status_code == 200 + assert "namespaces" in response.json + class TestEndpointContent: # Pre-build some test cases based on AMNH resources diff --git a/src/tests/test_stashcache.py b/src/tests/test_stashcache.py index d92206421..3ac2f00f4 100644 --- a/src/tests/test_stashcache.py +++ b/src/tests/test_stashcache.py @@ -5,6 +5,8 @@ import re from pytest_mock import MockerFixture import time +from typing import List, Dict +import urllib, urllib.parse # Rewrites the path so the app can be imported like it normally is import os @@ -18,8 +20,12 @@ from app import app, global_data from webapp import models, topology, vos_data from webapp.common import load_yaml_file +from webapp.data_federation import CredentialGeneration import stashcache +HOST_PORT_RE = re.compile(r"[a-zA-Z0-9.-]{3,63}:[0-9]{2,5}") +PROTOCOL_HOST_PORT_RE = re.compile(r"[a-z]+://" + HOST_PORT_RE.pattern) + GRID_MAPPING_REGEX = re.compile(r'^"(/[^"]*CN=[^"]+")\s+([0-9a-f]{8}[.]0)$') # ^^ the DN starts with a slash and will at least have a CN in it. EMPTY_LINE_REGEX = re.compile(r'^\s*(#|$)') # Empty or comment-only lines @@ -218,5 +224,87 @@ def test_cache_grid_mapfile_i2_cache(self, client: flask.Flask, mocker: MockerFi assert num_mappings > 5, f"Too few mappings found.\nFull text:\n{text}\n" +class TestNamespaces: + @pytest.fixture + def namespaces_json(self, test_global_data) -> Dict: + return stashcache.get_namespaces_info(test_global_data) + + @pytest.fixture + def namespaces(self, namespaces_json) -> List[Dict]: + assert "namespaces" in namespaces_json + return namespaces_json["namespaces"] + + @staticmethod + def validate_cache_schema(cc): + assert HOST_PORT_RE.match(cc["auth_endpoint"]) + assert HOST_PORT_RE.match(cc["endpoint"]) + assert cc["resource"] and isinstance(cc["resource"], str) + + @staticmethod + def validate_namespace_schema(ns): + assert isinstance(ns["caches"], list) # we do have a case where it's empty + assert ns["path"].startswith("/") # implies str + assert isinstance(ns["readhttps"], bool) + assert isinstance(ns["usetokenonread"], bool) + assert ns["dirlisthost"] is None or PROTOCOL_HOST_PORT_RE.match(ns["dirlisthost"]) + assert ns["writebackhost"] is None or PROTOCOL_HOST_PORT_RE.match(ns["writebackhost"]) + credgen = ns["credential_generation"] + if credgen is not None: + assert isinstance(credgen["max_scope_depth"], int) and credgen["max_scope_depth"] > -1 + assert credgen["strategy"] in CredentialGeneration.STRATEGIES + assert credgen["issuer"] + parsed_issuer = urllib.parse.urlparse(credgen["issuer"]) + assert parsed_issuer.netloc and parsed_issuer.scheme == "https" + if credgen["vault_server"]: + assert isinstance(credgen["vault_server"], str) + if credgen["vault_issuer"]: + assert isinstance(credgen["vault_issuer"], str) + if credgen["base_path"]: + assert isinstance(credgen["base_path"], str) + + def test_caches(self, namespaces_json): + assert "caches" in namespaces_json + caches = namespaces_json["caches"] + # Have a reasonable number of caches + assert len(caches) > 20 + for cache in caches: + self.validate_cache_schema(cache) + + def test_namespaces(self, namespaces): + # Have a reasonable number of namespaces + assert len(namespaces) > 15 + + found_credgen = False + for namespace in namespaces: + if namespace["credential_generation"] is not None: + found_credgen = True + self.validate_namespace_schema(namespace) + if namespace["caches"]: + for cache in namespace["caches"]: + self.validate_cache_schema(cache) + assert found_credgen, "At least one namespace with credential_generation" + + @staticmethod + def validate_scitokens_block(sci): + assert sci["issuer"] + assert isinstance(sci["issuer"], str) + assert "://" in sci["issuer"] + assert isinstance(sci["basepath"], list) + assert sci["basepath"] # must have at least 1 + for bp in sci["basepath"]: + assert bp.startswith("/") # implies str + assert "," not in bp + assert isinstance(sci["restrictedpath"], list) + for rp in sci["restrictedpath"]: # may be empty + assert rp.startswith("/") # implies str + assert "," not in rp + + def test_issuers_in_namespaces(self, namespaces): + for namespace in namespaces: + assert isinstance(namespace["scitokens"], list) + for scitokens_block in namespace["scitokens"]: + self.validate_scitokens_block(scitokens_block) + + if __name__ == '__main__': pytest.main() From b76132cb87d4160b83c27eb500b3e90115fd73a4 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 21 Nov 2023 13:44:02 -0600 Subject: [PATCH 7/9] Add detailed scitokens tests for the namespaces JSON using the test VO --- src/tests/test_stashcache.py | 44 +++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/tests/test_stashcache.py b/src/tests/test_stashcache.py index 3ac2f00f4..c9817245a 100644 --- a/src/tests/test_stashcache.py +++ b/src/tests/test_stashcache.py @@ -34,7 +34,9 @@ # fake origins in our test data: TEST_ITB_HELM_ORIGIN = "helm-origin.osgdev.test.io" TEST_SC_ORIGIN = "sc-origin.test.wisc.edu" - +TEST_ORIGIN_AUTH2000 = "origin-auth2000.test.wisc.edu" +TEST_ISSUER = "https://test.wisc.edu" +TEST_BASEPATH = "/testvo" # Some DNs I can use for testing and the hashes they map to. # All of these were generated with osg-ca-generator on alma8 @@ -305,6 +307,46 @@ def test_issuers_in_namespaces(self, namespaces): for scitokens_block in namespace["scitokens"]: self.validate_scitokens_block(scitokens_block) + def test_testvo_public_namespace(self, namespaces): + ns = [ + ns for ns in namespaces if ns["path"] == "/testvo/PUBLIC" + ][0] + + assert ns["readhttps"] is False + assert ns["usetokenonread"] is False + assert TEST_SC_ORIGIN in ns["writebackhost"] + assert len(ns["caches"]) > 10 + assert len(ns["origins"]) == 2 + assert ns["credential_generation"] is None + assert len(ns["scitokens"]) == 1 + sci = ns["scitokens"][0] + assert sci["issuer"] == TEST_ISSUER + assert sci["basepath"] == [TEST_BASEPATH] + assert sci["restrictedpath"] == [] + + + def test_testvo_namespace(self, namespaces): + ns = [ + ns for ns in namespaces if ns["path"] == "/testvo" + ][0] + + assert ns["readhttps"] is True + assert ns["usetokenonread"] is True + assert TEST_ORIGIN_AUTH2000 in ns["writebackhost"] + assert TEST_ORIGIN_AUTH2000 in ns["dirlisthost"] + assert len(ns["caches"]) > 10 + assert len(ns["origins"]) == 1 + credgen = ns["credential_generation"] + assert credgen["base_path"] == TEST_BASEPATH + assert credgen["strategy"] == "OAuth2" + assert credgen["issuer"] == TEST_ISSUER + assert credgen["max_scope_depth"] == 3 + assert len(ns["scitokens"]) == 1 + sci = ns["scitokens"][0] + assert sci["issuer"] == TEST_ISSUER + assert sci["basepath"] == [TEST_BASEPATH] + assert sci["restrictedpath"] == [] + if __name__ == '__main__': pytest.main() From 5cc2029f933139587f56ea1da20ee30f068b3625 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 21 Nov 2023 13:45:57 -0600 Subject: [PATCH 8/9] Schema change for issuer info in namespaces JSON: - `basepath` -> `base_path` - `restrictedpath` -> `restricted_path` --- src/tests/test_stashcache.py | 18 +++++++++--------- src/webapp/data_federation.py | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/tests/test_stashcache.py b/src/tests/test_stashcache.py index c9817245a..a1390095f 100644 --- a/src/tests/test_stashcache.py +++ b/src/tests/test_stashcache.py @@ -291,13 +291,13 @@ def validate_scitokens_block(sci): assert sci["issuer"] assert isinstance(sci["issuer"], str) assert "://" in sci["issuer"] - assert isinstance(sci["basepath"], list) - assert sci["basepath"] # must have at least 1 - for bp in sci["basepath"]: + assert isinstance(sci["base_path"], list) + assert sci["base_path"] # must have at least 1 + for bp in sci["base_path"]: assert bp.startswith("/") # implies str assert "," not in bp - assert isinstance(sci["restrictedpath"], list) - for rp in sci["restrictedpath"]: # may be empty + assert isinstance(sci["restricted_path"], list) + for rp in sci["restricted_path"]: # may be empty assert rp.startswith("/") # implies str assert "," not in rp @@ -321,8 +321,8 @@ def test_testvo_public_namespace(self, namespaces): assert len(ns["scitokens"]) == 1 sci = ns["scitokens"][0] assert sci["issuer"] == TEST_ISSUER - assert sci["basepath"] == [TEST_BASEPATH] - assert sci["restrictedpath"] == [] + assert sci["base_path"] == [TEST_BASEPATH] + assert sci["restricted_path"] == [] def test_testvo_namespace(self, namespaces): @@ -344,8 +344,8 @@ def test_testvo_namespace(self, namespaces): assert len(ns["scitokens"]) == 1 sci = ns["scitokens"][0] assert sci["issuer"] == TEST_ISSUER - assert sci["basepath"] == [TEST_BASEPATH] - assert sci["restrictedpath"] == [] + assert sci["base_path"] == [TEST_BASEPATH] + assert sci["restricted_path"] == [] if __name__ == '__main__': diff --git a/src/webapp/data_federation.py b/src/webapp/data_federation.py index 5c252a958..5f6ab796a 100644 --- a/src/webapp/data_federation.py +++ b/src/webapp/data_federation.py @@ -104,12 +104,12 @@ def get_scitokens_conf_block(self, service_name: str): return block def get_namespaces_scitokens_block(self): - basepath = re.split(r"\s*,\s*", self.base_path) - restrictedpath = re.split(r"\s*,\s*", self.restricted_path) if self.restricted_path else [] + base_path = re.split(r"\s*,\s*", self.base_path) + restricted_path = re.split(r"\s*,\s*", self.restricted_path) if self.restricted_path else [] return { "issuer": self.issuer, - "basepath": basepath, - "restrictedpath": restrictedpath, + "base_path": base_path, + "restricted_path": restricted_path, } From 35b06293cfa59d7f9d92fa311badcbeaae814d77 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 21 Nov 2023 14:27:49 -0600 Subject: [PATCH 9/9] Document namespaces json changes (SOFTWARE-5768) --- src/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/README.md b/src/README.md index dd6d52aec..6fd97c2db 100644 --- a/src/README.md +++ b/src/README.md @@ -538,6 +538,10 @@ The JSON also contains an attribute `namespaces` that is a list of namespaces wi Note that scopes are usually relative to the namespace path. - `vault_server`: the Vault server for the `Vault` strategy or null - `vault_issuer`: the Vault issuer for the `Vault` strategy (or null). +- `scitokens` is information about any `SciTokens` sections in the `Authorizations` list for that namespace (or the empty list if there are none). Each list item has: + - `issuer`: the value of the `Issuer` field in the scitokens block + - `base_path`: a list which is the value of the `BasePath` (or `Base Path`) field split on commas + - `restricted_path`: a list which is the value of the `RestrictedPath` (or `Restricted Path`) field split on commas, or the empty list if unspecified The final result looks like ```json @@ -567,6 +571,7 @@ The final result looks like "dirlisthost": null, "path": "/xenon/PROTECTED", "readhttps": true, + "scitokens": [], "usetokenonread": false, "writebackhost": null }, @@ -582,6 +587,11 @@ The final result looks like "dirlisthost": "https://origin-auth2001.chtc.wisc.edu:1095", "path": "/ospool/PROTECTED", "readhttps": true, + "scitokens": { + "issuer": "https://osg-htc.org/ospool", + "base_path": ["/ospool/PROTECTED", "/s3.amazonaws.com/us-east-1", "/s3.amazonaws.com/us-west-1"], + "restricted_path": [] + }, "usetokenonread": true, "writebackhost": "https://origin-auth2001.chtc.wisc.edu:1095" }