diff --git a/hera_librarian/cli.py b/hera_librarian/cli.py index dd4e26e..7752e21 100644 --- a/hera_librarian/cli.py +++ b/hera_librarian/cli.py @@ -6,18 +6,18 @@ """ - - - import argparse +import json import os import sys -import json import time +from pathlib import Path + +from . import LibrarianClient +from .exceptions import LibrarianClientRemovedFunctionality, LibrarianError +from .settings import client_settings -from . import __version__, LibrarianClient, RPCError -from . import base_store -from . import utils +__version__ = "TEST" # define some common help strings @@ -86,14 +86,16 @@ def print_table(dict_list, col_list=None, col_names=None): myList = [col_list] # 1st row = header else: if len(col_names) != len(col_list): - raise ValueError("Number of column headers specified must match number of columns") + raise ValueError( + "Number of column headers specified must match number of columns" + ) myList = [col_names] for item in dict_list: - myList.append([str(item[col] or '') for col in col_list]) + myList.append([str(item[col] or "") for col in col_list]) # figure out the maximum size for each column colSize = [max(list(map(len, col))) for col in zip(*myList)] - formatStr = ' | '.join(["{{:<{}}}".format(i) for i in colSize]) - myList.insert(1, ['-' * i for i in colSize]) # Seperating line + formatStr = " | ".join(["{{:<{}}}".format(i) for i in colSize]) + myList.insert(1, ["-" * i for i in colSize]) # Seperating line for item in myList: print(formatStr.format(*item)) @@ -101,7 +103,7 @@ def print_table(dict_list, col_list=None, col_names=None): # from https://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size -def sizeof_fmt(num, suffix='B'): +def sizeof_fmt(num, suffix="B"): """Format the size of a file in human-readable values. Parameters @@ -121,11 +123,202 @@ def sizeof_fmt(num, suffix='B'): Follows the Django convention of the web search where base-10 prefixes are used, but base-2 counting is done. """ - for unit in ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z']: + for unit in ["", "k", "M", "G", "T", "P", "E", "Z"]: if abs(num) < 1024.0: return "{0:3.1f} {1:}{2:}".format(num, unit, suffix) num /= 1024.0 - return "{0:.1f} {1:}{2:}".format(num, 'Y', suffix) + return "{0:.1f} {1:}{2:}".format(num, "Y", suffix) + + +def add_file_event(args): + """ + Add a file event to a file in the librarian. + """ + + raise LibrarianClientRemovedFunctionality( + "add_file_event", "File events are no longer part of the librarian." + ) + + +def add_obs(args): + """ + Register a list of files with the librarian. + """ + + raise LibrarianClientRemovedFunctionality( + "add_obs", "Consider using the 'upload' command instead." + ) + + +def launch_copy(args): + """ + Launch a copy from one Librarian to another. + """ + + raise LibrarianClientRemovedFunctionality( + "launch_copy", + "This is no longer required as it is handled by the background tasks.", + ) + + +def assign_sessions(args): + """ + Tell the Librarian to assign any recent Observations to grouped "observing sessions". + """ + + raise LibrarianClientRemovedFunctionality( + "assign_sessions", "Observing sessions are no longer tracked." + ) + + +def check_connections(args): + """ + Check this host's ability to connect to the other Librarians that have been configured, + as well as their stores. + + """ + + any_failed = False + + for conn_name, conn_info in client_settings.connections.items(): + client = LibrarianClient.from_info(conn_info) + + try: + client.ping() + print("Connection to {} ({}) succeeded.".format(conn_name, client.hostname)) + except Exception as e: + print( + "Connection to {} ({}) failed: {}".format(conn_name, client.hostname, e) + ) + any_failed = True + + if any_failed: + sys.exit(1) + + +def copy_metadata(args): + """ + Copy metadata for files from one librarian to another. + """ + + raise LibrarianClientRemovedFunctionality( + "copy_metadata", "Metadata copying is now handled using background tasks." + ) + + +def delete_files(args): + """ + Request to delete instances of files matching a given query. + """ + + raise LibrarianClientRemovedFunctionality( + "delete_files", "Deletion is currently not available using the client." + ) + + +def initiate_offload(args): + """ + Initiate an "offload": move a bunch of file instances from one store to another. + """ + + raise LibrarianClientRemovedFunctionality( + "initiate_offload", "Offloading is now handled using background tasks." + ) + + +def locate_file(args): + """ + Ask the Librarian where to find a file. + """ + + raise NotImplementedError( + "This needs to be implemented, but requires a change to the Librarian API." + ) + + +def offload_helper(args): + """ + Launch this script to implement the "offload" functionality. + """ + + raise LibrarianClientRemovedFunctionality( + "offload_helper", "Offloading is now handled using background tasks." + ) + + +def search_files(args): + """ + Search for files in the librarian. + """ + + raise NotImplementedError( + "This needs to be implemented, but requires a change to the Librarian API." + ) + + +def set_file_deletion_policy(args): + """ + Set the "deletion policy" of one instance of this file. + """ + + raise LibrarianClientRemovedFunctionality( + "set_file_deletion_policy", + "Deletion is currently not available using the client.", + ) + + +def stage_files(args): + """ + Tell the Librarian to stage files onto the local scratch disk. + """ + + raise LibrarianClientRemovedFunctionality( + "stage_files", "Staging is now handled automatically during upload." + ) + + +def upload(args): + """ + Upload a file to a Librarian. + """ + # Argument validation is pretty simple + if os.path.isabs(args.dest_store_path): + die( + "destination path must be relative to store top; got {}".format( + args.dest_store_path + ) + ) + + if args.null_obsid and args.meta != "infer": + die('illegal to specify --null-obsid when --meta is not "infer"') + + if args.meta == "json-stdin": + raise LibrarianClientRemovedFunctionality( + "upload::json-stdin", "JSON metadata is no longer supported." + ) + elif args.meta == "infer": + pass + else: + die("unexpected metadata-gathering method {}".format(args.meta)) + + # Let's do it + client = LibrarianClient.from_info(client_settings.connections[args.conn_name]) + + try: + client.upload( + local_path=Path(args.local_path), + dest_path=Path(args.dest_store_path), + deletion_policy=args.deletion, + null_obsid=args.null_obsid, + ) + except ValueError as e: + die("Upload failed, check paths: {}".format(e)) + except LibrarianError as e: + die("Upload failed, librarian not contactable: {}".format(e)) + except Exception as e: + die("Upload failed (unknown error): {}".format(e)) + + return # make the base parser @@ -146,10 +339,11 @@ def generate_parser(): description="librarian is a command for interacting with the hera_librarian" ) ap.add_argument( - "-V", "--version", + "-V", + "--version", action="version", version="librarian {}".format(__version__), - help="Show the librarian version and exit." + help="Show the librarian version and exit.", ) # add subparsers @@ -187,14 +381,25 @@ def config_add_file_event_subparser(sub_parsers): # add sub parser sp = sub_parsers.add_parser("add-file-event", description=doc, help=hlp) - sp.add_argument("conn_name", metavar="CONNECTION-NAME", type=str, - help=_conn_name_help) - sp.add_argument("file_path", metavar="PATH/TO/FILE", type=str, - help="The path to file in librarian.") - sp.add_argument("event_type", metavar="EVENT-TYPE", type=str, - help="The type of event.") - sp.add_argument("key_vals", metavar="key1=val1...", type=str, nargs="+", - help="key-value pairs of events.") + sp.add_argument( + "conn_name", metavar="CONNECTION-NAME", type=str, help=_conn_name_help + ) + sp.add_argument( + "file_path", + metavar="PATH/TO/FILE", + type=str, + help="The path to file in librarian.", + ) + sp.add_argument( + "event_type", metavar="EVENT-TYPE", type=str, help="The type of event." + ) + sp.add_argument( + "key_vals", + metavar="key1=val1...", + type=str, + nargs="+", + help="key-value pairs of events.", + ) sp.set_defaults(func=add_file_event) return @@ -209,12 +414,20 @@ def config_add_obs_subparser(sub_parsers): # add sub parser sp = sub_parsers.add_parser("add-obs", description=doc, help=hlp) - sp.add_argument("conn_name", metavar="CONNECTION-NAME", type=str, - help=_conn_name_help) - sp.add_argument("store_name", metavar="NAME", - help="The 'store' name under which the Librarian knows this computer.") - sp.add_argument("paths", metavar="PATHS", nargs="+", - help="The paths to the files on this computer.") + sp.add_argument( + "conn_name", metavar="CONNECTION-NAME", type=str, help=_conn_name_help + ) + sp.add_argument( + "store_name", + metavar="NAME", + help="The 'store' name under which the Librarian knows this computer.", + ) + sp.add_argument( + "paths", + metavar="PATHS", + nargs="+", + help="The paths to the files on this computer.", + ) sp.set_defaults(func=add_obs) return @@ -234,12 +447,21 @@ def config_assign_session_subparser(sub_parsers): # add sub parser sp = sub_parsers.add_parser("assign-sessions", description=doc, help=hlp) - sp.add_argument("--min-start-jd", dest="minimum_start_jd", metavar="JD", type=float, - help="Only consider observations starting after JD.") - sp.add_argument("--max-start-jd", dest="maximum_start_jd", metavar="JD", type=float, - help="Only consider observations starting before JD.") - sp.add_argument("conn_name", metavar="CONNECTION-NAME", - help=_conn_name_help) + sp.add_argument( + "--min-start-jd", + dest="minimum_start_jd", + metavar="JD", + type=float, + help="Only consider observations starting after JD.", + ) + sp.add_argument( + "--max-start-jd", + dest="maximum_start_jd", + metavar="JD", + type=float, + help="Only consider observations starting before JD.", + ) + sp.add_argument("conn_name", metavar="CONNECTION-NAME", help=_conn_name_help) sp.set_defaults(func=assign_sessions) return @@ -294,14 +516,24 @@ def config_delete_files_subparser(sub_parsers): # add sub parser sp = sub_parsers.add_parser("delete-files", description=doc, help=hlp) - sp.add_argument("-n", "--noop", dest="noop", action="store_true", - help="Enable no-op mode: nothing is actually deleted.") - sp.add_argument("--store", metavar="STORE-NAME", - help="Only delete instances found on the named store.") - sp.add_argument("conn_name", metavar="CONNECTION-NAME", - help=_conn_name_help) - sp.add_argument("query", metavar="QUERY", - help="The JSON-formatted search identifying files to delete.") + sp.add_argument( + "-n", + "--noop", + dest="noop", + action="store_true", + help="Enable no-op mode: nothing is actually deleted.", + ) + sp.add_argument( + "--store", + metavar="STORE-NAME", + help="Only delete instances found on the named store.", + ) + sp.add_argument("conn_name", metavar="CONNECTION-NAME", help=_conn_name_help) + sp.add_argument( + "query", + metavar="QUERY", + help="The JSON-formatted search identifying files to delete.", + ) sp.set_defaults(func=delete_files) return @@ -318,12 +550,13 @@ def config_initiate_offload_subparser(sub_parsers): # add sub parser sp = sub_parsers.add_parser("initiate-offload", description=doc, help=hlp) - sp.add_argument("conn_name", metavar="CONNECTION-NAME", - help=_conn_name_help) - sp.add_argument("source_name", metavar="SOURCE-NAME", - help="The name of the source store.") - sp.add_argument("dest_name", metavar="DEST-NAME", - help="The name of the destination store.") + sp.add_argument("conn_name", metavar="CONNECTION-NAME", help=_conn_name_help) + sp.add_argument( + "source_name", metavar="SOURCE-NAME", help="The name of the source store." + ) + sp.add_argument( + "dest_name", metavar="DEST-NAME", help="The name of the destination store." + ) sp.set_defaults(func=initiate_offload) return @@ -342,17 +575,33 @@ def config_launch_copy_subparser(sub_parsers): # add sub parser sp = sub_parsers.add_parser("launch-copy", description=doc, help=hlp) - sp.add_argument("--dest", type=str, - help="The path in which the file should be stored at the destination. " - "Default is the same as used locally.") - sp.add_argument("--pre-staged", dest="pre_staged", metavar="STORENAME:SUBDIR", - help="Specify that the data have already been staged at the destination.") - sp.add_argument("source_conn_name", metavar="SOURCE-CONNECTION-NAME", - help="Which Librarian originates the copy; as in ~/.hl_client.cfg.") - sp.add_argument("dest_conn_name", metavar="DEST-CONNECTION-NAME", - help="Which Librarian receives the copy; as in ~/.hl_client.cfg.") - sp.add_argument("file_name", metavar="FILE-NAME", - help="The name of the file to copy; need not be a local path.") + sp.add_argument( + "--dest", + type=str, + help="The path in which the file should be stored at the destination. " + "Default is the same as used locally.", + ) + sp.add_argument( + "--pre-staged", + dest="pre_staged", + metavar="STORENAME:SUBDIR", + help="Specify that the data have already been staged at the destination.", + ) + sp.add_argument( + "source_conn_name", + metavar="SOURCE-CONNECTION-NAME", + help="Which Librarian originates the copy; as in ~/.hl_client.cfg.", + ) + sp.add_argument( + "dest_conn_name", + metavar="DEST-CONNECTION-NAME", + help="Which Librarian receives the copy; as in ~/.hl_client.cfg.", + ) + sp.add_argument( + "file_name", + metavar="FILE-NAME", + help="The name of the file to copy; need not be a local path.", + ) sp.set_defaults(func=launch_copy) return @@ -368,10 +617,8 @@ def config_locate_file_subparser(sub_parsers): # add sub parser sp = sub_parsers.add_parser("locate-file", description=doc, help=hlp) - sp.add_argument("conn_name", metavar="CONNECTION-NAME", - help=_conn_name_help) - sp.add_argument("file_name", metavar="PATH", - help="The name of the file to locate.") + sp.add_argument("conn_name", metavar="CONNECTION-NAME", help=_conn_name_help) + sp.add_argument("file_name", metavar="PATH", help="The name of the file to locate.") sp.set_defaults(func=locate_file) return @@ -386,12 +633,25 @@ def config_offload_helper_subparser(sub_parsers): # add sub parser # purposely don't add help for this function, to prevent users from using it accidentally sp = sub_parsers.add_parser("offload-helper", description=doc) - sp.add_argument("--name", required=True, help="Displayed name of the destination store.") - sp.add_argument("--pp", required=True, help='"Path prefix" of the destination store.') - sp.add_argument("--host", required=True, help="Target SSH host of the destination store.") - sp.add_argument("--destrel", required=True, help="Destination path, relative to the path prefix.") - sp.add_argument("local_path", metavar="LOCAL-PATH", - help="The name of the file to upload on this machine.") + sp.add_argument( + "--name", required=True, help="Displayed name of the destination store." + ) + sp.add_argument( + "--pp", required=True, help='"Path prefix" of the destination store.' + ) + sp.add_argument( + "--host", required=True, help="Target SSH host of the destination store." + ) + sp.add_argument( + "--destrel", + required=True, + help="Destination path, relative to the path prefix.", + ) + sp.add_argument( + "local_path", + metavar="LOCAL-PATH", + help="The name of the file to upload on this machine.", + ) sp.set_defaults(func=offload_helper) return @@ -409,11 +669,15 @@ def config_search_files_subparser(sub_parsers): hlp = "Search for files matching a query" # add sub parser - sp = sub_parsers.add_parser("search-files", description=doc, epilog=example, help=hlp) - sp.add_argument("conn_name", metavar="CONNECTION-NAME", - help=_conn_name_help) - sp.add_argument("search", metavar="JSON-SEARCH", - help="A JSON search specification; files that match will be displayed.") + sp = sub_parsers.add_parser( + "search-files", description=doc, epilog=example, help=hlp + ) + sp.add_argument("conn_name", metavar="CONNECTION-NAME", help=_conn_name_help) + sp.add_argument( + "search", + metavar="JSON-SEARCH", + help="A JSON search specification; files that match will be displayed.", + ) sp.set_defaults(func=search_files) return @@ -428,14 +692,20 @@ def config_set_file_deletion_policy_subparser(sub_parsers): # add sub parser sp = sub_parsers.add_parser("set-file-deletion-policy", description=doc, help=hlp) - sp.add_argument("--store", metavar="STORE-NAME", - help="Only alter instances found on the named store.") - sp.add_argument("conn_name", metavar="CONNECTION-NAME", - help=_conn_name_help) - sp.add_argument("file_name", metavar="FILE-NAME", - help="The name of the file to modify.") - sp.add_argument("deletion", metavar="POLICY", - help='The new deletion policy: "allowed" or "disallowed"') + sp.add_argument( + "--store", + metavar="STORE-NAME", + help="Only alter instances found on the named store.", + ) + sp.add_argument("conn_name", metavar="CONNECTION-NAME", help=_conn_name_help) + sp.add_argument( + "file_name", metavar="FILE-NAME", help="The name of the file to modify." + ) + sp.add_argument( + "deletion", + metavar="POLICY", + help='The new deletion policy: "allowed" or "disallowed"', + ) sp.set_defaults(func=set_file_deletion_policy) return @@ -454,15 +724,27 @@ def config_stage_files_subparser(sub_parsers): hlp = "Stage the files matching a query" # add sub parser - sp = sub_parsers.add_parser("stage-files", description=doc, epilog=example, help=hlp) - sp.add_argument("-w", "--wait", dest="wait", action="store_true", - help="If specified, do not exit until the staging is done.") - sp.add_argument("conn_name", metavar="CONNECTION-NAME", - help=_conn_name_help) - sp.add_argument("dest_dir", metavar="DEST-PATH", - help="What directory to put the staged files in.") - sp.add_argument("search", metavar="JSON-SEARCH", - help="A JSON search specification; files that match will be staged.") + sp = sub_parsers.add_parser( + "stage-files", description=doc, epilog=example, help=hlp + ) + sp.add_argument( + "-w", + "--wait", + dest="wait", + action="store_true", + help="If specified, do not exit until the staging is done.", + ) + sp.add_argument("conn_name", metavar="CONNECTION-NAME", help=_conn_name_help) + sp.add_argument( + "dest_dir", + metavar="DEST-PATH", + help="What directory to put the staged files in.", + ) + sp.add_argument( + "search", + metavar="JSON-SEARCH", + help="A JSON search specification; files that match will be staged.", + ) sp.set_defaults(func=stage_files) return @@ -470,14 +752,14 @@ def config_stage_files_subparser(sub_parsers): def config_upload_subparser(sub_parsers): # function documentation - doc = """Upload a file to a Librarian. Do NOT use this script if the file that you + doc = """Upload a file to a Librarian. Do NOT use this script if the file that you wish to upload is already known to the local Librarian. In that case, use the "librarian launch-copy" script -- it will make sure to preserve the associated metadata correctly. """ - example = """The LOCAL-PATH specifies where to find the source data on this machine, and + example = """The LOCAL-PATH specifies where to find the source data on this machine, and can take any form. The DEST-PATH specifies where the data should be store in the Librarian and should look something like "2345678/data.txt". The 'basename' of DEST-PATH gives the unique filename under which the data @@ -489,550 +771,76 @@ def config_upload_subparser(sub_parsers): under the name "2345678" with an empty 'store path'. """ - hlp = "Upload files to the librarian" - - # add sub parser - sp = sub_parsers.add_parser("upload", description=doc, epilog=example, help=hlp) - sp.add_argument( - "--meta", - dest="meta", - default="infer", - help='How to gather metadata: "json-stdin" or "infer"', - ) - sp.add_argument( - "--null-obsid", - dest="null_obsid", - action="store_true", - help="Require the new file to have *no* obsid associate (for maintenance files)", - ) - sp.add_argument( - "--deletion", - dest="deletion", - default="disallowed", - help=( - 'Whether the created file instance will be deletable: "allowed" or "disallowed"' - ), - ) - sp.add_argument( - "--pre-staged", - dest="pre_staged", - metavar="STORENAME:SUBDIR", - help="Specify that the data have already been staged at the destination.", - ) - sp.add_argument( - "conn_name", metavar="CONNECTION-NAME", help=_conn_name_help, - ) - sp.add_argument( - "local_path", metavar="LOCAL-PATH", help="The path to the data on this machine.", - ) - sp.add_argument( - "dest_store_path", - metavar="DEST-PATH", - help='The destination location: combination of "store path" and file name.', - ) - sp.add_argument( - "--use_globus", - dest="use_globus", - action="store_true", - help="Specify that we should try to use globus to transfer data.", - ) - sp.add_argument( - "--client_id", - dest="client_id", - metavar="CLIENT-ID", - help="The globus client ID.", - ) - sp.add_argument( - "--transfer_token", - dest="transfer_token", - metavar="TRANSFER-TOKEN", - help="The globus transfer token.", - ) - sp.add_argument( - "--source_endpoint_id", - dest="source_endpoint_id", - metavar="SOURCE-ENDPOINT-ID", - help="The source endpoint ID for the globus transfer.", - ) - sp.set_defaults(func=upload) - - return - - -def add_file_event(args): - """ - Add a file event to a file in the librarian. - """ - payload = {} - for arg in args.key_vals: - bits = arg.split("=", 1) - if len(bits) != 2: - die('argument {} must take the form "key=value"'.format(arg)) - - # We parse each "value" as JSON ... and then re-encode it as JSON when - # talking to the server. So this is mostly about sanity checking. - key, text_val = bits - try: - value = json.loads(text_val) - except ValueError: - die("value {} for keyword {} does not parse as JSON".format(text_val, key)) + hlp = "Upload files to the librarian" - payload[key] = value - - path = os.path.basename(args.file_path) # in case user provided a real filesystem path - - # Let's do it - client = LibrarianClient(args.conn_name) - - try: - client.create_file_event(path, event_type, **payload) - except RPCError as e: - die("event creation failed: {}".format(e)) - - return - - -def add_obs(args): - """ - Register a list of files with the librarian. - """ - # Load the info ... - print("Gathering information ...") - file_info = {} - - for path in args.paths: - path = os.path.abspath(path) - print(" ", path) - file_info[path] = utils.gather_info_for_path(path) - - # ... and upload what we learned - print("Registering with Librarian.") - client = LibrarianClient(args.conn_name) - try: - client.register_instances(args.store_name, file_info) - except RPCError as e: - die("RPC failed: {}".format(e)) - - return - - -def launch_copy(args): - """ - Launch a copy from one Librarian to another. - """ - # Argument validation is pretty simple - known_staging_store = None - known_staging_subdir = None - - if args.pre_staged is not None: - known_staging_store, known_staging_subdir = args.pre_staged.split(":", 1) - - # Let's do it - file_name = os.path.basename(args.file_name) # in case the user has spelled out a path - client = LibrarianClient(args.source_conn_name) - - try: - client.launch_file_copy(file_name, args.dest_conn_name, remote_store_path=args.dest, - known_staging_store=known_staging_store, - known_staging_subdir=known_staging_subdir) - except RPCError as e: - die("launch failed: {}".format(e)) - - return - - -def assign_sessions(args): - """ - Tell the Librarian to assign any recent Observations to grouped "observing sessions". - """ - # Let's do it - client = LibrarianClient(args.conn_name) - try: - result = client.assign_observing_sessions( - minimum_start_jd=args.minimum_start_jd, - maximum_start_jd=args.maximum_start_jd, - ) - except RPCError as e: - die("assignment failed: {}".format(e)) - - try: - n = 0 - - for info in result["new_sessions"]: - if n == 0: - print("New sessions created:") - print(" {id:d}: start JD {start_time_jd:f}, stop JD {stop_time_jd:f}, n_obs {n_obs:d}".format(**info)) - n += 1 - - if n == 0: - print("No new sessions created.") - except Exception as e: - die("sessions created, but failed to print info: {}".format(e)) - - return - - -def check_connections(args): - """ - Check this host's ability to connect to the other Librarians that have been configured, - as well as their stores. - - """ - from . import all_connections - - any_failed = False - - for client in all_connections(): - print('Checking ability to establish HTTP connection to "%s" (%s) ...' % (client.conn_name, client.config['url'])) - - try: - result = client.ping() - print(' ... OK') - except Exception as e: - print(' ... error: %s' % e) - any_failed = True - continue - - print(' Querying "%s" for its stores and how to connect to them ...' % client.conn_name) - - for store in client.stores(): - print(' Checking ability to establish SSH connection to remote store "%s" (%s:%s) ...' - % (store.name, store.ssh_host, store.path_prefix)) - - try: - result = store.get_space_info() - print(' ... OK') - except Exception as e: - print(' ... error: %s' % e) - any_failed = True - - if any_failed: - sys.exit(1) - - print() - print('Everything worked!') - - -def copy_metadata(args): - """ - Copy metadata for files from one librarian to another. - """ - source_client = LibrarianClient(args.source_conn_name) - dest_client = LibrarianClient(args.dest_conn_name) - - # get metadata from source... - try: - rec_info = source_client.gather_file_record(args.file_name) - except RPCError as e: - die("fetching metadata failed: {}".format(e)) - - # ...and upload it to dest - try: - dest_client.create_file_record(rec_info) - except RPCError as e: - die("uploading metadata failed: {}".format(e)) - - return - - -def delete_files(args): - """ - Request to delete instances of files matching a given query. - """ - def str_or_huh(x): - if x is None: - return "???" - return str(x) - - # Let's do it - client = LibrarianClient(args.conn_name) - - if args.noop: - print("No-op mode enabled: files will not actually be deleted.") - print() - itemtext = "todelete" - summtext = "would have been deleted" - mode = "noop" - else: - itemtext = "deleted" - summtext = "were deleted" - mode = "standard" - - try: - result = client.delete_file_instances_matching_query(args.query, - mode=mode, - restrict_to_store=args.store) - allstats = result["stats"] - except RPCError as e: - die("multi-delete failed: {}".format(e)) - - n_files = 0 - n_noinst = 0 - n_deleted = 0 - n_error = 0 - - for fname, stats in sorted(iter(allstats.items()), key=lambda t: t[0]): - nd = stats.get("n_deleted", 0) - nk = stats.get("n_kept", 0) - ne = stats.get("n_error", 0) - - if nd + nk + ne == 0: - # This file had no instances. Don't bother printing it. - n_noinst += 1 - continue - - n_files += 1 - n_deleted += nd - n_error += ne - deltext = str_or_huh(stats.get("n_deleted")) - kepttext = str_or_huh(stats.get("n_kept")) - errtext = str_or_huh(stats.get("n_error")) - - print("{0}: {1}={2} kept={3} error={4}".format(fname, itemtext, deltext, kepttext, errtext)) - - if n_files: - print("") - print("{:d} files were matched, {:d} had instances; {:d} instances {}".format( - n_files + n_noinst, n_files, n_deleted, summtext)) - - if n_error: - print("WARNING: {:d} error(s) occurred; see server logs for information".format(n_error)) - sys.exit(1) - - return - - -def initiate_offload(args): - """ - Initiate an "offload": move a bunch of file instances from one store to another. - """ - # Let's do it - client = LibrarianClient(args.conn_name) - - try: - result = client.initiate_offload(args.source_name, args.dest_name) - except RPCError as e: - die("offload failed to launch: {}".format(e)) - - if "outcome" not in result: - die('malformed server response (no "outcome" field): {}'.format(repr(result))) - - if result["outcome"] == "store-shut-down": - print("The store has no file instances needing offloading. It was placed offline and may now be closed out.") - elif result["outcome"] == "task-launched": - print("Task launched, intending to offload {} instances".format(str(result.get("instance-count", "???")))) - print() - print("A noop-ified command to delete offloaded instances from the store is:") - print(" librarian delete-files --noop --store '{}' '{}' '{\"at-least-instances\": 2}'".format( - args.source_name, args.conn_name)) - else: - die('malformed server response (unrecognized "outcome" field): {}'.format(repr(result))) - - return - - -def locate_file(args): - """ - Ask the Librarian where to find a file. - """ - # Let's do it - # In case the user has provided directory components: - file_name = os.path.basename(args.file_name) - client = LibrarianClient(args.conn_name) - - try: - result = client.locate_file_instance(file_name) - except RPCError as e: - die("couldn't locate file: {}".format(e)) - - print("{store_ssh_host}:{full_path_on_store}".format(**result)) - - return - - -def offload_helper(args): - """ - Launch this script to implement the "offload" functionality. - """ - # Due to how the Librarian has to arrange things, it's possible that the - # instance that we want to copy was deleted before this script got run. If so, - # so be it -- don't signal an error. - if not os.path.exists(args.local_path): - print("source path {} does not exist -- doing nothing".format(args.local_path)) - sys.exit(0) - - # The rare librarian script that does not use the LibrarianClient class! - try: - dest = base_store.BaseStore(args.name, args.pp, args.host) - dest.copy_to_store(args.local_path, args.destrel) - except Exception as e: - die(e) - - return - - -def search_files(args): - """ - Search for files in the librarian. - """ - # Let's do it - client = LibrarianClient(args.conn_name) - - try: - result = client.search_files(args.search) - except RPCError as e: - die("search failed: {}".format(e)) - - nresults = len(result["results"]) - if nresults == 0: - # we didn't get anything - die("No files matched this search") - - print("Found {:d} matching files".format(nresults)) - # first go through entries to format file size and remove potential null obsids - for entry in result["results"]: - entry["size"] = sizeof_fmt(entry["size"]) - if entry["obsid"] is None: - entry["obsid"] = "None" - - # now print the results as a table - print_table(result["results"], ["name", "create_time", "obsid", "type", "size"], - ["Name", "Created", "Observation", "Type", "Size"]) - - return - - -def set_file_deletion_policy(args): - """ - Set the "deletion policy" of one instance of this file. - """ - # In case they gave a full path: - file_name = os.path.basename(args.file_name) - - # Let's do it - client = LibrarianClient(args.conn_name) - try: - result = client.set_one_file_deletion_policy(file_name, args.deletion_policy, - restrict_to_store=args.store) - except RPCError as e: - die("couldn't alter policy: {}".format(e)) - - return - - -def stage_files(args): - """ - Tell the Librarian to stage files onto the local scratch disk. - """ - # Let's do it - client = LibrarianClient(args.conn_name) - - # Get the username. We could make this a command-line option but I think it's - # better to keep this a semi-secret. Note that the server does absolutely no - # verification of the values that are passed in. - - import getpass - user = getpass.getuser() - - # Resolve the destination in case the user provides, say, `.`, where the - # server is not going to know what that means. This will need elaboration if - # we add options for the server to come up with a destination automatically or - # other things like that. - our_dest = os.path.realpath(args.dest_dir) - - try: - result = client.launch_local_disk_stage_operation(user, args.search, our_dest) - except RPCError as e: - die("couldn't start the stage operation: {}".format(e)) - - # This is a bit of future-proofing; we might teach the Librarian to choose a - # "reasonable" output directory on your behalf. - dest = result["destination"] - - print("Launched operation to stage {:d} instances ({:d} bytes) to {}".format( - result["n_instances"], result["n_bytes"], dest)) - - if not args.wait: - print("Operation is complete when {}/STAGING-IN-PROGRESS is removed.".format(dest)) - else: - # The API call should not return until the progress-marker file is - # created, so if we don't see that it exists, it should be the case that - # the staging started and finished before we could check. - if not os.path.isdir(dest): - die("cannot wait for staging to complete: destination directory {} not " - "visible on this machine. Missing network filesystem?".format(dest)) - - marker_path = os.path.join(dest, "STAGING-IN-PROGRESS") - t0 = time.time() - print("Started waiting for staging to finish at:", time.asctime(time.localtime(t0))) - - while os.path.exists(marker_path): - time.sleep(3) - - if os.path.exists(os.path.join(dest, "STAGING-SUCCEEDED")): - print("Staging completed successfully ({:0.1f}s elapsed).".format(time.time() - t0)) - sys.exit(0) - - try: - with open(os.path.join(dest, "STAGING-ERRORS"), "rt") as f: - print("Staging completed WITH ERRORS ({:0.1f}s elapsed).".format(time.time() - t0), - file=sys.stderr) - print("", file=sys.stderr) - for line in f: - print(line.rstrip(), file=sys.stderr) - sys.exit(1) - except IOError as e: - if e.errno == 2: - die("staging finished but neiher \"success\" nor \"error\" indicator was " - "created (no file {})".format(dest, "STAGING-ERRORS")) - raise - - return - - -def upload(args): - """ - Upload a file to a Librarian. - """ - # Argument validation is pretty simple - if os.path.isabs(args.dest_store_path): - die("destination path must be relative to store top; got {}".format(args.dest_store_path)) - - if args.null_obsid and args.meta != "infer": - die('illegal to specify --null-obsid when --meta is not "infer"') - - if args.meta == "json-stdin": - try: - rec_info = json.load(sys.stdin) - except Exception as e: - die("cannot parse stdin as JSON data: {}".format(e)) - meta_mode = "direct" - elif args.meta == "infer": - rec_info = {} - meta_mode = "infer" - else: - die("unexpected metadata-gathering method {}".format(args.meta)) - - known_staging_store = None - known_staging_subdir = None - - if args.pre_staged is not None: - known_staging_store, known_staging_subdir = args.pre_staged.split(":", 1) - - # Let's do it - client = LibrarianClient(args.conn_name) - - try: - from pathlib import Path - - client.upload( - local_path=Path(args.local_path), - dest_path=Path(args.dest_store_path), - deletion_policy=args.deletion, - null_obsid=args.null_obsid, - ) - except RPCError as e: - die("upload failed: {}".format(e)) + # add sub parser + sp = sub_parsers.add_parser("upload", description=doc, epilog=example, help=hlp) + sp.add_argument( + "--meta", + dest="meta", + default="infer", + help='How to gather metadata: "json-stdin" or "infer"', + ) + sp.add_argument( + "--null-obsid", + dest="null_obsid", + action="store_true", + help="Require the new file to have *no* obsid associate (for maintenance files)", + ) + sp.add_argument( + "--deletion", + dest="deletion", + default="disallowed", + help=( + 'Whether the created file instance will be deletable: "allowed" or "disallowed"' + ), + ) + sp.add_argument( + "--pre-staged", + dest="pre_staged", + metavar="STORENAME:SUBDIR", + help="Specify that the data have already been staged at the destination.", + ) + sp.add_argument( + "conn_name", + metavar="CONNECTION-NAME", + help=_conn_name_help, + ) + sp.add_argument( + "local_path", + metavar="LOCAL-PATH", + help="The path to the data on this machine.", + ) + sp.add_argument( + "dest_store_path", + metavar="DEST-PATH", + help='The destination location: combination of "store path" and file name.', + ) + sp.add_argument( + "--use_globus", + dest="use_globus", + action="store_true", + help="Specify that we should try to use globus to transfer data.", + ) + sp.add_argument( + "--client_id", + dest="client_id", + metavar="CLIENT-ID", + help="The globus client ID.", + ) + sp.add_argument( + "--transfer_token", + dest="transfer_token", + metavar="TRANSFER-TOKEN", + help="The globus transfer token.", + ) + sp.add_argument( + "--source_endpoint_id", + dest="source_endpoint_id", + metavar="SOURCE-ENDPOINT-ID", + help="The source endpoint ID for the globus transfer.", + ) + sp.set_defaults(func=upload) return diff --git a/hera_librarian/client.py b/hera_librarian/client.py index fab8eb0..c2211c9 100644 --- a/hera_librarian/client.py +++ b/hera_librarian/client.py @@ -13,6 +13,7 @@ from .models.ping import PingRequest, PingResponse from .models.uploads import (UploadCompletionRequest, UploadInitiationRequest, UploadInitiationResponse) +from .settings import ClientInfo from .utils import get_md5_from_path, get_size_from_path if TYPE_CHECKING: @@ -53,6 +54,28 @@ def __init__(self, host: str, port: int, user: str): def __repr__(self): return f"Librarian Client ({self.user}) for {self.host}:{self.port}" + @classmethod + def from_info(cls, client_info: ClientInfo): + """ + Create a LibrarianClient from a ClientInfo object. + + Parameters + ---------- + client_info : ClientInfo + The ClientInfo object. + + Returns + ------- + LibrarianClient + The LibrarianClient. + """ + + return cls( + host=client_info.host, + port=client_info.port, + user=client_info.user, + ) + @property def hostname(self): return f"{self.host}:{self.port}/api/v2" diff --git a/hera_librarian/exceptions.py b/hera_librarian/exceptions.py index c8b5094..f75eaaa 100644 --- a/hera_librarian/exceptions.py +++ b/hera_librarian/exceptions.py @@ -17,3 +17,10 @@ def __init__(self, url, status_code, reason, suggested_remedy): class LibrarianError(Exception): def __init__(self, message): super(LibrarianError, self).__init__(message) + + +class LibrarianClientRemovedFunctionality(Exception): + def __Init__(self, name, message): + super(LibrarianClientRemovedFunctionality, self).__init__( + f"{name} is no longer avaialble in Librarian v2.0. {message}" + ) diff --git a/hera_librarian/settings.py b/hera_librarian/settings.py new file mode 100644 index 0000000..68bcbbd --- /dev/null +++ b/hera_librarian/settings.py @@ -0,0 +1,89 @@ +""" +Client settings. +""" + +import os +from pathlib import Path + +from pydantic import BaseModel +from pydantic_settings import BaseSettings + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + client_settings: "ClientSettings" + + +class ClientInfo(BaseModel): + """ + Information for an individual client. + """ + + user: str + "Your username on this librarian" + port: int + "The port of this librarian server" + host: str + "The hostname of this librarian server" + + +class ClientSettings(BaseSettings): + connections: dict[str, ClientInfo] = {} + + @classmethod + def from_file(cls, config_path: Path | str) -> "ClientSettings": + """ + Loads the settings from the given path. + """ + + with open(config_path, "r") as handle: + return cls.model_validate_json(handle.read()) + + +# Automatically create a settings object on use. + +_settings = None + + +def load_settings() -> ClientSettings: + """ + Load the settings from the config file. + """ + + global _settings + + try_paths = [ + os.environ.get("HL_CLIENT_CONFIG", None), + Path.home() / ".hl_client.cfg", + Path.home() / ".hl_client.json", + ] + + for path in try_paths: + if path is not None: + path = Path(path) + else: + continue + + if path.exists(): + _settings = ClientSettings.from_file(path) + return _settings + + _settings = ClientSettings() + + return _settings + + +def __getattr__(name): + """ + Try to load the settings if they haven't been loaded yet. + """ + + if name == "client_settings": + global _settings + + if _settings is not None: + return _settings + + return load_settings() + + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") diff --git a/hera_librarian/tests/__init__.py b/hera_librarian/tests/__init__.py deleted file mode 100644 index 2911d77..0000000 --- a/hera_librarian/tests/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright 2019 the HERA Collaboration -# Licensed under the 2-clause BSD License - -"""Define test data files and attributes -""" - -import pytest -import os - - -# define where to find the data and their properties -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "test_data") - -ALL_FILES = pytest.mark.datafiles( - os.path.join(DATA_DIR, "zen.2458043.12552.xx.HH.uvA"), - os.path.join(DATA_DIR, "zen.2458432.34569.uvh5"), - keep_top_dir=True, -) - -filetypes = ["uvA", "uvh5"] - -obsids = [1192201262, 1225829886] # miriad, uvh5 - -md5sums = [ - "ab038eee080348eaa5abd221ec702a67", # miriad - "291a451139cf16e73d880437270dd0ed", # uvh5 -] - -pathsizes = [983251, 224073] # uvh5, miriad diff --git a/hera_librarian/tests/test_base_store.py b/hera_librarian/tests/test_base_store.py deleted file mode 100644 index c8614d8..0000000 --- a/hera_librarian/tests/test_base_store.py +++ /dev/null @@ -1,217 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright 2019 the HERA Collaboration -# Licensed under the 2-clause BSD License - -"""Test code in hera_librarian/store.py - -""" - -from __future__ import print_function, division, absolute_import -import pytest -import os -import tempfile -import json -import shutil -from hera_librarian import base_store, RPCError - -from . import ALL_FILES, filetypes, obsids, md5sums, pathsizes - - -@pytest.fixture -def local_store(): - tempdir = tempfile.mkdtemp(dir="/tmp") - return base_store.BaseStore("local_store", tempdir, "localhost"), tempdir - - -def test_path(local_store): - # test that store path is prepended - dirpath = os.path.join(local_store[1], "my_dir") - assert local_store[0]._path("my_dir") == dirpath - - # test passing in an absolute path - with pytest.raises(ValueError): - local_store[0]._path("/tmp/my_dir") - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_ssh_slurp(local_store): - # test simple command - assert local_store[0]._ssh_slurp("echo hello world").decode("utf-8") == "hello world\n" - - # try a bogus command - with pytest.raises(RPCError): - local_store[0]._ssh_slurp("foo") - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_copy_to_store(tmpdir, local_store): - # make a fake file in our tmpdir - tmppath = os.path.join(str(tmpdir), "my_file.txt") - with open(tmppath, "w") as f: - print("hello world", file=f) - - # copy it over - local_store[0].copy_to_store(str(tmpdir), "test_directory") - - # check that it exists - dirpath = os.path.join(local_store[1], "test_directory") - assert local_store[0]._ssh_slurp("ls {}".format(dirpath)).decode("utf-8") == "my_file.txt\n" - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_chmod(local_store): - # make a small test file on the store, then change permissions - tempdir = local_store[1] - temppath = os.path.join(tempdir, "my_empty_file") - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._chmod("my_empty_file", "664") - - # make sure permissions are correct - output = local_store[0]._ssh_slurp("ls -l {}".format(temppath)).decode("utf-8") - perms = output.split(" ")[0] - assert perms == "-rw-rw-r--" - - # clean up - shutil.rmtree(os.path.join("/tmp", local_store[1])) - - return - - -def test_move(local_store): - # test moving a file - temppath = os.path.join(local_store[1], "my_empty_file") - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._move("my_empty_file", "my_moved_file") - temppath2 = os.path.join("/tmp", local_store[1], "my_moved_file") - assert local_store[0]._ssh_slurp("ls {}".format(temppath2)).decode("utf-8") == "{}\n".format(temppath2) - - # test trying to overwrite a file that already exists - with pytest.raises(RPCError): - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._move("my_empty_file", "my_moved_file") - - # remove existing files; test using chmod - local_store[0]._ssh_slurp("rm -f {} {}".format(temppath, temppath2)) - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._move("my_empty_file", "my_moved_file", chmod_spec=664) - output = local_store[0]._ssh_slurp("ls -l {}".format(temppath2)).decode("utf-8") - perms = output.split(" ")[0] - assert perms == "-rw-rw-r--" - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_delete(local_store): - # test removing a file - temppath = os.path.join(local_store[1], "my_empty_file") - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._delete("my_empty_file") - assert ( - local_store[0]._ssh_slurp( - "if [ -f {} ]; then echo file_still_exists; fi".format(temppath) - ).decode("utf-8") - == "" - ) - - # test deleting a write-protected file - tempdir = os.path.join(local_store[1], "my_empty_dir") - local_store[0]._ssh_slurp("mkdir {0}; chmod 755 {0}".format(tempdir)) - local_store[0]._delete("my_empty_dir", chmod_before=True) - assert ( - local_store[0]._ssh_slurp( - "if [ -d {} ]; then echo dir_still_exists; fi".format(tempdir) - ).decode("utf-8") - == "" - ) - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_create_tempdir(local_store): - # make sure no temp dirs currently exist on host - tempdir = os.path.join(local_store[1]) - local_store[0]._ssh_slurp("rm -rf {}/libtmp.*".format(tempdir)) - tmppath = local_store[0]._create_tempdir() - # we don't know exactly what the directory name will be, because a random - # 6-digit string is appended to the end - assert tmppath.startswith("libtmp.") - assert len(tmppath) == len("libtmp.") + 6 - # make sure it exists on the host - assert ( - local_store[0]._ssh_slurp("ls -d1 {}/{}".format(tempdir, tmppath)).decode("utf-8") - == "{}/{}\n".format(tempdir, tmppath) - ) - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -@ALL_FILES -def test_get_info_for_path(local_store, datafiles): - # copy a datafile to store directory, so we can get its info - filepaths = sorted(list(map(str, datafiles.listdir()))) - filename = os.path.basename(filepaths[0]) - local_store[0].copy_to_store(filepaths[0], filename) - - # get the file info and check that it's right - info = local_store[0].get_info_for_path(filename) - # make a dict of the correct answers - # the uvh5 properties are first in these lists - correct_dict = { - "md5": md5sums[0], - "obsid": obsids[0], - "type": filetypes[0], - "size": pathsizes[0], - } - assert info == correct_dict - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_get_space_info(local_store): - # get the disk information of the store - info = local_store[0].get_space_info() - assert "used" in info.keys() - assert "available" in info.keys() - assert "total" in info.keys() - assert info["used"] + info["available"] == info["total"] - - # test using the cache -- make sure the info is the same - info_cached = local_store[0].get_space_info() - assert info == info_cached - - # we also test the capacity, space_left, and usage_percentage properties - capacity = local_store[0].capacity - space_left = local_store[0].space_left - usage_percentage = local_store[0].usage_percentage - assert capacity == info["total"] - assert space_left == info["available"] - assert usage_percentage == pytest.approx(100.0 * info["used"] / info["total"]) - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/flags b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/flags deleted file mode 100644 index 2853c1e..0000000 Binary files a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/flags and /dev/null differ diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/header b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/header deleted file mode 100644 index ce345b7..0000000 Binary files a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/header and /dev/null differ diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/history b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/history deleted file mode 100644 index a6c150a..0000000 --- a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/history +++ /dev/null @@ -1,3 +0,0 @@ -CORR-DACQ: created file. - Read/written with pyuvdata version: 1.1. Git origin: https://github.com/HERA-Team/pyuvdata.git. Git hash: 8c65e5432ad70a38102b6c09ad5851e4de10f76a. Git branch: h1c. Git description: v1.1-533-g8c65e54. Downselected to specific antennas using pyuvdata. Hera Hex antennas selected with hera_cal/scripts/extract_hh.py, hera_cal version: {'git_hash': '8de90f0f2afe01adec5a5e317716af5a8e7811b4', 'version': '1.0', 'git_description': 'v1.0', 'git_branch': 'h1c', 'git_origin': 'https://github.com/HERA-Team/hera_cal.git'}. - Read/written with pyuvdata version: 1.2.1. Git origin: https://github.com/HERA-Team/pyuvdata. Git hash: 1a3bf03ea8859d02a5c5e22fe5290f17fad653b4. Git branch: master. Git description: v1.2-49-g1a3bf03. Downselected to specific antennas, frequencies using pyuvdata. diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/vartable b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/vartable deleted file mode 100644 index da99589..0000000 --- a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/vartable +++ /dev/null @@ -1,39 +0,0 @@ -r corr -i nchan -i npol -i nspect -d inttime -d sdf -a source -a telescop -d latitud -d longitu -d antdiam -i nschan -i ischan -i nants -d antpos -d sfreq -i ntimes -i nbls -i nblts -a visunits -a instrume -d altitude -a cminfo -a st_type -d stopt -d startt -i obsid -a cmver -d duration -d antnums -a antnames -d lst -d ra -d dec -i pol -d cnt -d coord -d time -r baseline diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/visdata b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/visdata deleted file mode 100644 index 9e1a3cf..0000000 Binary files a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/visdata and /dev/null differ diff --git a/hera_librarian/tests/test_data/zen.2458432.34569.uvh5 b/hera_librarian/tests/test_data/zen.2458432.34569.uvh5 deleted file mode 100644 index a4bf6b9..0000000 Binary files a/hera_librarian/tests/test_data/zen.2458432.34569.uvh5 and /dev/null differ diff --git a/hera_librarian/tests/test_utils.py b/hera_librarian/tests/test_utils.py deleted file mode 100644 index 5aadfe0..0000000 --- a/hera_librarian/tests/test_utils.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright 2019 the HERA Collaboration -# Licensed under the 2-clause BSD License - -"""Test code in hera_librarian/utils.py - -""" - -import pytest -import os -import six -import sys -import json -from contextlib import contextmanager -from hera_librarian import utils - -# import test data attributes from __init__.py -from . import ALL_FILES, obsids, filetypes, md5sums, pathsizes - - -def test_get_type_from_path(): - """Test type checking from path""" - path = "/some/long/file.name.txt" - assert utils.get_type_from_path(path) == "txt" - - return - - -def test_get_pol_from_path(): - """Test polarization extraction from filename""" - filename = "zen.2458000.12345.xx.uvh5" - assert utils.get_pol_from_path(filename) == "xx" - - filename = "zen.2458000.12345.uvh5" - assert utils.get_pol_from_path(filename) is None - - return - - -@pytest.mark.filterwarnings("ignore:numpy.ufunc size changed") -@ALL_FILES -def test_get_obsid_from_path(datafiles): - """Test extracting obsid values from datasets""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - for obsid, path in zip(obsids, filepaths): - assert utils.get_obsid_from_path(path) == obsid - - return - - -def test_normalize_and_validate_md5(): - """Test md5sum normalization""" - md5sum = "d41d8cd98f00b204e9800998ecf8427e" - # function does not do anything for text already lowercase - assert utils.normalize_and_validate_md5(md5sum) == md5sum - - md5sum_padded = md5sum + " " - assert utils.normalize_and_validate_md5(md5sum_padded) == md5sum - - md5sum_upper = md5sum.upper() + " " - assert utils.normalize_and_validate_md5(md5sum_upper) == md5sum - - # make sure error is raised when length is incorrect - with pytest.raises(ValueError): - utils.normalize_and_validate_md5(md5sum[:-1]) - - return - - -@ALL_FILES -def test_md5_of_file(datafiles): - """Test generating md5sum of file""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - assert utils._md5_of_file(filepaths[1]) == md5sums[1] - - return - - -@ALL_FILES -def test_get_md5_from_path(datafiles): - """Test getting the md5sum for both a flat file and directory""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - # test normal execution - for md5sum, path in zip(md5sums, filepaths): - assert utils.get_md5_from_path(path) == md5sum - - # test adding funny bits to the ends of the directory names - datafile_miriad = filepaths[0] + "//." - assert utils.get_md5_from_path(datafile_miriad) == md5sums[0] - - return - - -@ALL_FILES -def test_get_size_from_path(datafiles): - """Test computing filesize from path""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - for pathsize, path in zip(pathsizes, filepaths): - assert utils.get_size_from_path(path) == pathsize - - return - - -@ALL_FILES -def test_gather_info_for_path(datafiles): - """Test getting all info for a given path""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - for filetype, md5, size, obsid, path in zip( - filetypes, md5sums, pathsizes, obsids, filepaths - ): - info = utils.gather_info_for_path(path) - assert info["type"] == filetype - assert info["md5"] == md5 - assert info["size"] == size - assert info["obsid"] == obsid - - return - - -@ALL_FILES -def test_print_info_for_path(datafiles, capsys): - """Test printing file info to stdout""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - for filetype, md5, size, obsid, path in zip( - filetypes, md5sums, pathsizes, obsids, filepaths - ): - utils.print_info_for_path(path) - out, err = capsys.readouterr() - # convert from json to dict - out_dict = json.loads(out) - - # build up correct dict - correct_info = {"type": filetype, "md5": md5, "size": size, "obsid": obsid} - assert out_dict == correct_info - - return - - -def test_format_jd_as_calendar_date(): - """Test converting JD to calendar date""" - jd = 2456000 - assert utils.format_jd_as_calendar_date(jd) == "2012-03-13" - - return - - -def test_format_jd_as_iso_date_time(): - """Test converting JD to ISO datetime""" - jd = 2456000 - assert utils.format_jd_as_iso_date_time(jd) == "2012-03-13 12:00:00" - - return - - -def test_format_obsid_as_calendar_date(): - """Test converting obsid to calendar date""" - assert utils.format_obsid_as_calendar_date(obsids[1]) == "2018-11-09" - - return diff --git a/librarian_server/settings.py b/librarian_server/settings.py index a193b61..8ece347 100644 --- a/librarian_server/settings.py +++ b/librarian_server/settings.py @@ -3,13 +3,19 @@ deserialized from the available librarian config path. """ -from pydantic import BaseModel, field_validator, ValidationError +import os +from pathlib import Path + +from pydantic import BaseModel, ValidationError, field_validator from pydantic_settings import BaseSettings from .stores import StoreNames -from pathlib import Path -import os +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + server_settings: "ServerSettings" + class StoreSettings(BaseModel): """ @@ -49,7 +55,7 @@ class ServerSettings(BaseSettings): secret_key: str sqlalchemy_database_uri: str - sqlalchemy_track_modifications: bool + sqlalchemy_track_modifications: bool log_level: str = "DEBUG" displayed_site_name: str = "Untitled Librarian" @@ -71,8 +77,53 @@ def from_file(cls, config_path: Path | str) -> "ServerSettings": with open(config_path, "r") as handle: return cls.model_validate_json(handle.read()) - + # Automatically create a variable, server_settings, from the environment variable -# on import! +# on _use_! + +_settings = None + + +def load_settings() -> ServerSettings: + """ + Load the settings from the config file. + """ + + global _settings + + try_paths = [ + os.environ.get("LIBRARIAN_CONFIG_PATH", None), + ] + + for path in try_paths: + if path is not None: + path = Path(path) + else: + continue + + if path.exists(): + _settings = ServerSettings.from_file(path) + return _settings + + _settings = ServerSettings() + + return _settings + + +def __getattr__(name): + """ + Try to load the settings if they haven't been loaded yet. + """ + + if name == "HELLO_WORLD": + return "Hello World!" + + if name == "server_settings": + global _settings + + if _settings is not None: + return _settings + + return load_settings() -server_settings = ServerSettings.from_file(os.environ["LIBRARIAN_CONFIG_PATH"]) \ No newline at end of file + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") diff --git a/pyproject.toml b/pyproject.toml index be826a5..17fb4d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,9 +68,10 @@ Legacy = "https://github.com/HERA-Team/librarian/" [tool.pytest.ini_options] testpaths = [ - "tests/integration_test", "tests/server_unit_test", "tests/background_unit_test", + "tests/client_unit_test", + "tests/integration_test", ] [tool.coverage.run] diff --git a/tests/client_unit_test/__init__.py b/tests/client_unit_test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hera_librarian/tests/test_cli.py b/tests/client_unit_test/test_cli.py similarity index 95% rename from hera_librarian/tests/test_cli.py rename to tests/client_unit_test/test_cli.py index e9bfb50..ea84d44 100644 --- a/hera_librarian/tests/test_cli.py +++ b/tests/client_unit_test/test_cli.py @@ -7,12 +7,14 @@ """ -import pytest import os +import pytest + import hera_librarian from hera_librarian import cli + def test_die(capsys): # test without specifying replacement args with pytest.raises(SystemExit) as e: @@ -75,6 +77,7 @@ def test_print_table(capsys): return + def test_sizeof_fmt(): # test a few known values bts = 512 @@ -126,9 +129,3 @@ def test_generate_parser(): assert "upload" in available_subparsers return - - -def test_main(script_runner): - version = hera_librarian.__version__ - ret = script_runner.run("librarian", "-V") - assert ret.stdout == "librarian {}\n".format(version) diff --git a/tests/client_unit_test/test_cli_expected.py b/tests/client_unit_test/test_cli_expected.py new file mode 100644 index 0000000..9e9ced9 --- /dev/null +++ b/tests/client_unit_test/test_cli_expected.py @@ -0,0 +1,42 @@ +""" +Tests that the CLI does what we expect it to. +""" + +import subprocess + + +def test_should_fail_removed(): + """ + Tests that our CLI fails when we have removed functionality. + """ + + calls = [ + ["add-file-event", "ABC", "/path/to/nowhere", "A", "AB=CD"], + ["add-obs", "ABC", "STORE", "/path/to/nowhere"], + ["assign-sessions", "ABC", "--min-start-jd=2400", "--max-start-jd=2450"], + ["copy-metadata", "ABC", "DEF", "abcd"], + ["delete-files", "DBC", "QUERY", "--store=ABC"], + ["initiate-offload", "ABC", "SOURCE", "DEST"], + ["launch-copy", "SOURCE", "DEST", "FILE"], + [ + "offload-helper", + "LOCAL", + "--name=HELLO", + "--pp=NONE", + "--host=HOST", + "--destrel=REL", + ], + ["set-file-deletion-policy", "CONN", "FILENAME", "DISALLOWED", "--store=hello"], + ["stage-files", "ABC", "DEF", "GHI"], + ] + + for call in calls: + with subprocess.Popen( + ["librarian", *call], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as proc: + _, stderr = proc.communicate() + + assert proc.returncode != 0 + assert b"LibrarianClientRemovedFunctionality" in stderr