From 806f375e87d571cbe1a057cbdaee1ff42550930f Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 18 Sep 2024 20:11:44 +0000 Subject: [PATCH 1/4] refactor: fork sshtunnel to raise exceptions --- src/teamster/libraries/ssh/sshtunnel.py | 1684 +++++++++++++++++++++++ 1 file changed, 1684 insertions(+) create mode 100644 src/teamster/libraries/ssh/sshtunnel.py diff --git a/src/teamster/libraries/ssh/sshtunnel.py b/src/teamster/libraries/ssh/sshtunnel.py new file mode 100644 index 0000000000..015b621892 --- /dev/null +++ b/src/teamster/libraries/ssh/sshtunnel.py @@ -0,0 +1,1684 @@ +# trunk-ignore-all(pyright) + +import getpass +import logging +import os +import queue +import random +import socket +import socketserver +import string +import sys +import threading +import warnings +from binascii import hexlify +from select import select + +import paramiko + +string_types = str +input_ = input + + +__version__ = "0.4.0" +__author__ = "pahaz" + + +#: Timeout (seconds) for transport socket (``socket.settimeout``) +SSH_TIMEOUT = 0.1 # ``None`` may cause a block of transport thread +#: Timeout (seconds) for tunnel connection (open_channel timeout) +TUNNEL_TIMEOUT = 10.0 + +_DAEMON = True #: Use daemon threads in connections +_DEPRECATIONS = { + "ssh_address": "ssh_address_or_host", + "ssh_host": "ssh_address_or_host", + "ssh_private_key": "ssh_pkey", + "raise_exception_if_any_forwarder_have_a_problem": "mute_exceptions", +} + +# logging +DEFAULT_LOGLEVEL = logging.ERROR #: default level if no logger passed (ERROR) +TRACE_LEVEL = 1 +logging.addLevelName(TRACE_LEVEL, "TRACE") +DEFAULT_SSH_DIRECTORY = "~/.ssh" + +_StreamServer = ( + socketserver.UnixStreamServer if os.name == "posix" else socketserver.TCPServer +) + +#: Path of optional ssh configuration file +DEFAULT_SSH_DIRECTORY = "~/.ssh" +SSH_CONFIG_FILE = os.path.join(DEFAULT_SSH_DIRECTORY, "config") + +######################## +# # +# Utils # +# # +######################## + + +def check_host(host): + assert isinstance(host, string_types), "IP is not a string ({0})".format( + type(host).__name__ + ) + + +def check_port(port): + assert isinstance(port, int), "PORT is not a number" + assert port >= 0, "PORT < 0 ({0})".format(port) + + +def check_address(address): + """ + Check if the format of the address is correct + + Arguments: + address (tuple): + (``str``, ``int``) representing an IP address and port, + respectively + + .. note:: + alternatively a local ``address`` can be a ``str`` when working + with UNIX domain sockets, if supported by the platform + Raises: + ValueError: + raised when address has an incorrect format + + Example: + >>> check_address(('127.0.0.1', 22)) + """ + if isinstance(address, tuple): + check_host(address[0]) + check_port(address[1]) + elif isinstance(address, string_types): + if os.name != "posix": + raise ValueError("Platform does not support UNIX domain sockets") + if not ( + os.path.exists(address) or os.access(os.path.dirname(address), os.W_OK) + ): + raise ValueError( + "ADDRESS not a valid socket domain socket ({0})".format(address) + ) + else: + raise ValueError( + "ADDRESS is not a tuple, string, or character buffer " "({0})".format( + type(address).__name__ + ) + ) + + +def check_addresses(address_list, is_remote=False): + """ + Check if the format of the addresses is correct + + Arguments: + address_list (list[tuple]): + Sequence of (``str``, ``int``) pairs, each representing an IP + address and port respectively + + .. note:: + when supported by the platform, one or more of the elements in + the list can be of type ``str``, representing a valid UNIX + domain socket + + is_remote (boolean): + Whether or not the address list + Raises: + AssertionError: + raised when ``address_list`` contains an invalid element + ValueError: + raised when any address in the list has an incorrect format + + Example: + + >>> check_addresses([('127.0.0.1', 22), ('127.0.0.1', 2222)]) + """ + assert all(isinstance(x, (tuple, string_types)) for x in address_list) + if is_remote and any(isinstance(x, string_types) for x in address_list): + raise AssertionError("UNIX domain sockets not allowed for remote" "addresses") + + for address in address_list: + check_address(address) + + +def create_logger( + logger=None, loglevel=None, capture_warnings=True, add_paramiko_handler=True +): + """ + Attach or create a new logger and add a console handler if not present + + Arguments: + + logger (Optional[logging.Logger]): + :class:`logging.Logger` instance; a new one is created if this + argument is empty + + loglevel (Optional[str or int]): + :class:`logging.Logger`'s level, either as a string (i.e. + ``ERROR``) or in numeric format (10 == ``DEBUG``) + + .. note:: a value of 1 == ``TRACE`` enables Tracing mode + + capture_warnings (boolean): + Enable/disable capturing the events logged by the warnings module + into ``logger``'s handlers + + Default: True + + .. note:: ignored in python 2.6 + + add_paramiko_handler (boolean): + Whether or not add a console handler for ``paramiko.transport``'s + logger if no handler present + + Default: True + Return: + :class:`logging.Logger` + """ + logger = logger or logging.getLogger("sshtunnel.SSHTunnelForwarder") + if not any(isinstance(x, logging.Handler) for x in logger.handlers): + logger.setLevel(loglevel or DEFAULT_LOGLEVEL) + console_handler = logging.StreamHandler() + _add_handler( + logger, handler=console_handler, loglevel=loglevel or DEFAULT_LOGLEVEL + ) + if loglevel: # override if loglevel was set + logger.setLevel(loglevel) + for handler in logger.handlers: + handler.setLevel(loglevel) + + if add_paramiko_handler: + _check_paramiko_handlers(logger=logger) + + if capture_warnings and sys.version_info >= (2, 7): + logging.captureWarnings(True) + pywarnings = logging.getLogger("py.warnings") + pywarnings.handlers.extend(logger.handlers) + return logger + + +def _add_handler(logger, handler=None, loglevel=None): + """ + Add a handler to an existing logging.Logger object + """ + handler.setLevel(loglevel or DEFAULT_LOGLEVEL) + if handler.level <= logging.DEBUG: + _fmt = ( + "%(asctime)s| %(levelname)-4.3s|%(threadName)10.9s/" + "%(lineno)04d@%(module)-10.9s| %(message)s" + ) + handler.setFormatter(logging.Formatter(_fmt)) + else: + handler.setFormatter( + logging.Formatter("%(asctime)s| %(levelname)-8s| %(message)s") + ) + logger.addHandler(handler) + + +def _check_paramiko_handlers(logger=None): + """ + Add a console handler for paramiko.transport's logger if not present + """ + paramiko_logger = logging.getLogger("paramiko.transport") + if not paramiko_logger.handlers: + if logger: + paramiko_logger.handlers = logger.handlers + else: + console_handler = logging.StreamHandler() + console_handler.setFormatter( + logging.Formatter( + "%(asctime)s | %(levelname)-8s| PARAMIKO: " + "%(lineno)03d@%(module)-10s| %(message)s" + ) + ) + paramiko_logger.addHandler(console_handler) + + +def address_to_str(address): + if isinstance(address, tuple): + return "{0[0]}:{0[1]}".format(address) + return str(address) + + +def _remove_none_values(dictionary): + """Remove dictionary keys whose value is None""" + return list(map(dictionary.pop, [i for i in dictionary if dictionary[i] is None])) + + +def generate_random_string(length): + letters = string.ascii_letters + string.digits + return "".join(random.choice(letters) for _ in range(length)) + + +######################## +# # +# Errors # +# # +######################## + + +class BaseSSHTunnelForwarderError(Exception): + """Exception raised by :class:`SSHTunnelForwarder` errors""" + + def __init__(self, *args, **kwargs): + self.value = kwargs.pop("value", args[0] if args else "") + + def __str__(self): + return self.value + + +class HandlerSSHTunnelForwarderError(BaseSSHTunnelForwarderError): + """Exception for Tunnel forwarder errors""" + + pass + + +######################## +# # +# Handlers # +# # +######################## + + +class _ForwardHandler(socketserver.BaseRequestHandler): + """Base handler for tunnel connections""" + + remote_address = None + ssh_transport = None + logger = None + info = None + + def _redirect(self, chan): + while chan.active: + rqst, _, _ = select([self.request, chan], [], [], 5) + if self.request in rqst: + data = self.request.recv(16384) + if not data: + self.logger.log( + TRACE_LEVEL, ">>> OUT {0} recv empty data >>>".format(self.info) + ) + break + if self.logger.isEnabledFor(TRACE_LEVEL): + self.logger.log( + TRACE_LEVEL, + ">>> OUT {0} send to {1}: {2} >>>".format( + self.info, self.remote_address, hexlify(data) + ), + ) + chan.sendall(data) + if chan in rqst: # else + if not chan.recv_ready(): + self.logger.log( + TRACE_LEVEL, + "<<< IN {0} recv is not ready <<<".format(self.info), + ) + break + data = chan.recv(16384) + if self.logger.isEnabledFor(TRACE_LEVEL): + hex_data = hexlify(data) + self.logger.log( + TRACE_LEVEL, + "<<< IN {0} recv: {1} <<<".format(self.info, hex_data), + ) + self.request.sendall(data) + + def handle(self): + uid = generate_random_string(5) + self.info = "#{0} <-- {1}".format( + uid, self.client_address or self.server.local_address + ) + src_address = self.request.getpeername() + if not isinstance(src_address, tuple): + src_address = ("dummy", 12345) + try: + chan = self.ssh_transport.open_channel( + kind="direct-tcpip", + dest_addr=self.remote_address, + src_addr=src_address, + timeout=TUNNEL_TIMEOUT, + ) + except Exception as e: # pragma: no cover + msg_tupe = "ssh " if isinstance(e, paramiko.SSHException) else "" + exc_msg = "open new channel {0}error: {1}".format(msg_tupe, e) + log_msg = "{0} {1}".format(self.info, exc_msg) + self.logger.log(TRACE_LEVEL, log_msg) + raise HandlerSSHTunnelForwarderError(exc_msg) from e + + self.logger.log(TRACE_LEVEL, "{0} connected".format(self.info)) + try: + self._redirect(chan) + except socket.error: + # Sometimes a RST is sent and a socket error is raised, treat this + # exception. It was seen that a 3way FIN is processed later on, so + # no need to make an ordered close of the connection here or raise + # the exception beyond this point... + self.logger.log(TRACE_LEVEL, "{0} sending RST".format(self.info)) + except Exception as e: + self.logger.log(TRACE_LEVEL, "{0} error: {1}".format(self.info, repr(e))) + finally: + chan.close() + self.request.close() + self.logger.log(TRACE_LEVEL, "{0} connection closed.".format(self.info)) + + +class _ForwardServer(socketserver.TCPServer): # Not Threading + """ + Non-threading version of the forward server + """ + + allow_reuse_address = True # faster rebinding + + def __init__(self, *args, **kwargs): + logger = kwargs.pop("logger", None) + self.logger = logger or create_logger() + self.tunnel_ok = queue.Queue(1) + socketserver.TCPServer.__init__(self, *args, **kwargs) + + def handle_error(self, request, client_address): + (exc_class, exc, tb) = sys.exc_info() + local_side = request.getsockname() + remote_side = self.remote_address + self.logger.error( + "Could not establish connection from local {0} " + "to remote {1} side of the tunnel: {2}".format(local_side, remote_side, exc) + ) + try: + self.tunnel_ok.put(False, block=False, timeout=0.1) + except queue.Full: + # wait untill tunnel_ok.get is called + pass + except exc: + self.logger.error("unexpected internal error: {0}".format(exc)) + + @property + def local_address(self): + return self.server_address + + @property + def local_host(self): + return self.server_address[0] + + @property + def local_port(self): + return self.server_address[1] + + @property + def remote_address(self): + return self.RequestHandlerClass.remote_address + + @property + def remote_host(self): + return self.RequestHandlerClass.remote_address[0] + + @property + def remote_port(self): + return self.RequestHandlerClass.remote_address[1] + + +class _ThreadingForwardServer(socketserver.ThreadingMixIn, _ForwardServer): + """ + Allow concurrent connections to each tunnel + """ + + # If True, cleanly stop threads created by ThreadingMixIn when quitting + # This value is overrides by SSHTunnelForwarder.daemon_forward_servers + daemon_threads = _DAEMON + + +class _StreamForwardServer(_StreamServer): + """ + Serve over domain sockets (does not work on Windows) + """ + + def __init__(self, *args, **kwargs): + logger = kwargs.pop("logger", None) + self.logger = logger or create_logger() + self.tunnel_ok = queue.Queue(1) + _StreamServer.__init__(self, *args, **kwargs) + + @property + def local_address(self): + return self.server_address + + @property + def local_host(self): + return None + + @property + def local_port(self): + return None + + @property + def remote_address(self): + return self.RequestHandlerClass.remote_address + + @property + def remote_host(self): + return self.RequestHandlerClass.remote_address[0] + + @property + def remote_port(self): + return self.RequestHandlerClass.remote_address[1] + + +class _ThreadingStreamForwardServer(socketserver.ThreadingMixIn, _StreamForwardServer): + """ + Allow concurrent connections to each tunnel + """ + + # If True, cleanly stop threads created by ThreadingMixIn when quitting + # This value is overrides by SSHTunnelForwarder.daemon_forward_servers + daemon_threads = _DAEMON + + +class SSHTunnelForwarder(object): + """ + **SSH tunnel class** + + - Initialize a SSH tunnel to a remote host according to the input + arguments + + - Optionally: + + Read an SSH configuration file (typically ``~/.ssh/config``) + + Load keys from a running SSH agent (i.e. Pageant, GNOME Keyring) + + Raises: + + :class:`.BaseSSHTunnelForwarderError`: + raised by SSHTunnelForwarder class methods + + :class:`.HandlerSSHTunnelForwarderError`: + raised by tunnel forwarder threads + + .. note:: + Attributes ``mute_exceptions`` and + ``raise_exception_if_any_forwarder_have_a_problem`` + (deprecated) may be used to silence most exceptions raised + from this class + + Keyword Arguments: + + ssh_address_or_host (tuple or str): + IP or hostname of ``REMOTE GATEWAY``. It may be a two-element + tuple (``str``, ``int``) representing IP and port respectively, + or a ``str`` representing the IP address only + + .. versionadded:: 0.0.4 + + ssh_config_file (str): + SSH configuration file that will be read. If explicitly set to + ``None``, parsing of this configuration is omitted + + Default: :const:`SSH_CONFIG_FILE` + + .. versionadded:: 0.0.4 + + ssh_host_key (str): + Representation of a line in an OpenSSH-style "known hosts" + file. + + ``REMOTE GATEWAY``'s key fingerprint will be compared to this + host key in order to prevent against SSH server spoofing. + Important when using passwords in order not to accidentally + do a login attempt to a wrong (perhaps an attacker's) machine + + ssh_username (str): + Username to authenticate as in ``REMOTE SERVER`` + + Default: current local user name + + ssh_password (str): + Text representing the password used to connect to ``REMOTE + SERVER`` or for unlocking a private key. + + .. note:: + Avoid coding secret password directly in the code, since this + may be visible and make your service vulnerable to attacks + + ssh_port (int): + Optional port number of the SSH service on ``REMOTE GATEWAY``, + when `ssh_address_or_host`` is a ``str`` representing the + IP part of ``REMOTE GATEWAY``'s address + + Default: 22 + + ssh_pkey (str or paramiko.PKey): + **Private** key file name (``str``) to obtain the public key + from or a **public** key (:class:`paramiko.pkey.PKey`) + + ssh_private_key_password (str): + Password for an encrypted ``ssh_pkey`` + + .. note:: + Avoid coding secret password directly in the code, since this + may be visible and make your service vulnerable to attacks + + ssh_proxy (socket-like object or tuple): + Proxy where all SSH traffic will be passed through. + It might be for example a :class:`paramiko.proxy.ProxyCommand` + instance. + See either the :class:`paramiko.transport.Transport`'s sock + parameter documentation or ``ProxyCommand`` in ``ssh_config(5)`` + for more information. + + It is also possible to specify the proxy address as a tuple of + type (``str``, ``int``) representing proxy's IP and port + + .. note:: + Ignored if ``ssh_proxy_enabled`` is False + + .. versionadded:: 0.0.5 + + ssh_proxy_enabled (boolean): + Enable/disable SSH proxy. If True and user's + ``ssh_config_file`` contains a ``ProxyCommand`` directive + that matches the specified ``ssh_address_or_host``, + a :class:`paramiko.proxy.ProxyCommand` object will be created where + all SSH traffic will be passed through + + Default: ``True`` + + .. versionadded:: 0.0.4 + + local_bind_address (tuple): + Local tuple in the format (``str``, ``int``) representing the + IP and port of the local side of the tunnel. Both elements in + the tuple are optional so both ``('', 8000)`` and + ``('10.0.0.1', )`` are valid values + + Default: ``('0.0.0.0', RANDOM_PORT)`` + + .. versionchanged:: 0.0.8 + Added the ability to use a UNIX domain socket as local bind + address + + local_bind_addresses (list[tuple]): + In case more than one tunnel is established at once, a list + of tuples (in the same format as ``local_bind_address``) + can be specified, such as [(ip1, port_1), (ip_2, port2), ...] + + Default: ``[local_bind_address]`` + + .. versionadded:: 0.0.4 + + remote_bind_address (tuple): + Remote tuple in the format (``str``, ``int``) representing the + IP and port of the remote side of the tunnel. + + remote_bind_addresses (list[tuple]): + In case more than one tunnel is established at once, a list + of tuples (in the same format as ``remote_bind_address``) + can be specified, such as [(ip1, port_1), (ip_2, port2), ...] + + Default: ``[remote_bind_address]`` + + .. versionadded:: 0.0.4 + + allow_agent (boolean): + Enable/disable load of keys from an SSH agent + + Default: ``True`` + + .. versionadded:: 0.0.8 + + host_pkey_directories (list): + Look for pkeys in folders on this list, for example ['~/.ssh']. + + Default: ``None`` (disabled) + + .. versionadded:: 0.1.4 + + compression (boolean): + Turn on/off transport compression. By default compression is + disabled since it may negatively affect interactive sessions + + Default: ``False`` + + .. versionadded:: 0.0.8 + + logger (logging.Logger): + logging instance for sshtunnel and paramiko + + Default: :class:`logging.Logger` instance with a single + :class:`logging.StreamHandler` handler and + :const:`DEFAULT_LOGLEVEL` level + + .. versionadded:: 0.0.3 + + mute_exceptions (boolean): + Allow silencing :class:`BaseSSHTunnelForwarderError` or + :class:`HandlerSSHTunnelForwarderError` exceptions when enabled + + Default: ``False`` + + .. versionadded:: 0.0.8 + + set_keepalive (float): + Interval in seconds defining the period in which, if no data + was sent over the connection, a *'keepalive'* packet will be + sent (and ignored by the remote host). This can be useful to + keep connections alive over a NAT. You can set to 0.0 for + disable keepalive. + + Default: 5.0 (no keepalive packets are sent) + + .. versionadded:: 0.0.7 + + threaded (boolean): + Allow concurrent connections over a single tunnel + + Default: ``True`` + + .. versionadded:: 0.0.3 + + ssh_address (str): + Superseded by ``ssh_address_or_host``, tuple of type (str, int) + representing the IP and port of ``REMOTE SERVER`` + + .. deprecated:: 0.0.4 + + ssh_host (str): + Superseded by ``ssh_address_or_host``, tuple of type + (str, int) representing the IP and port of ``REMOTE SERVER`` + + .. deprecated:: 0.0.4 + + ssh_private_key (str or paramiko.PKey): + Superseded by ``ssh_pkey``, which can represent either a + **private** key file name (``str``) or a **public** key + (:class:`paramiko.pkey.PKey`) + + .. deprecated:: 0.0.8 + + raise_exception_if_any_forwarder_have_a_problem (boolean): + Allow silencing :class:`BaseSSHTunnelForwarderError` or + :class:`HandlerSSHTunnelForwarderError` exceptions when set to + False + + Default: ``True`` + + .. versionadded:: 0.0.4 + + .. deprecated:: 0.0.8 (use ``mute_exceptions`` instead) + + Attributes: + + tunnel_is_up (dict): + Describe whether or not the other side of the tunnel was reported + to be up (and we must close it) or not (skip shutting down that + tunnel) + + .. note:: + This attribute should not be modified + + .. note:: + When :attr:`.skip_tunnel_checkup` is disabled or the local bind + is a UNIX socket, the value will always be ``True`` + + **Example**:: + + {('127.0.0.1', 55550): True, # this tunnel is up + ('127.0.0.1', 55551): False} # this one isn't + + where 55550 and 55551 are the local bind ports + + skip_tunnel_checkup (boolean): + Disable tunnel checkup (default for backwards compatibility). + + .. versionadded:: 0.1.0 + + """ + + skip_tunnel_checkup = True + # This option affects the `ForwardServer` and all his threads + daemon_forward_servers = _DAEMON #: flag tunnel threads in daemon mode + # This option affect only `Transport` thread + daemon_transport = _DAEMON #: flag SSH transport thread in daemon mode + + def local_is_up(self, target): + """ + Check if a tunnel is up (remote target's host is reachable on TCP + target's port) + + Arguments: + target (tuple): + tuple of type (``str``, ``int``) indicating the listen IP + address and port + Return: + boolean + + .. deprecated:: 0.1.0 + Replaced by :meth:`.check_tunnels()` and :attr:`.tunnel_is_up` + """ + try: + check_address(target) + except ValueError: + self.logger.warning( + "Target must be a tuple (IP, port), where IP " + 'is a string (i.e. "192.168.0.1") and port is ' + "an integer (i.e. 40000). Alternatively " + "target can be a valid UNIX domain socket." + ) + return False + + self.check_tunnels() + return self.tunnel_is_up.get(target, True) + + def check_tunnels(self): + """ + Check that if all tunnels are established and populates + :attr:`.tunnel_is_up` + """ + skip_tunnel_checkup = self.skip_tunnel_checkup + try: + # force tunnel check at this point + self.skip_tunnel_checkup = False + for _srv in self._server_list: + self._check_tunnel(_srv) + finally: + self.skip_tunnel_checkup = skip_tunnel_checkup # roll it back + + def _check_tunnel(self, _srv): + """Check if tunnel is already established""" + if self.skip_tunnel_checkup: + self.tunnel_is_up[_srv.local_address] = True + return + self.logger.info("Checking tunnel to: {0}".format(_srv.remote_address)) + if isinstance(_srv.local_address, string_types): # UNIX stream + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + else: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(TUNNEL_TIMEOUT) + try: + # Windows raises WinError 10049 if trying to connect to 0.0.0.0 + connect_to = ( + ("127.0.0.1", _srv.local_port) + if _srv.local_host == "0.0.0.0" + else _srv.local_address + ) + s.connect(connect_to) + self.tunnel_is_up[_srv.local_address] = _srv.tunnel_ok.get( + timeout=TUNNEL_TIMEOUT * 1.1 + ) + self.logger.debug("Tunnel to {0} is DOWN".format(_srv.remote_address)) + except socket.error: + self.logger.debug("Tunnel to {0} is DOWN".format(_srv.remote_address)) + self.tunnel_is_up[_srv.local_address] = False + + except queue.Empty: + self.logger.debug("Tunnel to {0} is UP".format(_srv.remote_address)) + self.tunnel_is_up[_srv.local_address] = True + finally: + s.close() + + def _make_ssh_forward_handler_class(self, remote_address_): + """ + Make SSH Handler class + """ + + class Handler(_ForwardHandler): + remote_address = remote_address_ + ssh_transport = self._transport + logger = self.logger + + return Handler + + def _make_ssh_forward_server_class(self, remote_address_): + return _ThreadingForwardServer if self._threaded else _ForwardServer + + def _make_stream_ssh_forward_server_class(self, remote_address_): + return _ThreadingStreamForwardServer if self._threaded else _StreamForwardServer + + def _make_ssh_forward_server(self, remote_address, local_bind_address): + """ + Make SSH forward proxy Server class + """ + _Handler = self._make_ssh_forward_handler_class(remote_address) + try: + forward_maker_class = ( + self._make_stream_ssh_forward_server_class + if isinstance(local_bind_address, string_types) + else self._make_ssh_forward_server_class + ) + _Server = forward_maker_class(remote_address) + ssh_forward_server = _Server( + local_bind_address, + _Handler, + logger=self.logger, + ) + + if ssh_forward_server: + ssh_forward_server.daemon_threads = self.daemon_forward_servers + self._server_list.append(ssh_forward_server) + self.tunnel_is_up[ssh_forward_server.server_address] = False + else: + self._raise( + BaseSSHTunnelForwarderError, + "Problem setting up ssh {0} <> {1} forwarder. You can " + "suppress this exception by using the `mute_exceptions`" + "argument".format( + address_to_str(local_bind_address), + address_to_str(remote_address), + ), + ) + except IOError: + self._raise( + BaseSSHTunnelForwarderError, + "Couldn't open tunnel {0} <> {1} might be in use or " + "destination not reachable".format( + address_to_str(local_bind_address), address_to_str(remote_address) + ), + ) + + def __init__( + self, + ssh_address_or_host=None, + ssh_config_file=SSH_CONFIG_FILE, + ssh_host_key=None, + ssh_password=None, + ssh_pkey=None, + ssh_private_key_password=None, + ssh_proxy=None, + ssh_proxy_enabled=True, + ssh_username=None, + local_bind_address=None, + local_bind_addresses=None, + logger=None, + mute_exceptions=False, + remote_bind_address=None, + remote_bind_addresses=None, + set_keepalive=5.0, + threaded=True, # old version False + compression=None, + allow_agent=True, # look for keys from an SSH agent + host_pkey_directories=None, # look for keys in ~/.ssh + *args, + **kwargs, # for backwards compatibility + ): + self.logger = logger or create_logger() + + self.ssh_host_key = ssh_host_key + self.set_keepalive = set_keepalive + self._server_list = [] # reset server list + self.tunnel_is_up = {} # handle tunnel status + self._threaded = threaded + self.is_alive = False + # Check if deprecated arguments ssh_address or ssh_host were used + for deprecated_argument in ["ssh_address", "ssh_host"]: + ssh_address_or_host = self._process_deprecated( + ssh_address_or_host, deprecated_argument, kwargs + ) + # other deprecated arguments + ssh_pkey = self._process_deprecated(ssh_pkey, "ssh_private_key", kwargs) + + self._raise_fwd_exc = ( + self._process_deprecated( + None, "raise_exception_if_any_forwarder_have_a_problem", kwargs + ) + or not mute_exceptions + ) + + if isinstance(ssh_address_or_host, tuple): + check_address(ssh_address_or_host) + (ssh_host, ssh_port) = ssh_address_or_host + else: + ssh_host = ssh_address_or_host + ssh_port = kwargs.pop("ssh_port", None) + + if kwargs: + raise ValueError("Unknown arguments: {0}".format(kwargs)) + + # remote binds + self._remote_binds = self._get_binds( + remote_bind_address, remote_bind_addresses, is_remote=True + ) + # local binds + self._local_binds = self._get_binds(local_bind_address, local_bind_addresses) + self._local_binds = self._consolidate_binds( + self._local_binds, self._remote_binds + ) + + ( + self.ssh_host, + self.ssh_username, + ssh_pkey, # still needs to go through _consolidate_auth + self.ssh_port, + self.ssh_proxy, + self.compression, + ) = self._read_ssh_config( + ssh_host, + ssh_config_file, + ssh_username, + ssh_pkey, + ssh_port, + ssh_proxy if ssh_proxy_enabled else None, + compression, + self.logger, + ) + + (self.ssh_password, self.ssh_pkeys) = self._consolidate_auth( + ssh_password=ssh_password, + ssh_pkey=ssh_pkey, + ssh_pkey_password=ssh_private_key_password, + allow_agent=allow_agent, + host_pkey_directories=host_pkey_directories, + logger=self.logger, + ) + + check_host(self.ssh_host) + check_port(self.ssh_port) + + self.logger.info( + "Connecting to gateway: {0}:{1} as user '{2}'".format( + self.ssh_host, self.ssh_port, self.ssh_username + ) + ) + + self.logger.debug("Concurrent connections allowed: {0}".format(self._threaded)) + + @staticmethod + def _read_ssh_config( + ssh_host, + ssh_config_file, + ssh_username=None, + ssh_pkey=None, + ssh_port=None, + ssh_proxy=None, + compression=None, + logger=None, + ): + """ + Read ssh_config_file and tries to look for user (ssh_username), + identityfile (ssh_pkey), port (ssh_port) and proxycommand + (ssh_proxy) entries for ssh_host + """ + ssh_config = paramiko.SSHConfig() + if not ssh_config_file: # handle case where it's an empty string + ssh_config_file = None + + # Try to read SSH_CONFIG_FILE + try: + # open the ssh config file + with open(os.path.expanduser(ssh_config_file), "r") as f: + ssh_config.parse(f) + # looks for information for the destination system + hostname_info = ssh_config.lookup(ssh_host) + # gather settings for user, port and identity file + # last resort: use the 'login name' of the user + ssh_username = ssh_username or hostname_info.get("user") + ssh_pkey = ssh_pkey or hostname_info.get("identityfile", [None])[0] + ssh_host = hostname_info.get("hostname") + ssh_port = ssh_port or hostname_info.get("port") + + proxycommand = hostname_info.get("proxycommand") + ssh_proxy = ssh_proxy or ( + paramiko.ProxyCommand(proxycommand) if proxycommand else None + ) + if compression is None: + compression = hostname_info.get("compression", "") + compression = True if compression.upper() == "YES" else False + except IOError: + if logger: + logger.warning( + "Could not read SSH configuration file: {0}".format(ssh_config_file) + ) + except (AttributeError, TypeError): # ssh_config_file is None + if logger: + logger.info("Skipping loading of ssh configuration file") + + return ( + ssh_host, + ssh_username or getpass.getuser(), + ssh_pkey, + int(ssh_port) if ssh_port else 22, # fallback value + ssh_proxy, + compression, + ) + + @staticmethod + def get_agent_keys(logger=None): + """Load public keys from any available SSH agent + + Arguments: + logger (Optional[logging.Logger]) + + Return: + list + """ + paramiko_agent = paramiko.Agent() + agent_keys = paramiko_agent.get_keys() + if logger: + logger.info("{0} keys loaded from agent".format(len(agent_keys))) + return list(agent_keys) + + @staticmethod + def get_keys(logger=None, host_pkey_directories=None, allow_agent=False): + """ + Load public keys from any available SSH agent or local + .ssh directory. + + Arguments: + logger (Optional[logging.Logger]) + + host_pkey_directories (Optional[list[str]]): + List of local directories where host SSH pkeys in the format + "id_*" are searched. For example, ['~/.ssh'] + + .. versionadded:: 0.1.0 + + allow_agent (Optional[boolean]): + Whether or not load keys from agent + + Default: False + + Return: + list + """ + keys = SSHTunnelForwarder.get_agent_keys(logger=logger) if allow_agent else [] + + if host_pkey_directories is None: + host_pkey_directories = [DEFAULT_SSH_DIRECTORY] + + paramiko_key_types = { + "rsa": paramiko.RSAKey, + "dsa": paramiko.DSSKey, + "ecdsa": paramiko.ECDSAKey, + } + if hasattr(paramiko, "Ed25519Key"): + # NOQA: new in paramiko>=2.2: http://docs.paramiko.org/en/stable/api/keys.html#module-paramiko.ed25519key + paramiko_key_types["ed25519"] = paramiko.Ed25519Key + for directory in host_pkey_directories: + for keytype in paramiko_key_types.keys(): + ssh_pkey_expanded = os.path.expanduser( + os.path.join(directory, "id_{}".format(keytype)) + ) + try: + if os.path.isfile(ssh_pkey_expanded): + ssh_pkey = SSHTunnelForwarder.read_private_key_file( + pkey_file=ssh_pkey_expanded, + logger=logger, + key_type=paramiko_key_types[keytype], + ) + if ssh_pkey: + keys.append(ssh_pkey) + except OSError as exc: + if logger: + logger.warning( + "Private key file {0} check error: {1}".format( + ssh_pkey_expanded, exc + ) + ) + if logger: + logger.info("{0} key(s) loaded".format(len(keys))) + return keys + + @staticmethod + def _consolidate_binds(local_binds, remote_binds): + """ + Fill local_binds with defaults when no value/s were specified, + leaving paramiko to decide in which local port the tunnel will be open + """ + count = len(remote_binds) - len(local_binds) + if count < 0: + raise ValueError( + "Too many local bind addresses " + "(local_bind_addresses > remote_bind_addresses)" + ) + local_binds.extend([("0.0.0.0", 0) for x in range(count)]) + return local_binds + + @staticmethod + def _consolidate_auth( + ssh_password=None, + ssh_pkey=None, + ssh_pkey_password=None, + allow_agent=True, + host_pkey_directories=None, + logger=None, + ): + """ + Get sure authentication information is in place. + ``ssh_pkey`` may be of classes: + - ``str`` - in this case it represents a private key file; public + key will be obtained from it + - ``paramiko.Pkey`` - it will be transparently added to loaded keys + + """ + ssh_loaded_pkeys = SSHTunnelForwarder.get_keys( + logger=logger, + host_pkey_directories=host_pkey_directories, + allow_agent=allow_agent, + ) + + if isinstance(ssh_pkey, string_types): + ssh_pkey_expanded = os.path.expanduser(ssh_pkey) + if os.path.exists(ssh_pkey_expanded): + ssh_pkey = SSHTunnelForwarder.read_private_key_file( + pkey_file=ssh_pkey_expanded, + pkey_password=ssh_pkey_password or ssh_password, + logger=logger, + ) + elif logger: + logger.warning("Private key file not found: {0}".format(ssh_pkey)) + if isinstance(ssh_pkey, paramiko.pkey.PKey): + ssh_loaded_pkeys.insert(0, ssh_pkey) + + if not ssh_password and not ssh_loaded_pkeys: + raise ValueError("No password or public key available!") + return (ssh_password, ssh_loaded_pkeys) + + def _raise(self, exception=BaseSSHTunnelForwarderError, reason=None): + if self._raise_fwd_exc: + raise exception(reason) + else: + self.logger.error(repr(exception(reason))) + + def _get_transport(self): + """Return the SSH transport to the remote gateway""" + if self.ssh_proxy: + if isinstance(self.ssh_proxy, paramiko.proxy.ProxyCommand): + proxy_repr = repr(self.ssh_proxy.cmd[1]) + else: + proxy_repr = repr(self.ssh_proxy) + self.logger.debug("Connecting via proxy: {0}".format(proxy_repr)) + _socket = self.ssh_proxy + else: + _socket = (self.ssh_host, self.ssh_port) + if isinstance(_socket, socket.socket): + _socket.settimeout(SSH_TIMEOUT) + _socket.connect((self.ssh_host, self.ssh_port)) + transport = paramiko.Transport(_socket) + sock = transport.sock + if isinstance(sock, socket.socket): + sock.settimeout(SSH_TIMEOUT) + transport.set_keepalive(self.set_keepalive) + transport.use_compression(compress=self.compression) + transport.daemon = self.daemon_transport + # try to solve https://github.com/paramiko/paramiko/issues/1181 + # transport.banner_timeout = 200 + if isinstance(sock, socket.socket): + sock_timeout = sock.gettimeout() + sock_info = repr((sock.family, sock.type, sock.proto)) + self.logger.debug( + "Transport socket info: {0}, timeout={1}".format( + sock_info, sock_timeout + ) + ) + return transport + + def _create_tunnels(self): + """ + Create SSH tunnels on top of a transport to the remote gateway + """ + if not self.is_active: + try: + self._connect_to_gateway() + except socket.gaierror as e: # raised by paramiko.Transport + self.logger.error( + msg=f"Could not resolve IP address for {self.ssh_host}, aborting!" + ) + raise e + except (paramiko.SSHException, socket.error) as e: + self.logger.error( + msg=( + "Could not connect to gateway " + f"{self.ssh_host}:{self.ssh_port} : {e.args[0]}" + ) + ) + raise e + + for rem, loc in zip(self._remote_binds, self._local_binds): + try: + self._make_ssh_forward_server(rem, loc) + except BaseSSHTunnelForwarderError as e: + self.logger.error(f"Problem setting SSH Forwarder up: {e.value}") + raise e + + @staticmethod + def _get_binds(bind_address, bind_addresses, is_remote=False): + addr_kind = "remote" if is_remote else "local" + + if not bind_address and not bind_addresses: + if is_remote: + raise ValueError( + "No {0} bind addresses specified. Use " + "'{0}_bind_address' or '{0}_bind_addresses'" + " argument".format(addr_kind) + ) + else: + return [] + elif bind_address and bind_addresses: + raise ValueError( + "You can't use both '{0}_bind_address' and " + "'{0}_bind_addresses' arguments. Use one of " + "them.".format(addr_kind) + ) + + if bind_address: + bind_addresses = [bind_address] + + if not is_remote: + # Add random port if missing in local bind + for i, local_bind in enumerate(bind_addresses): + if isinstance(local_bind, tuple) and len(local_bind) == 1: + bind_addresses[i] = (local_bind[0], 0) + + check_addresses(bind_addresses, is_remote) + + return bind_addresses + + @staticmethod + def _process_deprecated(attrib, deprecated_attrib, kwargs): + """ + Processes optional deprecate arguments + """ + if deprecated_attrib not in _DEPRECATIONS: + raise ValueError( + "{0} not included in deprecations list".format(deprecated_attrib) + ) + if deprecated_attrib in kwargs: + warnings.warn( + message="'{0}' is DEPRECATED use '{1}' instead".format( + deprecated_attrib, _DEPRECATIONS[deprecated_attrib] + ), + category=DeprecationWarning, + stacklevel=2, + ) + if attrib: + raise ValueError( + "You can't use both '{0}' and '{1}'. " + "Please only use one of them".format( + deprecated_attrib, _DEPRECATIONS[deprecated_attrib] + ) + ) + else: + return kwargs.pop(deprecated_attrib) + return attrib + + @staticmethod + def read_private_key_file( + pkey_file, pkey_password=None, key_type=None, logger=None + ): + """ + Get SSH Public key from a private key file, given an optional password + + Arguments: + pkey_file (str): + File containing a private key (RSA, DSS or ECDSA) + Keyword Arguments: + pkey_password (Optional[str]): + Password to decrypt the private key + logger (Optional[logging.Logger]) + Return: + paramiko.Pkey + """ + ssh_pkey = None + key_types = (paramiko.RSAKey, paramiko.DSSKey, paramiko.ECDSAKey) + if hasattr(paramiko, "Ed25519Key"): + # NOQA: new in paramiko>=2.2: http://docs.paramiko.org/en/stable/api/keys.html#module-paramiko.ed25519key + key_types += (paramiko.Ed25519Key,) + for pkey_class in (key_type,) if key_type else key_types: + try: + ssh_pkey = pkey_class.from_private_key_file( + pkey_file, password=pkey_password + ) + if logger: + logger.debug( + "Private key file ({0}, {1}) successfully " "loaded".format( + pkey_file, pkey_class + ) + ) + break + except paramiko.PasswordRequiredException: + if logger: + logger.error("Password is required for key {0}".format(pkey_file)) + break + except paramiko.SSHException: + if logger: + logger.debug( + "Private key file ({0}) could not be loaded " + "as type {1} or bad password".format(pkey_file, pkey_class) + ) + return ssh_pkey + + def start(self): + """Start the SSH tunnels""" + if self.is_alive: + self.logger.warning("Already started!") + return + + self._create_tunnels() + + if not self.is_active: + self._raise( + BaseSSHTunnelForwarderError, + reason="Could not establish session to SSH gateway", + ) + + for _srv in self._server_list: + thread = threading.Thread( + target=self._serve_forever_wrapper, + args=(_srv,), + name="Srv-{0}".format(address_to_str(_srv.local_port)), + ) + + thread.daemon = self.daemon_forward_servers + thread.start() + + self._check_tunnel(_srv) + + self.is_alive = any(self.tunnel_is_up.values()) + + if not self.is_alive: + self._raise( + HandlerSSHTunnelForwarderError, + "An error occurred while opening tunnels.", + ) + + def stop(self, force=False): + """ + Shut the tunnel down. By default we are always waiting until closing + all connections. You can use `force=True` to force close connections + + Keyword Arguments: + force (bool): + Force close current connections + + Default: False + + .. versionadded:: 0.2.2 + + .. note:: This **had** to be handled with care before ``0.1.0``: + + - if a port redirection is opened + - the destination is not reachable + - we attempt a connection to that tunnel (``SYN`` is sent and + acknowledged, then a ``FIN`` packet is sent and never + acknowledged... weird) + - we try to shutdown: it will not succeed until ``FIN_WAIT_2`` and + ``CLOSE_WAIT`` time out. + + .. note:: + Handle these scenarios with :attr:`.tunnel_is_up`: if False, server + ``shutdown()`` will be skipped on that tunnel + """ + self.logger.info("Closing all open connections...") + opened_address_text = ( + ", ".join((address_to_str(k.local_address) for k in self._server_list)) + or "None" + ) + + self.logger.debug("Listening tunnels: " + opened_address_text) + self._stop_transport(force=force) + self._server_list = [] # reset server list + self.tunnel_is_up = {} # reset tunnel status + + def close(self): + """Stop the an active tunnel, alias to :meth:`.stop`""" + self.stop() + + def restart(self): + """Restart connection to the gateway and tunnels""" + self.stop() + self.start() + + def _connect_to_gateway(self): + """ + Open connection to SSH gateway + - First try with all keys loaded from an SSH agent (if allowed) + - Then with those passed directly or read from ~/.ssh/config + - As last resort, try with a provided password + """ + for key in self.ssh_pkeys: + self.logger.debug( + "Trying to log in with key: {0}".format(hexlify(key.get_fingerprint())) + ) + try: + self._transport = self._get_transport() + self._transport.connect( + hostkey=self.ssh_host_key, username=self.ssh_username, pkey=key + ) + if self._transport.is_alive: + return + except paramiko.AuthenticationException: + self.logger.debug("Authentication error") + self._stop_transport() + + if self.ssh_password: # avoid conflict using both pass and pkey + self.logger.debug( + "Trying to log in with password: {0}".format( + "*" * len(self.ssh_password) + ) + ) + try: + self._transport = self._get_transport() + self._transport.connect( + hostkey=self.ssh_host_key, + username=self.ssh_username, + password=self.ssh_password, + ) + if self._transport.is_alive: + return + except paramiko.AuthenticationException: + self.logger.debug("Authentication error") + self._stop_transport() + + self.logger.error("Could not open connection to gateway") + + def _serve_forever_wrapper(self, _srv, poll_interval=0.1): + """ + Wrapper for the server created for a SSH forward + """ + self.logger.info( + "Opening tunnel: {0} <> {1}".format( + address_to_str(_srv.local_address), address_to_str(_srv.remote_address) + ) + ) + _srv.serve_forever(poll_interval) # blocks until finished + + self.logger.info( + "Tunnel: {0} <> {1} released".format( + address_to_str(_srv.local_address), address_to_str(_srv.remote_address) + ) + ) + + def _stop_transport(self, force=False): + """Close the underlying transport when nothing more is needed""" + try: + self._check_is_started() + except (BaseSSHTunnelForwarderError, HandlerSSHTunnelForwarderError) as e: + self.logger.warning(e) + + if force and self.is_active: + # don't wait connections + self.logger.info("Closing ssh transport") + self._transport.close() + self._transport.stop_thread() + + for _srv in self._server_list: + status = "up" if self.tunnel_is_up[_srv.local_address] else "down" + self.logger.info( + "Shutting down tunnel: {0} <> {1} ({2})".format( + address_to_str(_srv.local_address), + address_to_str(_srv.remote_address), + status, + ) + ) + _srv.shutdown() + _srv.server_close() + # clean up the UNIX domain socket if we're using one + if isinstance(_srv, _StreamForwardServer): + try: + os.unlink(_srv.local_address) + except Exception as e: + self.logger.error( + "Unable to unlink socket {0}: {1}".format( + _srv.local_address, repr(e) + ) + ) + self.is_alive = False + if self.is_active: + self.logger.info("Closing ssh transport") + self._transport.close() + self._transport.stop_thread() + self.logger.debug("Transport is closed") + + @property + def local_bind_port(self): + # BACKWARDS COMPATIBILITY + self._check_is_started() + if len(self._server_list) != 1: + raise BaseSSHTunnelForwarderError( + "Use .local_bind_ports property for more than one tunnel" + ) + return self.local_bind_ports[0] + + @property + def local_bind_host(self): + # BACKWARDS COMPATIBILITY + self._check_is_started() + if len(self._server_list) != 1: + raise BaseSSHTunnelForwarderError( + "Use .local_bind_hosts property for more than one tunnel" + ) + return self.local_bind_hosts[0] + + @property + def local_bind_address(self): + # BACKWARDS COMPATIBILITY + self._check_is_started() + if len(self._server_list) != 1: + raise BaseSSHTunnelForwarderError( + "Use .local_bind_addresses property for more than one tunnel" + ) + return self.local_bind_addresses[0] + + @property + def local_bind_ports(self): + """ + Return a list containing the ports of local side of the TCP tunnels + """ + self._check_is_started() + return [ + _server.local_port + for _server in self._server_list + if _server.local_port is not None + ] + + @property + def local_bind_hosts(self): + """ + Return a list containing the IP addresses listening for the tunnels + """ + self._check_is_started() + return [ + _server.local_host + for _server in self._server_list + if _server.local_host is not None + ] + + @property + def local_bind_addresses(self): + """ + Return a list of (IP, port) pairs for the local side of the tunnels + """ + self._check_is_started() + return [_server.local_address for _server in self._server_list] + + @property + def tunnel_bindings(self): + """ + Return a dictionary containing the active local<>remote tunnel_bindings + """ + return dict( + (_server.remote_address, _server.local_address) + for _server in self._server_list + if self.tunnel_is_up[_server.local_address] + ) + + @property + def is_active(self): + """Return True if the underlying SSH transport is up""" + if "_transport" in self.__dict__ and self._transport.is_active(): + return True + return False + + def _check_is_started(self): + if not self.is_active: # underlying transport not alive + msg = "Server is not started. Please .start() first!" + raise BaseSSHTunnelForwarderError(msg) + if not self.is_alive: + msg = "Tunnels are not started. Please .start() first!" + raise HandlerSSHTunnelForwarderError(msg) + + def __str__(self): + credentials = { + "password": self.ssh_password, + "pkeys": [ + (key.get_name(), hexlify(key.get_fingerprint())) + for key in self.ssh_pkeys + ] + if any(self.ssh_pkeys) + else None, + } + _remove_none_values(credentials) + template = os.linesep.join( + [ + "{0} object", + "ssh gateway: {1}:{2}", + "proxy: {3}", + "username: {4}", + "authentication: {5}", + "hostkey: {6}", + "status: {7}started", + "keepalive messages: {8}", + "tunnel connection check: {9}", + "concurrent connections: {10}allowed", + "compression: {11}requested", + "logging level: {12}", + "local binds: {13}", + "remote binds: {14}", + ] + ) + return template.format( + self.__class__, + self.ssh_host, + self.ssh_port, + self.ssh_proxy.cmd[1] if self.ssh_proxy else "no", + self.ssh_username, + credentials, + self.ssh_host_key if self.ssh_host_key else "not checked", + "" if self.is_alive else "not ", + "disabled" + if not self.set_keepalive + else "every {0} sec".format(self.set_keepalive), + "disabled" if self.skip_tunnel_checkup else "enabled", + "" if self._threaded else "not ", + "" if self.compression else "not ", + logging.getLevelName(self.logger.level), + self._local_binds, + self._remote_binds, + ) + + def __repr__(self): + return self.__str__() + + def __enter__(self): + try: + self.start() + return self + except KeyboardInterrupt: + self.__exit__() + + def __exit__(self, *args): + self.stop(force=True) + + def __del__(self): + if self.is_active or self.is_alive: + self.logger.warning( + "It looks like you didn't call the .stop() before " + "the SSHTunnelForwarder obj was collected by " + "the garbage collector! Running .stop(force=True)" + ) + self.stop(force=True) From 0231c9fb57c7c42f53b640800c4267c8ac551977 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 18 Sep 2024 20:25:57 +0000 Subject: [PATCH 2/4] style --- src/teamster/libraries/ssh/sshtunnel.py | 213 ++++++++++++------------ 1 file changed, 107 insertions(+), 106 deletions(-) diff --git a/src/teamster/libraries/ssh/sshtunnel.py b/src/teamster/libraries/ssh/sshtunnel.py index 015b621892..093bd5803e 100644 --- a/src/teamster/libraries/ssh/sshtunnel.py +++ b/src/teamster/libraries/ssh/sshtunnel.py @@ -736,6 +736,113 @@ class SSHTunnelForwarder(object): # This option affect only `Transport` thread daemon_transport = _DAEMON #: flag SSH transport thread in daemon mode + def __init__( + self, + ssh_address_or_host=None, + ssh_config_file=SSH_CONFIG_FILE, + ssh_host_key=None, + ssh_password=None, + ssh_pkey=None, + ssh_private_key_password=None, + ssh_proxy=None, + ssh_proxy_enabled=True, + ssh_username=None, + local_bind_address=None, + local_bind_addresses=None, + logger=None, + mute_exceptions=False, + remote_bind_address=None, + remote_bind_addresses=None, + set_keepalive=5.0, + threaded=True, # old version False + compression=None, + allow_agent=True, # look for keys from an SSH agent + host_pkey_directories=None, # look for keys in ~/.ssh + *args, + **kwargs, # for backwards compatibility + ): + self.logger = logger or create_logger() + + self.ssh_host_key = ssh_host_key + self.set_keepalive = set_keepalive + self._server_list = [] # reset server list + self.tunnel_is_up = {} # handle tunnel status + self._threaded = threaded + self.is_alive = False + + # Check if deprecated arguments ssh_address or ssh_host were used + for deprecated_argument in ["ssh_address", "ssh_host"]: + ssh_address_or_host = self._process_deprecated( + ssh_address_or_host, deprecated_argument, kwargs + ) + # other deprecated arguments + ssh_pkey = self._process_deprecated(ssh_pkey, "ssh_private_key", kwargs) + + self._raise_fwd_exc = ( + self._process_deprecated( + None, "raise_exception_if_any_forwarder_have_a_problem", kwargs + ) + or not mute_exceptions + ) + + if isinstance(ssh_address_or_host, tuple): + check_address(ssh_address_or_host) + (ssh_host, ssh_port) = ssh_address_or_host + else: + ssh_host = ssh_address_or_host + ssh_port = kwargs.pop("ssh_port", None) + + if kwargs: + raise ValueError("Unknown arguments: {0}".format(kwargs)) + + # remote binds + self._remote_binds = self._get_binds( + remote_bind_address, remote_bind_addresses, is_remote=True + ) + # local binds + self._local_binds = self._get_binds(local_bind_address, local_bind_addresses) + self._local_binds = self._consolidate_binds( + self._local_binds, self._remote_binds + ) + + ( + self.ssh_host, + self.ssh_username, + ssh_pkey, # still needs to go through _consolidate_auth + self.ssh_port, + self.ssh_proxy, + self.compression, + ) = self._read_ssh_config( + ssh_host, + ssh_config_file, + ssh_username, + ssh_pkey, + ssh_port, + ssh_proxy if ssh_proxy_enabled else None, + compression, + self.logger, + ) + + (self.ssh_password, self.ssh_pkeys) = self._consolidate_auth( + ssh_password=ssh_password, + ssh_pkey=ssh_pkey, + ssh_pkey_password=ssh_private_key_password, + allow_agent=allow_agent, + host_pkey_directories=host_pkey_directories, + logger=self.logger, + ) + + check_host(self.ssh_host) + check_port(self.ssh_port) + + self.logger.info( + "Connecting to gateway: {0}:{1} as user '{2}'".format( + self.ssh_host, self.ssh_port, self.ssh_username + ) + ) + + self.logger.debug("Concurrent connections allowed: {0}".format(self._threaded)) + def local_is_up(self, target): """ Check if a tunnel is up (remote target's host is reachable on TCP @@ -871,112 +978,6 @@ def _make_ssh_forward_server(self, remote_address, local_bind_address): ), ) - def __init__( - self, - ssh_address_or_host=None, - ssh_config_file=SSH_CONFIG_FILE, - ssh_host_key=None, - ssh_password=None, - ssh_pkey=None, - ssh_private_key_password=None, - ssh_proxy=None, - ssh_proxy_enabled=True, - ssh_username=None, - local_bind_address=None, - local_bind_addresses=None, - logger=None, - mute_exceptions=False, - remote_bind_address=None, - remote_bind_addresses=None, - set_keepalive=5.0, - threaded=True, # old version False - compression=None, - allow_agent=True, # look for keys from an SSH agent - host_pkey_directories=None, # look for keys in ~/.ssh - *args, - **kwargs, # for backwards compatibility - ): - self.logger = logger or create_logger() - - self.ssh_host_key = ssh_host_key - self.set_keepalive = set_keepalive - self._server_list = [] # reset server list - self.tunnel_is_up = {} # handle tunnel status - self._threaded = threaded - self.is_alive = False - # Check if deprecated arguments ssh_address or ssh_host were used - for deprecated_argument in ["ssh_address", "ssh_host"]: - ssh_address_or_host = self._process_deprecated( - ssh_address_or_host, deprecated_argument, kwargs - ) - # other deprecated arguments - ssh_pkey = self._process_deprecated(ssh_pkey, "ssh_private_key", kwargs) - - self._raise_fwd_exc = ( - self._process_deprecated( - None, "raise_exception_if_any_forwarder_have_a_problem", kwargs - ) - or not mute_exceptions - ) - - if isinstance(ssh_address_or_host, tuple): - check_address(ssh_address_or_host) - (ssh_host, ssh_port) = ssh_address_or_host - else: - ssh_host = ssh_address_or_host - ssh_port = kwargs.pop("ssh_port", None) - - if kwargs: - raise ValueError("Unknown arguments: {0}".format(kwargs)) - - # remote binds - self._remote_binds = self._get_binds( - remote_bind_address, remote_bind_addresses, is_remote=True - ) - # local binds - self._local_binds = self._get_binds(local_bind_address, local_bind_addresses) - self._local_binds = self._consolidate_binds( - self._local_binds, self._remote_binds - ) - - ( - self.ssh_host, - self.ssh_username, - ssh_pkey, # still needs to go through _consolidate_auth - self.ssh_port, - self.ssh_proxy, - self.compression, - ) = self._read_ssh_config( - ssh_host, - ssh_config_file, - ssh_username, - ssh_pkey, - ssh_port, - ssh_proxy if ssh_proxy_enabled else None, - compression, - self.logger, - ) - - (self.ssh_password, self.ssh_pkeys) = self._consolidate_auth( - ssh_password=ssh_password, - ssh_pkey=ssh_pkey, - ssh_pkey_password=ssh_private_key_password, - allow_agent=allow_agent, - host_pkey_directories=host_pkey_directories, - logger=self.logger, - ) - - check_host(self.ssh_host) - check_port(self.ssh_port) - - self.logger.info( - "Connecting to gateway: {0}:{1} as user '{2}'".format( - self.ssh_host, self.ssh_port, self.ssh_username - ) - ) - - self.logger.debug("Concurrent connections allowed: {0}".format(self._threaded)) - @staticmethod def _read_ssh_config( ssh_host, From 8c43775a63552362d6cd3c8c332d4d5bd0ebdeaa Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 18 Sep 2024 22:49:35 +0000 Subject: [PATCH 3/4] build: use sshtunnel from source --- pdm.lock | 178 ++--- pyproject.toml | 3 +- requirements.txt | 947 ++++-------------------- src/teamster/libraries/ssh/sshtunnel.py | 393 +++++----- 4 files changed, 465 insertions(+), 1056 deletions(-) diff --git a/pdm.lock b/pdm.lock index bc08aab85e..5571508303 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:6e9918eacbdc2a671a475e04067e90a39719f3dea733072e14d48a3f6417fc20" +content_hash = "sha256:aa47064038fa484701b55e9c2e38f38f06c6db3d24edf4b4ff85612482e64701" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -736,7 +736,7 @@ files = [ [[package]] name = "dbt-adapters" -version = "1.6.0" +version = "1.6.1" requires_python = ">=3.8.0" summary = "The set of adapter protocols and base functionality that supports integration with dbt-core" groups = ["default"] @@ -749,8 +749,8 @@ dependencies = [ "typing-extensions<5.0,>=4.0", ] files = [ - {file = "dbt_adapters-1.6.0-py3-none-any.whl", hash = "sha256:fdda113eaae4d4e31c42257c4d68e94d1bd3519476b3326c4d859d3f33278cfc"}, - {file = "dbt_adapters-1.6.0.tar.gz", hash = "sha256:ac3b73b28691d95a9b56b70dae5a0e2e989bf539d601984cfcd3e868462cf037"}, + {file = "dbt_adapters-1.6.1-py3-none-any.whl", hash = "sha256:6a09da3c6d11ef1b9ec4c44bbd8258294e0e0f9a2e0017adbe7f0234b23aa7ee"}, + {file = "dbt_adapters-1.6.1.tar.gz", hash = "sha256:6953a4133c2701eb989c3ffe950b0500f328e86283ec0658452ab237ebdb9b35"}, ] [[package]] @@ -960,13 +960,13 @@ files = [ [[package]] name = "filelock" -version = "3.16.0" +version = "3.16.1" requires_python = ">=3.8" summary = "A platform independent file lock." groups = ["default", "dev"] files = [ - {file = "filelock-3.16.0-py3-none-any.whl", hash = "sha256:f6ed4c963184f4c84dd5557ce8fece759a3724b37b80c6c4f20a2f63a4dc6609"}, - {file = "filelock-3.16.0.tar.gz", hash = "sha256:81de9eb8453c769b63369f87f11131a7ab04e367f8d97ad39dc230daa07e3bec"}, + {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, + {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, ] [[package]] @@ -1047,7 +1047,7 @@ files = [ [[package]] name = "google-api-python-client" -version = "2.145.0" +version = "2.146.0" requires_python = ">=3.7" summary = "Google API Client Library for Python" groups = ["default"] @@ -1059,8 +1059,8 @@ dependencies = [ "uritemplate<5,>=3.0.1", ] files = [ - {file = "google_api_python_client-2.145.0-py2.py3-none-any.whl", hash = "sha256:d74da1358f3f2d63daf3c6f26bd96d89652051183bc87cf10a56ceb2a70beb50"}, - {file = "google_api_python_client-2.145.0.tar.gz", hash = "sha256:8b84dde11aaccadc127e4846f5cd932331d804ea324e353131595e3f25376e97"}, + {file = "google_api_python_client-2.146.0-py2.py3-none-any.whl", hash = "sha256:b1e62c9889c5ef6022f11d30d7ef23dc55100300f0e8aaf8aa09e8e92540acad"}, + {file = "google_api_python_client-2.146.0.tar.gz", hash = "sha256:41f671be10fa077ee5143ee9f0903c14006d39dc644564f4e044ae96b380bf68"}, ] [[package]] @@ -1165,7 +1165,7 @@ files = [ [[package]] name = "google-cloud-dataproc" -version = "5.11.0" +version = "5.12.0" requires_python = ">=3.7" summary = "Google Cloud Dataproc API client library" groups = ["default"] @@ -1177,8 +1177,8 @@ dependencies = [ "protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.2", ] files = [ - {file = "google_cloud_dataproc-5.11.0-py2.py3-none-any.whl", hash = "sha256:771bcdeb46434177383433a8f092e52674482596eaa7aa5770d1b1b136727fe5"}, - {file = "google_cloud_dataproc-5.11.0.tar.gz", hash = "sha256:ea38803987fd7818d133243db6f4771167e7aa4d4c7d82144e6f7c3e59df9218"}, + {file = "google_cloud_dataproc-5.12.0-py2.py3-none-any.whl", hash = "sha256:0a64a9202e2f3781e90e0e281b148df55da6039ffee540581acd06542d7b1a19"}, + {file = "google_cloud_dataproc-5.12.0.tar.gz", hash = "sha256:4b58d720f020c4e35378a29ea87ce8286af631eb3bc382ccbcbff39dc0199e29"}, ] [[package]] @@ -1503,13 +1503,13 @@ files = [ [[package]] name = "idna" -version = "3.8" +version = "3.10" requires_python = ">=3.6" summary = "Internationalized Domain Names in Applications (IDNA)" groups = ["default", "dev"] files = [ - {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, - {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, ] [[package]] @@ -2017,7 +2017,7 @@ files = [ [[package]] name = "paramiko" -version = "3.4.1" +version = "3.5.0" requires_python = ">=3.6" summary = "SSH2 protocol library" groups = ["default"] @@ -2027,8 +2027,8 @@ dependencies = [ "pynacl>=1.5", ] files = [ - {file = "paramiko-3.4.1-py3-none-any.whl", hash = "sha256:8e49fd2f82f84acf7ffd57c64311aa2b30e575370dc23bdb375b10262f7eac32"}, - {file = "paramiko-3.4.1.tar.gz", hash = "sha256:8b15302870af7f6652f2e038975c1d2973f06046cb5d7d65355668b3ecbece0c"}, + {file = "paramiko-3.5.0-py3-none-any.whl", hash = "sha256:1fedf06b085359051cd7d0d270cebe19e755a8a921cc2ddbfa647fb0cd7d68f9"}, + {file = "paramiko-3.5.0.tar.gz", hash = "sha256:ad11e540da4f55cedda52931f1a3f812a8238a7af7f62a60de538cd80bb28124"}, ] [[package]] @@ -2081,24 +2081,24 @@ files = [ [[package]] name = "pex" -version = "2.18.0" +version = "2.19.1" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<3.14,>=2.7" summary = "The PEX packaging toolchain." groups = ["default"] files = [ - {file = "pex-2.18.0-py2.py3-none-any.whl", hash = "sha256:159045724c4b8c67015daad5907ca51e01058e511f218079a63bbba0158fa367"}, - {file = "pex-2.18.0.tar.gz", hash = "sha256:92f2c2f59514561f95870b00aac07b20f2d87b47703f5b226edd3a1464227b8e"}, + {file = "pex-2.19.1-py2.py3-none-any.whl", hash = "sha256:e837a1415e65fddad288de6e8b2209a0bca5a7e369cd0c20d02509364890c2ef"}, + {file = "pex-2.19.1.tar.gz", hash = "sha256:9416ff00002a9eb21c725c67caf914cf387ea94c01329437328a9655d76b8311"}, ] [[package]] name = "platformdirs" -version = "4.3.2" +version = "4.3.6" requires_python = ">=3.8" summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." groups = ["dev"] files = [ - {file = "platformdirs-4.3.2-py3-none-any.whl", hash = "sha256:eb1c8582560b34ed4ba105009a4badf7f6f85768b30126f351328507b2beb617"}, - {file = "platformdirs-4.3.2.tar.gz", hash = "sha256:9e5e27a08aa095dd127b9f2e764d74254f482fef22b0970773bfba79d091ab8c"}, + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, ] [[package]] @@ -2142,18 +2142,18 @@ files = [ [[package]] name = "protobuf" -version = "4.25.4" +version = "4.25.5" requires_python = ">=3.8" summary = "" groups = ["default", "dev"] files = [ - {file = "protobuf-4.25.4-cp310-abi3-win32.whl", hash = "sha256:db9fd45183e1a67722cafa5c1da3e85c6492a5383f127c86c4c4aa4845867dc4"}, - {file = "protobuf-4.25.4-cp310-abi3-win_amd64.whl", hash = "sha256:ba3d8504116a921af46499471c63a85260c1a5fc23333154a427a310e015d26d"}, - {file = "protobuf-4.25.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:eecd41bfc0e4b1bd3fa7909ed93dd14dd5567b98c941d6c1ad08fdcab3d6884b"}, - {file = "protobuf-4.25.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:4c8a70fdcb995dcf6c8966cfa3a29101916f7225e9afe3ced4395359955d3835"}, - {file = "protobuf-4.25.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3319e073562e2515c6ddc643eb92ce20809f5d8f10fead3332f71c63be6a7040"}, - {file = "protobuf-4.25.4-py3-none-any.whl", hash = "sha256:bfbebc1c8e4793cfd58589acfb8a1026be0003e852b9da7db5a4285bde996978"}, - {file = "protobuf-4.25.4.tar.gz", hash = "sha256:0dc4a62cc4052a036ee2204d26fe4d835c62827c855c8a03f29fe6da146b380d"}, + {file = "protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8"}, + {file = "protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea"}, + {file = "protobuf-4.25.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2fde3d805354df675ea4c7c6338c1aecd254dfc9925e88c6d31a2bcb97eb173"}, + {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:919ad92d9b0310070f8356c24b855c98df2b8bd207ebc1c0c6fcc9ab1e007f3d"}, + {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fe14e16c22be926d3abfcb500e60cab068baf10b542b8c858fa27e098123e331"}, + {file = "protobuf-4.25.5-py3-none-any.whl", hash = "sha256:0aebecb809cae990f8129ada5ca273d9d670b76d9bfc9b1809f0a9c02b7dbf41"}, + {file = "protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584"}, ] [[package]] @@ -2270,24 +2270,24 @@ files = [ [[package]] name = "pydantic" -version = "2.9.1" +version = "2.9.2" requires_python = ">=3.8" summary = "Data validation using Python type hints" groups = ["default", "dev"] dependencies = [ "annotated-types>=0.6.0", - "pydantic-core==2.23.3", + "pydantic-core==2.23.4", "typing-extensions>=4.12.2; python_version >= \"3.13\"", "typing-extensions>=4.6.1; python_version < \"3.13\"", ] files = [ - {file = "pydantic-2.9.1-py3-none-any.whl", hash = "sha256:7aff4db5fdf3cf573d4b3c30926a510a10e19a0774d38fc4967f78beb6deb612"}, - {file = "pydantic-2.9.1.tar.gz", hash = "sha256:1363c7d975c7036df0db2b4a61f2e062fbc0aa5ab5f2772e0ffc7191a4f4bce2"}, + {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, + {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, ] [[package]] name = "pydantic-core" -version = "2.23.3" +version = "2.23.4" requires_python = ">=3.8" summary = "Core functionality for Pydantic validation and serialization" groups = ["default", "dev"] @@ -2295,24 +2295,24 @@ dependencies = [ "typing-extensions!=4.7.0,>=4.6.0", ] files = [ - {file = "pydantic_core-2.23.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e0ec50663feedf64d21bad0809f5857bac1ce91deded203efc4a84b31b2e4305"}, - {file = "pydantic_core-2.23.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:db6e6afcb95edbe6b357786684b71008499836e91f2a4a1e55b840955b341dbb"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98ccd69edcf49f0875d86942f4418a4e83eb3047f20eb897bffa62a5d419c8fa"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a678c1ac5c5ec5685af0133262103defb427114e62eafeda12f1357a12140162"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01491d8b4d8db9f3391d93b0df60701e644ff0894352947f31fff3e52bd5c801"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fcf31facf2796a2d3b7fe338fe8640aa0166e4e55b4cb108dbfd1058049bf4cb"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7200fd561fb3be06827340da066df4311d0b6b8eb0c2116a110be5245dceb326"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dc1636770a809dee2bd44dd74b89cc80eb41172bcad8af75dd0bc182c2666d4c"}, - {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:67a5def279309f2e23014b608c4150b0c2d323bd7bccd27ff07b001c12c2415c"}, - {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:748bdf985014c6dd3e1e4cc3db90f1c3ecc7246ff5a3cd4ddab20c768b2f1dab"}, - {file = "pydantic_core-2.23.3-cp312-none-win32.whl", hash = "sha256:255ec6dcb899c115f1e2a64bc9ebc24cc0e3ab097775755244f77360d1f3c06c"}, - {file = "pydantic_core-2.23.3-cp312-none-win_amd64.whl", hash = "sha256:40b8441be16c1e940abebed83cd006ddb9e3737a279e339dbd6d31578b802f7b"}, - {file = "pydantic_core-2.23.3.tar.gz", hash = "sha256:3cb0f65d8b4121c1b015c60104a685feb929a29d7cf204387c7f2688c7974690"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, + {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, + {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, + {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, ] [[package]] name = "pydantic" -version = "2.9.1" +version = "2.9.2" extras = ["email"] requires_python = ">=3.8" summary = "Data validation using Python type hints" @@ -2320,11 +2320,11 @@ groups = ["dev"] marker = "python_version ~= \"3.11\"" dependencies = [ "email-validator>=2.0.0", - "pydantic==2.9.1", + "pydantic==2.9.2", ] files = [ - {file = "pydantic-2.9.1-py3-none-any.whl", hash = "sha256:7aff4db5fdf3cf573d4b3c30926a510a10e19a0774d38fc4967f78beb6deb612"}, - {file = "pydantic-2.9.1.tar.gz", hash = "sha256:1363c7d975c7036df0db2b4a61f2e062fbc0aa5ab5f2772e0ffc7191a4f4bce2"}, + {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, + {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, ] [[package]] @@ -2401,13 +2401,14 @@ files = [ [[package]] name = "pyreadline3" -version = "3.4.3" +version = "3.5.3" +requires_python = ">=3.8" summary = "A python implementation of GNU readline." groups = ["default", "dev"] marker = "sys_platform == \"win32\" and python_version >= \"3.8\"" files = [ - {file = "pyreadline3-3.4.3-py3-none-any.whl", hash = "sha256:f832c5898f4f9a0f81d48a8c499b39d0179de1a465ea3def1a7e7231840b4ed6"}, - {file = "pyreadline3-3.4.3.tar.gz", hash = "sha256:ebab0baca37f50e2faa1dd99a6da1c75de60e0d68a3b229c134bbd12786250e2"}, + {file = "pyreadline3-3.5.3-py3-none-any.whl", hash = "sha256:ddede153a92e5aad9c1fe63d692efd6a3e478f686adcd4938a051ffb63ec4f52"}, + {file = "pyreadline3-3.5.3.tar.gz", hash = "sha256:9234684ca75a00a702fda42b17cc26ca665bc9d7c2da06af450468253099ff61"}, ] [[package]] @@ -2691,13 +2692,13 @@ files = [ [[package]] name = "setuptools" -version = "74.1.2" +version = "75.1.0" requires_python = ">=3.8" summary = "Easily download, build, install, upgrade, and uninstall Python packages" groups = ["default", "dev"] files = [ - {file = "setuptools-74.1.2-py3-none-any.whl", hash = "sha256:5f4c08aa4d3ebcb57a50c33b1b07e94315d7fc7230f7115e47fc99776c8ce308"}, - {file = "setuptools-74.1.2.tar.gz", hash = "sha256:95b40ed940a1c67eb70fc099094bd6e99c6ee7c23aa2306f4d2697ba7916f9c6"}, + {file = "setuptools-75.1.0-py3-none-any.whl", hash = "sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2"}, + {file = "setuptools-75.1.0.tar.gz", hash = "sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538"}, ] [[package]] @@ -2746,7 +2747,7 @@ files = [ [[package]] name = "sqlalchemy" -version = "2.0.34" +version = "2.0.35" requires_python = ">=3.7" summary = "Database Abstraction Library" groups = ["default", "dev"] @@ -2756,43 +2757,43 @@ dependencies = [ "typing-extensions>=4.6.0", ] files = [ - {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:53e68b091492c8ed2bd0141e00ad3089bcc6bf0e6ec4142ad6505b4afe64163e"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bcd18441a49499bf5528deaa9dee1f5c01ca491fc2791b13604e8f972877f812"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:165bbe0b376541092bf49542bd9827b048357f4623486096fc9aaa6d4e7c59a2"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3330415cd387d2b88600e8e26b510d0370db9b7eaf984354a43e19c40df2e2b"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97b850f73f8abbffb66ccbab6e55a195a0eb655e5dc74624d15cff4bfb35bd74"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee4c6917857fd6121ed84f56d1dc78eb1d0e87f845ab5a568aba73e78adf83"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-win32.whl", hash = "sha256:fbb034f565ecbe6c530dff948239377ba859420d146d5f62f0271407ffb8c580"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-win_amd64.whl", hash = "sha256:707c8f44931a4facd4149b52b75b80544a8d824162602b8cd2fe788207307f9a"}, - {file = "SQLAlchemy-2.0.34-py3-none-any.whl", hash = "sha256:7286c353ee6475613d8beff83167374006c6b3e3f0e6491bfe8ca610eb1dec0f"}, - {file = "sqlalchemy-2.0.34.tar.gz", hash = "sha256:10d8f36990dd929690666679b0f42235c159a7051534adb135728ee52828dd22"}, + {file = "SQLAlchemy-2.0.35-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:eb60b026d8ad0c97917cb81d3662d0b39b8ff1335e3fabb24984c6acd0c900a2"}, + {file = "SQLAlchemy-2.0.35-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6921ee01caf375363be5e9ae70d08ce7ca9d7e0e8983183080211a062d299468"}, + {file = "SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cdf1a0dbe5ced887a9b127da4ffd7354e9c1a3b9bb330dce84df6b70ccb3a8d"}, + {file = "SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93a71c8601e823236ac0e5d087e4f397874a421017b3318fd92c0b14acf2b6db"}, + {file = "SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e04b622bb8a88f10e439084486f2f6349bf4d50605ac3e445869c7ea5cf0fa8c"}, + {file = "SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1b56961e2d31389aaadf4906d453859f35302b4eb818d34a26fab72596076bb8"}, + {file = "SQLAlchemy-2.0.35-cp312-cp312-win32.whl", hash = "sha256:0f9f3f9a3763b9c4deb8c5d09c4cc52ffe49f9876af41cc1b2ad0138878453cf"}, + {file = "SQLAlchemy-2.0.35-cp312-cp312-win_amd64.whl", hash = "sha256:25b0f63e7fcc2a6290cb5f7f5b4fc4047843504983a28856ce9b35d8f7de03cc"}, + {file = "SQLAlchemy-2.0.35-py3-none-any.whl", hash = "sha256:2ab3f0336c0387662ce6221ad30ab3a5e6499aab01b9790879b6578fd9b8faa1"}, + {file = "sqlalchemy-2.0.35.tar.gz", hash = "sha256:e11d7ea4d24f0a262bccf9a7cd6284c976c5369dac21db237cff59586045ab9f"}, ] [[package]] name = "sqlglot" -version = "25.21.0" +version = "25.21.3" requires_python = ">=3.7" summary = "An easily customizable SQL parser and transpiler" groups = ["default"] files = [ - {file = "sqlglot-25.21.0-py3-none-any.whl", hash = "sha256:609659e30cc94bc7d2cb4b4d8e5ab129ad5df06fbf7ec362be506dc5b383ffaa"}, - {file = "sqlglot-25.21.0.tar.gz", hash = "sha256:80572e1c79c567ce26b07f98321208820a4c3a575096c7d4949beec6e7d033b8"}, + {file = "sqlglot-25.21.3-py3-none-any.whl", hash = "sha256:dc63b429b80a69f2240ef892f776830883667fc9d978984ab98e7ce07edb7057"}, + {file = "sqlglot-25.21.3.tar.gz", hash = "sha256:273a447f71434ab2f9a36b81a6327706369735a0756a61cd576ac6896a5086a4"}, ] [[package]] name = "sqlglot" -version = "25.21.0" +version = "25.21.3" extras = ["rs"] requires_python = ">=3.7" summary = "An easily customizable SQL parser and transpiler" groups = ["default"] dependencies = [ - "sqlglot==25.21.0", + "sqlglot==25.21.3", "sqlglotrs==0.2.12", ] files = [ - {file = "sqlglot-25.21.0-py3-none-any.whl", hash = "sha256:609659e30cc94bc7d2cb4b4d8e5ab129ad5df06fbf7ec362be506dc5b383ffaa"}, - {file = "sqlglot-25.21.0.tar.gz", hash = "sha256:80572e1c79c567ce26b07f98321208820a4c3a575096c7d4949beec6e7d033b8"}, + {file = "sqlglot-25.21.3-py3-none-any.whl", hash = "sha256:dc63b429b80a69f2240ef892f776830883667fc9d978984ab98e7ce07edb7057"}, + {file = "sqlglot-25.21.3.tar.gz", hash = "sha256:273a447f71434ab2f9a36b81a6327706369735a0756a61cd576ac6896a5086a4"}, ] [[package]] @@ -2829,15 +2830,14 @@ files = [ [[package]] name = "sshtunnel" version = "0.4.0" +git = "https://github.com/pahaz/sshtunnel.git" +ref = "master" +revision = "aa7342e55e49529fea020a09ba8f1eed33eb896f" summary = "Pure python SSH tunnels" groups = ["default"] dependencies = [ "paramiko>=2.7.2", ] -files = [ - {file = "sshtunnel-0.4.0-py2.py3-none-any.whl", hash = "sha256:98e54c26f726ab8bd42b47a3a21fca5c3e60f58956f0f70de2fb8ab0046d0606"}, - {file = "sshtunnel-0.4.0.tar.gz", hash = "sha256:e7cb0ea774db81bf91844db22de72a40aae8f7b0f9bb9ba0f666d474ef6bf9fc"}, -] [[package]] name = "starlette" @@ -2867,7 +2867,7 @@ files = [ [[package]] name = "tableauserverclient" -version = "0.32" +version = "0.33" requires_python = ">=3.7" summary = "A Python module for working with the Tableau Server REST API." groups = ["default"] @@ -2879,8 +2879,8 @@ dependencies = [ "urllib3==2.2.2", ] files = [ - {file = "tableauserverclient-0.32-py3-none-any.whl", hash = "sha256:ac704c1711912b37eb1689047937059d472570ded8a13d54f6a4066adc6ceb71"}, - {file = "tableauserverclient-0.32.tar.gz", hash = "sha256:f51b9dabfff5fce5b649785d1ed3a732bf23f8596836e870e4b1ef5ca6616227"}, + {file = "tableauserverclient-0.33-py3-none-any.whl", hash = "sha256:1be70f1f71e29ed2f6c9568474872d29101553db269b6ed8ba16cbba8c3b4bfa"}, + {file = "tableauserverclient-0.33.tar.gz", hash = "sha256:ef28ff132de6211d8667482d364aebb682849ae0392e285994cf6a3e16d1390c"}, ] [[package]] @@ -3260,11 +3260,11 @@ files = [ [[package]] name = "zipp" -version = "3.20.1" +version = "3.20.2" requires_python = ">=3.8" summary = "Backport of pathlib-compatible object wrapper for zip files" groups = ["default"] files = [ - {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, - {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, + {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, + {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] diff --git a/pyproject.toml b/pyproject.toml index 11df7f63ce..8c0be6b898 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "tableauserverclient>=0.25", "tenacity>=8.2.3", "pendulum>=3.0.0", + "sshtunnel @ git+https://github.com/pahaz/sshtunnel.git@master", ] requires-python = ">=3.12,<3.13" license = { text = "GPL-3.0-or-later" } @@ -42,13 +43,13 @@ dev = ["pytest>=7.4.3", "datamodel-code-generator>=0.25.4", "dagster-webserver"] [tool.pdm.scripts] _.env_file = "env/.env" +post_install = { shell = "pdm export --prod -o requirements.txt --no-hashes" } clean = { shell = "bash .pdm/scripts/clean.sh", help = "Remove all build, test, coverage, and Python artifacts" } install-dagster = { shell = "bash .pdm/scripts/install-dagster.sh", help = "Install Dagster Cloud via Helm" } install-1password = { shell = "bash .pdm/scripts/install-1password.sh", help = "Install 1Password Connect via Helm" } dagster-dev = { shell = "bash .pdm/scripts/dagster-dev.sh", help = "Validate Dagster imports and configs" } dbt = { shell = "bash .pdm/scripts/dbt.sh" } dbt-sxs = { shell = "pdm run bash .pdm/scripts/dbt-stage-external-sources.sh" } -post_install = { shell = "pdm export --prod -o requirements.txt" } json2py = { shell = "bash .pdm/scripts/json2py.sh" } op-inject = { shell = "op inject -f --in-file=.devcontainer/tpl/.env.tpl --out-file=env/.env" } op-items = { shell = "kubectl apply -f .k8s/1password/items.yaml" } diff --git a/requirements.txt b/requirements.txt index 2d95401495..eea077ab55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,788 +1,165 @@ # This file is @generated by PDM. # Please do not edit it manually. -agate==1.9.1 \ - --hash=sha256:1cf329510b3dde07c4ad1740b7587c9c679abc3dcd92bb1107eabc10c2e03c50 \ - --hash=sha256:bc60880c2ee59636a2a80cd8603d63f995be64526abf3cbba12f00767bcd5b3d -alembic==1.13.2 \ - --hash=sha256:1ff0ae32975f4fd96028c39ed9bb3c867fe3af956bd7bb37343b54c9fe7445ef \ - --hash=sha256:6b8733129a6224a9a711e17c99b08462dbf7cc9670ba8f2e2ae9af860ceb1953 -annotated-types==0.7.0 \ - --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ - --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 -attrs==24.2.0 \ - --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \ - --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2 -avro==1.12.0 \ - --hash=sha256:9a255c72e1837341dd4f6ff57b2b6f68c0f0cecdef62dd04962e10fd33bec05b \ - --hash=sha256:cad9c53b23ceed699c7af6bddced42e2c572fd6b408c257a7d4fc4e8cf2e2d6b -babel==2.16.0 \ - --hash=sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b \ - --hash=sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316 -bcrypt==4.2.0 \ - --hash=sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb \ - --hash=sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399 \ - --hash=sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291 \ - --hash=sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d \ - --hash=sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7 \ - --hash=sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d \ - --hash=sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe \ - --hash=sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060 \ - --hash=sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68 \ - --hash=sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c \ - --hash=sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458 \ - --hash=sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9 \ - --hash=sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328 \ - --hash=sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7 \ - --hash=sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34 \ - --hash=sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e \ - --hash=sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2 \ - --hash=sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5 \ - --hash=sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae \ - --hash=sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00 \ - --hash=sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841 \ - --hash=sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8 \ - --hash=sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221 -beautifulsoup4==4.12.3 \ - --hash=sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051 \ - --hash=sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed -cachetools==5.5.0 \ - --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \ - --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a -certifi==2024.8.30 \ - --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \ - --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9 -cffi==1.17.1 \ - --hash=sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36 \ - --hash=sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824 \ - --hash=sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3 \ - --hash=sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8 \ - --hash=sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903 \ - --hash=sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c \ - --hash=sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4 \ - --hash=sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65 \ - --hash=sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93 \ - --hash=sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff \ - --hash=sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5 \ - --hash=sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99 -charset-normalizer==3.3.2 \ - --hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \ - --hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \ - --hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \ - --hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \ - --hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \ - --hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \ - --hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \ - --hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \ - --hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \ - --hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \ - --hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \ - --hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \ - --hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \ - --hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \ - --hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \ - --hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \ - --hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 -click==8.1.7 \ - --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ - --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de -cmake==3.30.3 \ - --hash=sha256:1616e2806c4c85e21fd0b6e92a61d41cb47479b5305bfa6f0c00baacfd029d7d \ - --hash=sha256:1ca7e29f5952634274d33ec1cb0cd9ddb79cb0b09cc3887b55d24c9852eed9d0 \ - --hash=sha256:30c2cdf8a863573a5fd7bf39159fbb96e75ac1955e481d35e5295ac601ea23af \ - --hash=sha256:3b41b0fbf3b449dd387c71444c9eb7f23e9a8061554bbf8fd8157ee355427220 \ - --hash=sha256:592cfcf280570713b8743bf8a8dec3753e0b82a7791d7d79f5ddb4f2be8b48b8 \ - --hash=sha256:6e294e3f424175b085809f713dd7ee36edd36b6b8a579911ef90359d8f884658 \ - --hash=sha256:81e5dc3103a4c6594d3efdf652e21e21d610e264f0c489ebefa3db04b1cdd2bc \ - --hash=sha256:870ebf590fb2f7cc58c8aa5b4dc32b50d4ca9c2fb9f1e46cd0426a995a2ef71e \ - --hash=sha256:8cc4c67432cca5e7a24a74eb102bc0472581a71231e58c224e544373dcb147a7 \ - --hash=sha256:a5ac1157eaa1e95bd67f11bd6ebc6f85b42ce6f2aac7b93d28dd84a5230be55b \ - --hash=sha256:a9e14118824992313bd0e2b3b86d9c85d7883c39b784199ea755fc32aeeb9e81 \ - --hash=sha256:ba26cb3c19f5b4cb83787394647a5dafbd2922a6de4af39409d7d287536a617f \ - --hash=sha256:c015d02e5f25973b66b66a060d3ad8c1c382cf38ba7b09712770d9de50b67b80 \ - --hash=sha256:c98cf8980ed75dd15be9948da559a51ce4cd0f017fc44969a72dcd37f507fa61 \ - --hash=sha256:ca990748d1a1d778a1a31cc1e33dcb01f2ed6fb0a752e945ff9e2d5435cff191 \ - --hash=sha256:e0fd7746f8895ec54e20c5d5dcc76a42256483e1f4736050264a180a13f9f8ef \ - --hash=sha256:fc5fba153bd0255adb246f27358d98db597a62264b61970d32038f9c7f355a70 -colorama==0.4.6 \ - --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ - --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 -coloredlogs==14.0 \ - --hash=sha256:346f58aad6afd48444c2468618623638dadab76e4e70d5e10822676f2d32226a \ - --hash=sha256:a1fab193d2053aa6c0a97608c4342d031f1f93a3d1218432c59322441d31a505 -cramjam==2.8.3 \ - --hash=sha256:00524bb23f4abb3a3bfff08aa32b9274843170c5b43855807e0f59670e2ac98c \ - --hash=sha256:080f3eb7b648f5ba9d35084d8dddc68246a8f365df239792f6712908f0aa568e \ - --hash=sha256:24990be4010b2185dcecc67133cd727657036e7b132d7de598148f5b1eb8e452 \ - --hash=sha256:2be92c6f0bcffaf8ea6a8164fe0388a188fec2fa9eff1828e8b64dc3a83740f9 \ - --hash=sha256:572cb9a8dc5a189691d6e03a9bf9b4305fd9a9f36bb0f9fde55fc36837c2e6b3 \ - --hash=sha256:6b1fa0a6ea8183831d04572597c182bd6cece62d583a36cde1e6a86e72ce2389 \ - --hash=sha256:832224f52fa1e601e0ab678dba9bdfde3686fc4cd1a9f2ed4748f29eaf1cb553 \ - --hash=sha256:962b7106287bcc463150766b5b8c69f32dcc69713a8dbce00e0ca6936f95c55b \ - --hash=sha256:9efe6915aa7ef176f3a7f42a4e46504573215953331b139abefd20d07d8aba82 \ - --hash=sha256:ab67f29094165f0771acad8dd16e840259cfedcc94067af229530496dbf1a24c \ - --hash=sha256:afa065bab70e27565695441f69f493af3d379b8723030f2c3d2547d2e312a4be \ - --hash=sha256:be6fb5dd5bf1c89c717a73a1057505959f35c08e0e97a76d4cc6391b90d2263b \ - --hash=sha256:c14728e3360cd212d5b606ca703c3bd1c8912efcdbc1aa032c81c2882509ebd5 \ - --hash=sha256:d93b42d22bf3e17290c5e4cf58e715a419330bb5255c35933c14db82ecf3872c \ - --hash=sha256:fe84440100e7045190da7f80219be9989b0b6db6acadb3ae9cfe0935d93ebf8c -croniter==3.0.3 \ - --hash=sha256:34117ec1741f10a7bd0ec3ad7d8f0eb8fa457a2feb9be32e6a2250e158957668 \ - --hash=sha256:b3bd11f270dc54ccd1f2397b813436015a86d30ffc5a7a9438eec1ed916f2101 -cryptography==43.0.1 \ - --hash=sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494 \ - --hash=sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806 \ - --hash=sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d \ - --hash=sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062 \ - --hash=sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2 \ - --hash=sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4 \ - --hash=sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1 \ - --hash=sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85 \ - --hash=sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042 \ - --hash=sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d \ - --hash=sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962 \ - --hash=sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa \ - --hash=sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d \ - --hash=sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47 \ - --hash=sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d \ - --hash=sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c \ - --hash=sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb \ - --hash=sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277 \ - --hash=sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a -daff==1.3.46 \ - --hash=sha256:22d0da9fd6a3275b54c926a9c97b180f9258aad65113ea18f3fec52cbadcd818 -dagster==1.8.7 \ - --hash=sha256:7117eb7ba6486377bdb0fb985b82cc86e8c6debc5f60d92e0bbe5bcc02837fa4 \ - --hash=sha256:b50c43654a1c3685413fa3a73a1d0eb3f9be6b42d7a3f0b648954faa542facf4 -dagster-airbyte==0.24.7 \ - --hash=sha256:36c2feeacd7cb484a936ed45090c42e9405b10edc2e5c844bbf936204f940134 \ - --hash=sha256:e7ef478e4780abce349e6a08785693df53790a1a921b0875bad31371007704b7 -dagster-cloud==1.8.7 \ - --hash=sha256:3490b42d449503d0c5f57c675ce579358f82851c2e2aabc0325b18f00c9abb28 \ - --hash=sha256:aad2b0f6c5b68dea4d75cbeb2c7d553faf101b1c3d5bd1c4e7bbe8c6110d5040 -dagster-cloud-cli==1.8.7 \ - --hash=sha256:371aebdae56586caa1a1a6a36ad26fa0b317d22a7eeade88c18753d034349b02 \ - --hash=sha256:4711f319bf10079175fa8205be3f7b530ee96bf39050207ecce8f7fc045dd161 -dagster-dbt==0.24.7 \ - --hash=sha256:5f2d3a63b32a80dc3ef82901817621c893aec13ed2246a1fa9ead10820669593 \ - --hash=sha256:8dee70bfc3295c045ed34cc910218d224b9ffefc1d6d876ff999acf51f052a08 -dagster-fivetran==0.24.7 \ - --hash=sha256:4521bc2150aa7650130e6ee3404db32042b2d5d64d15403298e0c238afb7f2f3 \ - --hash=sha256:923e731955d10b525bbbd69d2d95e06014a80a7867b43fb2ab26ecbc503217f1 -dagster-gcp==0.24.7 \ - --hash=sha256:412efb6a9dcfbe381c9eb74df983b7359fee619c614a5c4bcb488e9dec94995b \ - --hash=sha256:dd3b3c20dfe3436b3bc7111af983a6d84d942cfea8d6e0593255250966aa359e -dagster-k8s==0.24.7 \ - --hash=sha256:22ccee333da8910c597fafc55aca492225f436efb53cd3166f369d3b11133d3f \ - --hash=sha256:5012b784eb68e1fa588f26c97f4b262c30a94813a258a666bc01e920504b7b6c -dagster-pandas==0.24.7 \ - --hash=sha256:72ec5d54b94e882b2467e86363b478b580f29dd8af55307242d5b2edfdb70290 \ - --hash=sha256:bb3c3f566054d27ef0f0e6b32a4e4fe80e31435c06b7797de4aa6f9cc053fe71 -dagster-pipes==1.8.7 \ - --hash=sha256:9808077299bb9c753311926e83c13abfcb5502d6f444291370d2455f188ef98f \ - --hash=sha256:ff1bae741a26d37cdacbc524a65d206cbe848d2104beafe96cd32aab2e450fb3 -dagster-ssh==0.24.7 \ - --hash=sha256:346a790b3cf8ddd34df8b82798015b0cf9ffbc599ff9d3904bcc4eb72e59c590 \ - --hash=sha256:e8b2974c9c2922183c812ad4ab646c915bf7ed44080da588018ce4cfffbf91a9 -db-dtypes==1.3.0 \ - --hash=sha256:7bcbc8858b07474dc85b77bb2f3ae488978d1336f5ea73b58c39d9118bc3e91b \ - --hash=sha256:7e65c59f849ccbe6f7bc4d0253edcc212a7907662906921caba3e4aadd0bc277 -dbt-adapters==1.6.0 \ - --hash=sha256:ac3b73b28691d95a9b56b70dae5a0e2e989bf539d601984cfcd3e868462cf037 \ - --hash=sha256:fdda113eaae4d4e31c42257c4d68e94d1bd3519476b3326c4d859d3f33278cfc -dbt-bigquery==1.8.2 \ - --hash=sha256:b18bb2f90665a77e9588fe5939d0251325da214048a7250ffd3cfd9a8443c1d3 \ - --hash=sha256:beb44cce8b8dcd1c874bf1d52cbd91116c531668a7b45ff00c966b4b034872cb -dbt-common==1.8.0 \ - --hash=sha256:78fc3855d1b53c56e1728edd58b83e3a7a2e6c0063bb0fd724c3b84fc7f90196 \ - --hash=sha256:7a167e6b7cf39e758c63d349c7d2df46d915d6f1e2207d07121b1859f31b99be -dbt-core==1.8.6 \ - --hash=sha256:a0d7187ff69615613f091b48909b3ab719def643e7ccb74670f76dd482b4933c \ - --hash=sha256:a155573745b62c892950b20fb8c4947844fa32ccced89d1cea4f5c81e11438b9 -dbt-extractor==0.5.1 \ - --hash=sha256:100453ba06e169cbdb118234ab3f06f6722a2e0e316089b81c88dea701212abc \ - --hash=sha256:1b25fa7a276ab26aa2d70ff6e0cf4cfb1490d7831fb57ee1337c24d2b0333b84 \ - --hash=sha256:3614ce9f83ae4cd0dc95f77730034a793a1c090a52dcf698ba1c94050afe3a8b \ - --hash=sha256:3b91e6106b967d908b34f83929d3f50ee2b498876a1be9c055fe060ed728c556 \ - --hash=sha256:475e2c05b17eb4976eff6c8f7635be42bec33f15a74ceb87a40242c94a99cebf \ - --hash=sha256:62e4f040fd338b652683421ce48e903812e27fd6e7af58b1b70a4e1f9f2c79e3 \ - --hash=sha256:6916aae085fd5f2af069fd6947933e78b742c9e3d2165e1740c2e28ae543309a \ - --hash=sha256:91e25ad78f1f4feadd27587ebbcc46ad909cfad843118908f30336d08d8400ca \ - --hash=sha256:c0ce901d4ebf0664977e4e1cbf596d4afc6c1339fcc7d2cf67ce3481566a626f \ - --hash=sha256:c5651e458be910ff567c0da3ea2eb084fd01884cc88888ac2cf1e240dcddacc2 \ - --hash=sha256:cbe338b76e9ffaa18275456e041af56c21bb517f6fbda7a58308138703da0996 \ - --hash=sha256:cd5d95576a8dea4190240aaf9936a37fd74b4b7913ca69a3c368fc4472bb7e13 \ - --hash=sha256:cdf9938b36cd098bcdd80f43dc03864da3f69f57d903a9160a32236540d4ddcd \ - --hash=sha256:d3b9bf50eb062b4344d9546fe42038996c6e7e7daa10724aa955d64717260e5d \ - --hash=sha256:ea4edf33035d0a060b1e01c42fb2d99316457d44c954d6ed4eed9f1948664d87 \ - --hash=sha256:eecc08f3743e802a8ede60c89f7b2bce872acc86120cbc0ae7df229bb8a95083 -dbt-semantic-interfaces==0.5.1 \ - --hash=sha256:3a497abef1ba8112affdf804b26bfdcd5468ed95cc924b509068e18d371c7c4d \ - --hash=sha256:b95ff3a6721dc30f6278cb84933d95e0ef27766e67eeb6bb41906242e77f7c9b -deepdiff==7.0.1 \ - --hash=sha256:260c16f052d4badbf60351b4f77e8390bee03a0b516246f6839bc813fb429ddf \ - --hash=sha256:447760081918216aa4fd4ca78a4b6a848b81307b2ea94c810255334b759e1dc3 -defusedxml==0.7.1 \ - --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ - --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 -docstring-parser==0.16 \ - --hash=sha256:538beabd0af1e2db0146b6bd3caa526c35a34d61af9fd2887f3a8a27a739aa6e \ - --hash=sha256:bf0a1387354d3691d102edef7ec124f219ef639982d096e26e3b60aeffa90637 -fastavro==1.9.7 \ - --hash=sha256:13e11c6cb28626da85290933027cd419ce3f9ab8e45410ef24ce6b89d20a1f6c \ - --hash=sha256:4e1289b731214a7315884c74b2ec058b6e84380ce9b18b8af5d387e64b18fc44 \ - --hash=sha256:9be089be8c00f68e343bbc64ca6d9a13e5e5b0ba8aa52bcb231a762484fb270e \ - --hash=sha256:b6b2ccdc78f6afc18c52e403ee68c00478da12142815c1bd8a00973138a166d0 \ - --hash=sha256:d576eccfd60a18ffa028259500df67d338b93562c6700e10ef68bbd88e499731 \ - --hash=sha256:eac69666270a76a3a1d0444f39752061195e79e146271a568777048ffbd91a27 \ - --hash=sha256:ee9bf23c157bd7dcc91ea2c700fa3bd924d9ec198bb428ff0b47fa37fe160659 -filelock==3.16.0 \ - --hash=sha256:81de9eb8453c769b63369f87f11131a7ab04e367f8d97ad39dc230daa07e3bec \ - --hash=sha256:f6ed4c963184f4c84dd5557ce8fece759a3724b37b80c6c4f20a2f63a4dc6609 -fsspec==2024.9.0; python_version >= "3.12" \ - --hash=sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8 \ - --hash=sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b -github3-py==4.0.1 \ - --hash=sha256:30d571076753efc389edc7f9aaef338a4fcb24b54d8968d5f39b1342f45ddd36 \ - --hash=sha256:a89af7de25650612d1da2f0609622bcdeb07ee8a45a1c06b2d16a05e4234e753 -google-api-core[grpc]==2.19.2 \ - --hash=sha256:53ec0258f2837dd53bbd3d3df50f5359281b3cc13f800c941dd15a9b5a415af4 \ - --hash=sha256:ca07de7e8aa1c98a8bfca9321890ad2340ef7f2eb136e558cee68f24b94b0a8f -google-api-python-client==2.145.0 \ - --hash=sha256:8b84dde11aaccadc127e4846f5cd932331d804ea324e353131595e3f25376e97 \ - --hash=sha256:d74da1358f3f2d63daf3c6f26bd96d89652051183bc87cf10a56ceb2a70beb50 -google-auth==2.34.0 \ - --hash=sha256:72fd4733b80b6d777dcde515628a9eb4a577339437012874ea286bca7261ee65 \ - --hash=sha256:8eb87396435c19b20d32abd2f984e31c191a15284af72eb922f10e5bde9c04cc -google-auth-httplib2==0.2.0 \ - --hash=sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05 \ - --hash=sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d -google-auth-oauthlib==1.2.1 \ - --hash=sha256:2d58a27262d55aa1b87678c3ba7142a080098cbc2024f903c62355deb235d91f \ - --hash=sha256:afd0cad092a2eaa53cd8e8298557d6de1034c6cb4a740500b5357b648af97263 -google-cloud-bigquery[pandas]==3.25.0 \ - --hash=sha256:5b2aff3205a854481117436836ae1403f11f2594e6810a98886afd57eda28509 \ - --hash=sha256:7f0c371bc74d2a7fb74dacbc00ac0f90c8c2bec2289b51dd6685a275873b1ce9 -google-cloud-core==2.4.1 \ - --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ - --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 -google-cloud-dataproc==5.11.0 \ - --hash=sha256:771bcdeb46434177383433a8f092e52674482596eaa7aa5770d1b1b136727fe5 \ - --hash=sha256:ea38803987fd7818d133243db6f4771167e7aa4d4c7d82144e6f7c3e59df9218 -google-cloud-storage==2.18.2 \ - --hash=sha256:97a4d45c368b7d401ed48c4fdfe86e1e1cb96401c9e199e419d289e2c0370166 \ - --hash=sha256:aaf7acd70cdad9f274d29332673fcab98708d0e1f4dceb5a5356aaef06af4d99 -google-crc32c==1.6.0 \ - --hash=sha256:62f6d4a29fea082ac4a3c9be5e415218255cf11684ac6ef5488eea0c9132689b \ - --hash=sha256:6eceb6ad197656a1ff49ebfbbfa870678c75be4344feb35ac1edf694309413dc \ - --hash=sha256:7aec8e88a3583515f9e0957fe4f5f6d8d4997e36d0f61624e70469771584c760 \ - --hash=sha256:bd5e7d2445d1a958c266bfa5d04c39932dc54093fa391736dbfdb0f1929c1fb3 \ - --hash=sha256:c87d98c7c4a69066fd31701c4e10d178a648c2cac3452e62c6b24dc51f9fcc00 \ - --hash=sha256:ed767bf4ba90104c1216b68111613f0d5926fb3780660ea1198fc469af410e9d -google-resumable-media==2.7.2 \ - --hash=sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa \ - --hash=sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0 -googleapis-common-protos[grpc]==1.65.0 \ - --hash=sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63 \ - --hash=sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0 -greenlet==3.1.0; (platform_machine == "win32" or platform_machine == "WIN32" or platform_machine == "AMD64" or platform_machine == "amd64" or platform_machine == "x86_64" or platform_machine == "ppc64le" or platform_machine == "aarch64") and python_version < "3.13" \ - --hash=sha256:243a223c96a4246f8a30ea470c440fe9db1f5e444941ee3c3cd79df119b8eebf \ - --hash=sha256:24fc216ec7c8be9becba8b64a98a78f9cd057fd2dc75ae952ca94ed8a893bf27 \ - --hash=sha256:26811df4dc81271033a7836bc20d12cd30938e6bd2e9437f56fa03da81b0f8fc \ - --hash=sha256:26d9c1c4f1748ccac0bae1dbb465fb1a795a75aba8af8ca871503019f4285e2a \ - --hash=sha256:28fe80a3eb673b2d5cc3b12eea468a5e5f4603c26aa34d88bf61bba82ceb2f9b \ - --hash=sha256:3d07c28b85b350564bdff9f51c1c5007dfb2f389385d1bc23288de51134ca303 \ - --hash=sha256:a53dfe8f82b715319e9953330fa5c8708b610d48b5c59f1316337302af5c0811 \ - --hash=sha256:b395121e9bbe8d02a750886f108d540abe66075e61e22f7353d9acb0b81be0f0 \ - --hash=sha256:c9d86401550b09a55410f32ceb5fe7efcd998bd2dad9e82521713cb148a4a15f \ - --hash=sha256:cd468ec62257bb4544989402b19d795d2305eccb06cde5da0eb739b63dc04665 -grpc-google-iam-v1==0.13.1 \ - --hash=sha256:3ff4b2fd9d990965e410965253c0da6f66205d5a8291c4c31c6ebecca18a9001 \ - --hash=sha256:c3e86151a981811f30d5e7330f271cee53e73bb87755e88cc3b6f0c7b5fe374e -grpcio==1.66.1 \ - --hash=sha256:2ca2559692d8e7e245d456877a85ee41525f3ed425aa97eb7a70fc9a79df91a0 \ - --hash=sha256:35334f9c9745add3e357e3372756fd32d925bd52c41da97f4dfdafbde0bf0ee2 \ - --hash=sha256:7101db1bd4cd9b880294dec41a93fcdce465bdbb602cd8dc5bd2d6362b618759 \ - --hash=sha256:84ca1be089fb4446490dd1135828bd42a7c7f8421e74fa581611f7afdf7ab761 \ - --hash=sha256:a92c4f58c01c77205df6ff999faa008540475c39b835277fb8883b11cada127a \ - --hash=sha256:b0aa03d240b5539648d996cc60438f128c7f46050989e35b25f5c18286c86734 \ - --hash=sha256:b9feb4e5ec8dc2d15709f4d5fc367794d69277f5d680baf1910fc9915c633524 \ - --hash=sha256:d639c939ad7c440c7b2819a28d559179a4508783f7e5b991166f8d7a34b52815 \ - --hash=sha256:f03a5884c56256e08fd9e262e11b5cfacf1af96e2ce78dc095d2c41ccae2c80d \ - --hash=sha256:fdb14bad0835914f325349ed34a51940bc2ad965142eb3090081593c6e347be9 -grpcio-health-checking==1.62.3 \ - --hash=sha256:5074ba0ce8f0dcfe328408ec5c7551b2a835720ffd9b69dade7fa3e0dc1c7a93 \ - --hash=sha256:f29da7dd144d73b4465fe48f011a91453e9ff6c8af0d449254cf80021cab3e0d -grpcio-status==1.62.2 \ - --hash=sha256:206ddf0eb36bc99b033f03b2c8e95d319f0044defae9b41ae21408e7e0cda48f \ - --hash=sha256:62e1bfcb02025a1cd73732a2d33672d3e9d0df4d21c12c51e0bbcaf09bab742a -gspread==6.1.2 \ - --hash=sha256:345996fbb74051ee574e3d330a375ac625774f289459f73cb1f8b6fb3cf4cac5 \ - --hash=sha256:b147688b8c7a18c9835d5f998997ec17c97c0470babcab17f65ac2b3a32402b7 -httplib2==0.22.0 \ - --hash=sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc \ - --hash=sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81 -humanfriendly==10.0 \ - --hash=sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477 \ - --hash=sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc -idna==3.8 \ - --hash=sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac \ - --hash=sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603 -importlib-metadata==6.11.0 \ - --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ - --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b -isodate==0.6.1 \ - --hash=sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96 \ - --hash=sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9 -jinja2==3.1.4 \ - --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ - --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d -joblib==1.4.2 \ - --hash=sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6 \ - --hash=sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e -jsonschema==4.23.0 \ - --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ - --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 -jsonschema-specifications==2023.12.1 \ - --hash=sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc \ - --hash=sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c -kubernetes==30.1.0 \ - --hash=sha256:41e4c77af9f28e7a6c314e3bd06a8c6229ddd787cad684e0ab9f69b498e98ebc \ - --hash=sha256:e212e8b7579031dd2e512168b617373bc1e03888d41ac4e04039240a292d478d -ldap3==2.9.1 \ - --hash=sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70 \ - --hash=sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f -leather==0.4.0 \ - --hash=sha256:18290bc93749ae39039af5e31e871fcfad74d26c4c3ea28ea4f681f4571b3a2b \ - --hash=sha256:f964bec2086f3153a6c16e707f20cb718f811f57af116075f4c0f4805c608b95 -logbook==1.5.3 \ - --hash=sha256:66f454ada0f56eae43066f604a222b09893f98c1adc18df169710761b8f32fe8 -mako==1.3.5 \ - --hash=sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a \ - --hash=sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc -markdown-it-py==3.0.0 \ - --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ - --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb -markupsafe==2.1.5 \ - --hash=sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4 \ - --hash=sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169 \ - --hash=sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb \ - --hash=sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad \ - --hash=sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1 \ - --hash=sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee \ - --hash=sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f \ - --hash=sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a \ - --hash=sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b \ - --hash=sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b \ - --hash=sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5 -mashumaro[msgpack]==3.13.1 \ - --hash=sha256:169f0290253b3e6077bcb39c14a9dd0791a3fdedd9e286e536ae561d4ff1975b \ - --hash=sha256:ad0a162b8f4ea232dadd2891d77ff20165b855b9d84610f36ac84462d4576aa0 -mdurl==0.1.2 \ - --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ - --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba -memoization==0.4.0 \ - --hash=sha256:fde5e7cd060ef45b135e0310cfec17b2029dc472ccb5bbbbb42a503d4538a135 -minimal-snowplow-tracker==0.0.2 \ - --hash=sha256:acabf7572db0e7f5cbf6983d495eef54081f71be392330eb3aadb9ccb39daaa4 -more-itertools==10.5.0 \ - --hash=sha256:037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef \ - --hash=sha256:5482bfef7849c25dc3c6dd53a6173ae4795da2a41a80faea6700d9f5846c5da6 -msgpack==1.0.8 \ - --hash=sha256:0726c282d188e204281ebd8de31724b7d749adebc086873a59efb8cf7ae27df3 \ - --hash=sha256:114be227f5213ef8b215c22dde19532f5da9652e56e8ce969bf0a26d7c419fee \ - --hash=sha256:64d0fcd436c5683fdd7c907eeae5e2cbb5eb872fafbc03a43609d7941840995c \ - --hash=sha256:74398a4cf19de42e1498368c36eed45d9528f5fd0155241e82c4082b7e16cffd \ - --hash=sha256:8db8e423192303ed77cff4dce3a4b88dbfaf43979d280181558af5e2c3c71afc \ - --hash=sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3 \ - --hash=sha256:99881222f4a8c2f641f25703963a5cefb076adffd959e0558dc9f803a52d6a58 \ - --hash=sha256:b5505774ea2a73a86ea176e8a9a4a7c8bf5d521050f0f6f8426afe798689243f \ - --hash=sha256:d56fd9f1f1cdc8227d7b7918f55091349741904d9520c65f0139a9755952c9e8 \ - --hash=sha256:d661dc4785affa9d0edfdd1e59ec056a58b3dbb9f196fa43587f3ddac654ac7b \ - --hash=sha256:e1dd7839443592d00e96db831eddb4111a2a81a46b028f0facd60a09ebbdd543 \ - --hash=sha256:ef254a06bcea461e65ff0373d8a0dd1ed3aa004af48839f002a0c994a6f72d04 -networkx==3.3 \ - --hash=sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9 \ - --hash=sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2 -numpy==2.1.1 \ - --hash=sha256:3269c9eb8745e8d975980b3a7411a98976824e1fdef11f0aacf76147f662b15f \ - --hash=sha256:3fc5eabfc720db95d68e6646e88f8b399bfedd235994016351b1d9e062c4b270 \ - --hash=sha256:6435c48250c12f001920f0751fe50c0348f5f240852cfddc5e2f97e007544cbe \ - --hash=sha256:7c803b7934a7f59563db459292e6aa078bb38b7ab1446ca38dd138646a38203e \ - --hash=sha256:8661c94e3aad18e1ea17a11f60f843a4933ccaf1a25a7c6a9182af70610b2313 \ - --hash=sha256:950802d17a33c07cba7fd7c3dcfa7d64705509206be1606f196d179e539111ed \ - --hash=sha256:afd9c680df4de71cd58582b51e88a61feed4abcc7530bcd3d48483f20fc76f2a \ - --hash=sha256:d0cf7d55b1051387807405b3898efafa862997b4cba8aa5dbe657be794afeafd \ - --hash=sha256:d2b9cd92c8f8e7b313b80e93cedc12c0112088541dcedd9197b5dee3738c1201 \ - --hash=sha256:fac6e277a41163d27dfab5f4ec1f7a83fac94e170665a4a50191b545721c6521 \ - --hash=sha256:fcd8f556cdc8cfe35e70efb92463082b7f43dd7e547eb071ffc36abc0ca4699b -oauth2client==4.1.3 \ - --hash=sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac \ - --hash=sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6 -oauthlib==3.2.2 \ - --hash=sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca \ - --hash=sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918 -oracledb==2.4.1 \ - --hash=sha256:047fa173868fae989150bd8e8fa7d4d28d9228ae0f3367a3c2f662c9202599b1 \ - --hash=sha256:24c68c030cada6db5611a2d915576741cf34e369d324756fbefcd295ba6a551c \ - --hash=sha256:3cfaab99b2b84318c34a74af18452f59279c520a08a9307f0ec041ab2bf4d9d8 \ - --hash=sha256:70efa2f6caf958fb0234fee9514f6de219f71b1b16e69176f09290a33024e553 \ - --hash=sha256:7e9612ec44dfae89bd2ca08b6d655de2f83b274d9732766797fdb4759cfb9952 \ - --hash=sha256:bd5976bef0e466e0f9d1b9f6531fb5b8171dc8534717ccb04b26e680b6c7571d -ordered-set==4.1.0 \ - --hash=sha256:046e1132c71fcf3330438a539928932caf51ddbc582496833e23de611de14562 \ - --hash=sha256:694a8e44c87657c59292ede72891eb91d34131f6531463aab3009191c77364a8 -orjson==3.10.7 \ - --hash=sha256:1d9c0e733e02ada3ed6098a10a8ee0052dd55774de3d9110d29868d24b17faa1 \ - --hash=sha256:23820a1563a1d386414fef15c249040042b8e5d07b40ab3fe3efbfbbcbcb8864 \ - --hash=sha256:44a96f2d4c3af51bfac6bc4ef7b182aa33f2f054fd7f34cc0ee9a320d051d41f \ - --hash=sha256:75ef0640403f945f3a1f9f6400686560dbfb0fb5b16589ad62cd477043c4eee3 \ - --hash=sha256:76ac14cd57df0572453543f8f2575e2d01ae9e790c21f57627803f5e79b0d3c3 \ - --hash=sha256:7c4c17f8157bd520cdb7195f75ddbd31671997cbe10aee559c2d613592e7d7eb \ - --hash=sha256:a0c6a008e91d10a2564edbb6ee5069a9e66df3fbe11c9a005cb411f441fd2c09 \ - --hash=sha256:b48b3db6bb6e0a08fa8c83b47bc169623f801e5cc4f24442ab2b6617da3b5313 \ - --hash=sha256:bdbb61dcc365dd9be94e8f7df91975edc9364d6a78c8f7adb69c1cdff318ec93 \ - --hash=sha256:d2d9f990623f15c0ae7ac608103c33dfe1486d2ed974ac3f40b693bad1a22a7b \ - --hash=sha256:d352ee8ac1926d6193f602cbe36b1643bbd1bbcb25e3c1a657a4390f3000c9a5 -packaging==24.1 \ - --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ - --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 -pandas==2.2.2 \ - --hash=sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad \ - --hash=sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76 \ - --hash=sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32 \ - --hash=sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef \ - --hash=sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54 \ - --hash=sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23 \ - --hash=sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce \ - --hash=sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad -paramiko==3.4.1 \ - --hash=sha256:8b15302870af7f6652f2e038975c1d2973f06046cb5d7d65355668b3ecbece0c \ - --hash=sha256:8e49fd2f82f84acf7ffd57c64311aa2b30e575370dc23bdb375b10262f7eac32 -parsedatetime==2.6 \ - --hash=sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455 \ - --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b -pathspec==0.12.1 \ - --hash=sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 \ - --hash=sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712 -pendulum==3.0.0 \ - --hash=sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9 \ - --hash=sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f \ - --hash=sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f \ - --hash=sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7 \ - --hash=sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319 \ - --hash=sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e \ - --hash=sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5 \ - --hash=sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5 \ - --hash=sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc \ - --hash=sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518 \ - --hash=sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37 -pex==2.18.0 \ - --hash=sha256:159045724c4b8c67015daad5907ca51e01058e511f218079a63bbba0158fa367 \ - --hash=sha256:92f2c2f59514561f95870b00aac07b20f2d87b47703f5b226edd3a1464227b8e -prompt-toolkit==3.0.36 \ - --hash=sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63 \ - --hash=sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305 -proto-plus==1.24.0 \ - --hash=sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445 \ - --hash=sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12 -protobuf==4.25.4 \ - --hash=sha256:0dc4a62cc4052a036ee2204d26fe4d835c62827c855c8a03f29fe6da146b380d \ - --hash=sha256:3319e073562e2515c6ddc643eb92ce20809f5d8f10fead3332f71c63be6a7040 \ - --hash=sha256:4c8a70fdcb995dcf6c8966cfa3a29101916f7225e9afe3ced4395359955d3835 \ - --hash=sha256:ba3d8504116a921af46499471c63a85260c1a5fc23333154a427a310e015d26d \ - --hash=sha256:bfbebc1c8e4793cfd58589acfb8a1026be0003e852b9da7db5a4285bde996978 \ - --hash=sha256:db9fd45183e1a67722cafa5c1da3e85c6492a5383f127c86c4c4aa4845867dc4 \ - --hash=sha256:eecd41bfc0e4b1bd3fa7909ed93dd14dd5567b98c941d6c1ad08fdcab3d6884b -psutil==6.0.0; platform_system == "Windows" \ - --hash=sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3 \ - --hash=sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd \ - --hash=sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0 \ - --hash=sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2 \ - --hash=sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d \ - --hash=sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0 \ - --hash=sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132 \ - --hash=sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0 -py-avro-schema==3.8.2 \ - --hash=sha256:43577490e50bcc8bb4e32397e774661f8c21b6acaebf929bc46bce92bf441dcf \ - --hash=sha256:54bbffec7aad34222299dd2742e505f4a53bb7d2300a950a76cb32d414ceaaae -pyarrow==17.0.0 \ - --hash=sha256:0071ce35788c6f9077ff9ecba4858108eebe2ea5a3f7cf2cf55ebc1dbc6ee24a \ - --hash=sha256:392bc9feabc647338e6c89267635e111d71edad5fcffba204425a7c8d13610d7 \ - --hash=sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28 \ - --hash=sha256:757074882f844411fcca735e39aae74248a1531367a7c80799b4266390ae51cc \ - --hash=sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22 \ - --hash=sha256:9ba11c4f16976e89146781a83833df7f82077cdab7dc6232c897789343f7891a \ - --hash=sha256:b0c6ac301093b42d34410b187bba560b17c0330f64907bfa4f7f7f2444b0cf9b \ - --hash=sha256:f1e70de6cb5790a50b01d2b686d54aaf73da01266850b05e3af2a1bc89e16053 -pyasn1==0.6.1 \ - --hash=sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629 \ - --hash=sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034 -pyasn1-modules==0.4.1 \ - --hash=sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd \ - --hash=sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c -pycparser==2.22 \ - --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ - --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc -pycryptodome==3.20.0 \ - --hash=sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7 \ - --hash=sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4 \ - --hash=sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25 \ - --hash=sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a \ - --hash=sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c \ - --hash=sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72 \ - --hash=sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9 \ - --hash=sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044 \ - --hash=sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c \ - --hash=sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2 \ - --hash=sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128 -pydantic==2.9.1 \ - --hash=sha256:1363c7d975c7036df0db2b4a61f2e062fbc0aa5ab5f2772e0ffc7191a4f4bce2 \ - --hash=sha256:7aff4db5fdf3cf573d4b3c30926a510a10e19a0774d38fc4967f78beb6deb612 -pydantic-core==2.23.3 \ - --hash=sha256:01491d8b4d8db9f3391d93b0df60701e644ff0894352947f31fff3e52bd5c801 \ - --hash=sha256:255ec6dcb899c115f1e2a64bc9ebc24cc0e3ab097775755244f77360d1f3c06c \ - --hash=sha256:3cb0f65d8b4121c1b015c60104a685feb929a29d7cf204387c7f2688c7974690 \ - --hash=sha256:40b8441be16c1e940abebed83cd006ddb9e3737a279e339dbd6d31578b802f7b \ - --hash=sha256:67a5def279309f2e23014b608c4150b0c2d323bd7bccd27ff07b001c12c2415c \ - --hash=sha256:7200fd561fb3be06827340da066df4311d0b6b8eb0c2116a110be5245dceb326 \ - --hash=sha256:748bdf985014c6dd3e1e4cc3db90f1c3ecc7246ff5a3cd4ddab20c768b2f1dab \ - --hash=sha256:98ccd69edcf49f0875d86942f4418a4e83eb3047f20eb897bffa62a5d419c8fa \ - --hash=sha256:a678c1ac5c5ec5685af0133262103defb427114e62eafeda12f1357a12140162 \ - --hash=sha256:db6e6afcb95edbe6b357786684b71008499836e91f2a4a1e55b840955b341dbb \ - --hash=sha256:dc1636770a809dee2bd44dd74b89cc80eb41172bcad8af75dd0bc182c2666d4c \ - --hash=sha256:e0ec50663feedf64d21bad0809f5857bac1ce91deded203efc4a84b31b2e4305 \ - --hash=sha256:fcf31facf2796a2d3b7fe338fe8640aa0166e4e55b4cb108dbfd1058049bf4cb -pygments==2.18.0 \ - --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ - --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a -pyjwt[crypto]==2.9.0 \ - --hash=sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850 \ - --hash=sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c -pynacl==1.5.0 \ - --hash=sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858 \ - --hash=sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d \ - --hash=sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93 \ - --hash=sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1 \ - --hash=sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92 \ - --hash=sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff \ - --hash=sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba \ - --hash=sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394 \ - --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \ - --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543 -pyparsing==3.1.4; python_version > "3.0" \ - --hash=sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c \ - --hash=sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032 -pyreadline3==3.4.3; sys_platform == "win32" and python_version >= "3.8" \ - --hash=sha256:ebab0baca37f50e2faa1dd99a6da1c75de60e0d68a3b229c134bbd12786250e2 \ - --hash=sha256:f832c5898f4f9a0f81d48a8c499b39d0179de1a465ea3def1a7e7231840b4ed6 -python-dateutil==2.9.0.post0 \ - --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ - --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 -python-dotenv==1.0.1 \ - --hash=sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca \ - --hash=sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a -python-slugify==8.0.4 \ - --hash=sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8 \ - --hash=sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856 -pytimeparse==1.1.8 \ - --hash=sha256:04b7be6cc8bd9f5647a6325444926c3ac34ee6bc7e69da4367ba282f076036bd \ - --hash=sha256:e86136477be924d7e670646a98561957e8ca7308d44841e21f5ddea757556a0a -pytz==2024.2 \ - --hash=sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a \ - --hash=sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725 -pywin32==306; platform_system == "Windows" \ - --hash=sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e \ - --hash=sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b \ - --hash=sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040 -pyyaml==6.0.2 \ - --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ - --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ - --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ - --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ - --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ - --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ - --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ - --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ - --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ - --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 -questionary==2.0.1 \ - --hash=sha256:8ab9a01d0b91b68444dff7f6652c1e754105533f083cbe27597c8110ecc230a2 \ - --hash=sha256:bcce898bf3dbb446ff62830c86c5c6fb9a22a54146f0f5597d3da43b10d8fc8b -referencing==0.35.1 \ - --hash=sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c \ - --hash=sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de -requests==2.32.3 \ - --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ - --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 -requests-oauthlib==2.0.0 \ - --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ - --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 -rich==13.8.1 \ - --hash=sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06 \ - --hash=sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a -rpds-py==0.20.0 \ - --hash=sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585 \ - --hash=sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef \ - --hash=sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739 \ - --hash=sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174 \ - --hash=sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96 \ - --hash=sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee \ - --hash=sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b \ - --hash=sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139 \ - --hash=sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6 \ - --hash=sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c \ - --hash=sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821 \ - --hash=sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121 \ - --hash=sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940 \ - --hash=sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4 -rsa==4.9 \ - --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ - --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 -scikit-learn==1.5.2 \ - --hash=sha256:394397841449853c2290a32050382edaec3da89e35b3e03d6cc966aebc6a8ae6 \ - --hash=sha256:3b923d119d65b7bd555c73be5423bf06c0105678ce7e1f558cb4b40b0a5502b1 \ - --hash=sha256:57cc1786cfd6bd118220a92ede80270132aa353647684efa385a74244a41e3b1 \ - --hash=sha256:b4237ed7b3fdd0a4882792e68ef2545d5baa50aca3bb45aa7df468138ad8f94d \ - --hash=sha256:f60021ec1574e56632be2a36b946f8143bf4e5e6af4a06d85281adc22938e0dd \ - --hash=sha256:f932a02c3f4956dfb981391ab24bda1dbd90fe3d628e4b42caef3e041c67707a -scipy==1.14.1 \ - --hash=sha256:2843f2d527d9eebec9a43e6b406fb7266f3af25a751aa91d62ff416f54170bc5 \ - --hash=sha256:2ff38e22128e6c03ff73b6bb0f85f897d2362f8c052e3b8ad00532198fbdae3f \ - --hash=sha256:30ac8812c1d2aab7131a79ba62933a2a76f582d5dbbc695192453dae67ad6310 \ - --hash=sha256:5a275584e726026a5699459aa72f828a610821006228e841b94275c4a7c08417 \ - --hash=sha256:631f07b3734d34aced009aaf6fedfd0eb3498a97e581c3b1e5f14a04164a456d \ - --hash=sha256:8f9ea80f2e65bdaa0b7627fb00cbeb2daf163caa015e59b7516395fe3bd1e066 \ - --hash=sha256:af29a935803cc707ab2ed7791c44288a682f9c8107bc00f0eccc4f92c08d6e07 \ - --hash=sha256:eb58ca0abd96911932f688528977858681a59d61a7ce908ffd355957f7025cfc \ - --hash=sha256:edaf02b82cd7639db00dbff629995ef185c8df4c3ffa71a5562a595765a06ce1 -setuptools==74.1.2 \ - --hash=sha256:5f4c08aa4d3ebcb57a50c33b1b07e94315d7fc7230f7115e47fc99776c8ce308 \ - --hash=sha256:95b40ed940a1c67eb70fc099094bd6e99c6ee7c23aa2306f4d2697ba7916f9c6 -shellingham==1.5.4 \ - --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ - --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de -six==1.16.0 \ - --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ - --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 -soupsieve==2.6 \ - --hash=sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb \ - --hash=sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9 -sqlalchemy==2.0.34 \ - --hash=sha256:10d8f36990dd929690666679b0f42235c159a7051534adb135728ee52828dd22 \ - --hash=sha256:165bbe0b376541092bf49542bd9827b048357f4623486096fc9aaa6d4e7c59a2 \ - --hash=sha256:53e68b091492c8ed2bd0141e00ad3089bcc6bf0e6ec4142ad6505b4afe64163e \ - --hash=sha256:707c8f44931a4facd4149b52b75b80544a8d824162602b8cd2fe788207307f9a \ - --hash=sha256:7286c353ee6475613d8beff83167374006c6b3e3f0e6491bfe8ca610eb1dec0f \ - --hash=sha256:7cee4c6917857fd6121ed84f56d1dc78eb1d0e87f845ab5a568aba73e78adf83 \ - --hash=sha256:97b850f73f8abbffb66ccbab6e55a195a0eb655e5dc74624d15cff4bfb35bd74 \ - --hash=sha256:bcd18441a49499bf5528deaa9dee1f5c01ca491fc2791b13604e8f972877f812 \ - --hash=sha256:c3330415cd387d2b88600e8e26b510d0370db9b7eaf984354a43e19c40df2e2b \ - --hash=sha256:fbb034f565ecbe6c530dff948239377ba859420d146d5f62f0271407ffb8c580 -sqlglot[rs]==25.21.0 \ - --hash=sha256:609659e30cc94bc7d2cb4b4d8e5ab129ad5df06fbf7ec362be506dc5b383ffaa \ - --hash=sha256:80572e1c79c567ce26b07f98321208820a4c3a575096c7d4949beec6e7d033b8 -sqlglotrs==0.2.12 \ - --hash=sha256:08e8be22da77c964be76ab4438da2c77096f5871088466ca950ee1b4712a97d4 \ - --hash=sha256:147cda8412f45af290ad190d9a98b5829a5f46a575ce768279ccebf9b7b53785 \ - --hash=sha256:1fc98b7649445e726a492841b8b8b39a4e5724ec2787cd1436404ebccf42519a \ - --hash=sha256:4c07d3dba9c3ae8b56a0e45a9e47aa2a2c6ed95870c5bcc67dacaadb873843ff \ - --hash=sha256:5be231acf95920bed473524dd1cac93e4cb320ed7e6ae937531b232c54cfc232 \ - --hash=sha256:954ccd912391ab5922adb23159ebcc0c5dccb468381e2a1ce92117cb4b0f0ed3 \ - --hash=sha256:9d5b9a9d6259b72258f6764f88a89faa3c648438bd1b2c3a9598b725d42bf6f2 \ - --hash=sha256:acc25d651eb663332157c2e5d2736516cddf4cd0effe67a887723934de5051d1 \ - --hash=sha256:b10bf6b71961b31951bf4dff937d8d5d399ea1b3bd47fb5c5810386710fe7dfb \ - --hash=sha256:c8bf7ae29c0fc66e9c998d7f8e6f6fc26309c6eb5a4728e1443cb628218bc307 \ - --hash=sha256:f104a98182761d4613f920eda7ec5fc921afb3608f7db648206ce06dd10a6be5 -sqlparse==0.5.1 \ - --hash=sha256:773dcbf9a5ab44a090f3441e2180efe2560220203dc2f8c0b0fa141e18b505e4 \ - --hash=sha256:bb6b4df465655ef332548e24f08e205afc81b9ab86cb1c45657a7ff173a3a00e -sshtunnel==0.4.0 \ - --hash=sha256:98e54c26f726ab8bd42b47a3a21fca5c3e60f58956f0f70de2fb8ab0046d0606 \ - --hash=sha256:e7cb0ea774db81bf91844db22de72a40aae8f7b0f9bb9ba0f666d474ef6bf9fc -structlog==24.4.0 \ - --hash=sha256:597f61e80a91cc0749a9fd2a098ed76715a1c8a01f73e336b746504d1aad7610 \ - --hash=sha256:b27bfecede327a6d2da5fbc96bd859f114ecc398a6389d664f62085ee7ae6fc4 -tableauserverclient==0.32 \ - --hash=sha256:ac704c1711912b37eb1689047937059d472570ded8a13d54f6a4066adc6ceb71 \ - --hash=sha256:f51b9dabfff5fce5b649785d1ed3a732bf23f8596836e870e4b1ef5ca6616227 -tabulate==0.9.0 \ - --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \ - --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f -tenacity==9.0.0 \ - --hash=sha256:807f37ca97d62aa361264d497b0e31e92b8027044942bfa756160d908320d73b \ - --hash=sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539 -text-unidecode==1.3 \ - --hash=sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8 \ - --hash=sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93 -threadpoolctl==3.5.0 \ - --hash=sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107 \ - --hash=sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467 -time-machine==2.15.0; implementation_name != "pypy" \ - --hash=sha256:31af56399bf7c9ef76a3f7b6d9471dffa8f06ee373c194a374b69523f9061de9 \ - --hash=sha256:3862dda89bdb05f9d521b08fdcb24b19a7dd9f559ae324f4301ba7a07b6eea64 \ - --hash=sha256:5f7add997684bc6141e1c80f6ba0c38ffe316ba277a4074e61b1b7b4f5a172bf \ - --hash=sha256:9479530e3fce65f6149058071fa4df8150025f15b43b103445f619842981a87c \ - --hash=sha256:a22f47c34ee1fcf7d93a8c5c93135499aac879d9d5d8f820bd28571a30fdabcd \ - --hash=sha256:a731c03bc00552ee6cc685a59616d36003124e7e04c6ddf65c2c47f1c3d85480 \ - --hash=sha256:b5f3ab4185c1f72010846ca9fccb08349e23a2b52982a18d9870e848ce9f1c86 \ - --hash=sha256:b684f8ecdeacd6baabc17b15ac1b054ca62029193e6c5367ef00b3516671de80 \ - --hash=sha256:c0473dfa8f17c6a9a250b2bd6a5b62af3aa7d22518f701649115f1085d5e35ab \ - --hash=sha256:e1790481a6b9ce38888f22ce30710244067898c3ac4805a0e061e381f3db3506 \ - --hash=sha256:e6776840aea3ff5ab6924b50117957da62db51b109b3b491c0d5817a804b1a8e \ - --hash=sha256:ebd2e63baa117ded04b978813fcd1279d3fc6be2149c9cac75c716b6f1db774c \ - --hash=sha256:f5b94cba3edfc54bcb3ab5be616a2f50fa48be438e5af970824efdf882d1bc31 -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f -toposort==1.10 \ - --hash=sha256:bfbb479c53d0a696ea7402601f4e693c97b0367837c8898bc6471adfca37a6bd \ - --hash=sha256:cbdbc0d0bee4d2695ab2ceec97fe0679e9c10eab4b2a87a9372b929e70563a87 -tqdm==4.66.5 \ - --hash=sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd \ - --hash=sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad -typeguard==4.3.0 \ - --hash=sha256:4d24c5b39a117f8a895b9da7a9b3114f04eb63bade45a4492de49b175b6f7dfa \ - --hash=sha256:92ee6a0aec9135181eae6067ebd617fd9de8d75d714fb548728a4933b1dea651 -typer==0.12.5 \ - --hash=sha256:62fe4e471711b147e3365034133904df3e235698399bc4de2b36c8579298d52b \ - --hash=sha256:f592f089bedcc8ec1b974125d64851029c3b1af145f04aca64d69410f0c9b722 -typing-extensions==4.12.2 \ - --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ - --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 -tzdata==2024.1 \ - --hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \ - --hash=sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252 -universal-pathlib==0.2.5; python_version >= "3.12" \ - --hash=sha256:a634f700eca827b4ad03bfa0267e51161560dd1de83b051cf0fccf39b3e56b32 \ - --hash=sha256:ea5d4fb8178c2ab469cf4fa46d0ceb16ccb378da46dbbc28a8b9c1eebdccc655 -uritemplate==4.1.1 \ - --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ - --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e -urllib3==2.2.2 \ - --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \ - --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168 -watchdog==5.0.2 \ - --hash=sha256:29e4a2607bd407d9552c502d38b45a05ec26a8e40cc7e94db9bb48f861fa5abc \ - --hash=sha256:3960136b2b619510569b90f0cd96408591d6c251a75c97690f4553ca88889769 \ - --hash=sha256:53ed1bf71fcb8475dd0ef4912ab139c294c87b903724b6f4a8bd98e026862e6d \ - --hash=sha256:5597c051587f8757798216f2485e85eac583c3b343e9aa09127a3a6f82c65ee8 \ - --hash=sha256:726eef8f8c634ac6584f86c9c53353a010d9f311f6c15a034f3800a7a891d941 \ - --hash=sha256:7d1aa7e4bb0f0c65a1a91ba37c10e19dabf7eaaa282c5787e51371f090748f4b \ - --hash=sha256:aa9cd6e24126d4afb3752a3e70fce39f92d0e1a58a236ddf6ee823ff7dba28ee \ - --hash=sha256:b6dc8f1d770a8280997e4beae7b9a75a33b268c59e033e72c8a10990097e5fde \ - --hash=sha256:bda40c57115684d0216556671875e008279dea2dc00fcd3dde126ac8e0d7a2fb \ - --hash=sha256:d010be060c996db725fbce7e3ef14687cdcc76f4ca0e4339a68cc4532c382a73 \ - --hash=sha256:d2ab34adc9bf1489452965cdb16a924e97d4452fcf88a50b21859068b50b5c3b \ - --hash=sha256:d7594a6d32cda2b49df3fd9abf9b37c8d2f3eab5df45c24056b4a671ac661619 \ - --hash=sha256:dcebf7e475001d2cdeb020be630dc5b687e9acdd60d16fea6bb4508e7b94cf76 \ - --hash=sha256:f627c5bf5759fdd90195b0c0431f99cff4867d212a67b384442c51136a098ed7 -wcwidth==0.2.13 \ - --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ - --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 -websocket-client==1.8.0 \ - --hash=sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 \ - --hash=sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da -zipp==3.20.1 \ - --hash=sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064 \ - --hash=sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b +agate==1.9.1 +alembic==1.13.2 +annotated-types==0.7.0 +attrs==24.2.0 +avro==1.12.0 +babel==2.16.0 +bcrypt==4.2.0 +beautifulsoup4==4.12.3 +cachetools==5.5.0 +certifi==2024.8.30 +cffi==1.17.1 +charset-normalizer==3.3.2 +click==8.1.7 +cmake==3.30.3 +colorama==0.4.6 +coloredlogs==14.0 +cramjam==2.8.3 +croniter==3.0.3 +cryptography==43.0.1 +daff==1.3.46 +dagster==1.8.7 +dagster-airbyte==0.24.7 +dagster-cloud==1.8.7 +dagster-cloud-cli==1.8.7 +dagster-dbt==0.24.7 +dagster-fivetran==0.24.7 +dagster-gcp==0.24.7 +dagster-k8s==0.24.7 +dagster-pandas==0.24.7 +dagster-pipes==1.8.7 +dagster-ssh==0.24.7 +db-dtypes==1.3.0 +dbt-adapters==1.6.1 +dbt-bigquery==1.8.2 +dbt-common==1.8.0 +dbt-core==1.8.6 +dbt-extractor==0.5.1 +dbt-semantic-interfaces==0.5.1 +deepdiff==7.0.1 +defusedxml==0.7.1 +docstring-parser==0.16 +fastavro==1.9.7 +filelock==3.16.1 +fsspec==2024.9.0; python_version >= "3.12" +github3-py==4.0.1 +google-api-core[grpc]==2.19.2 +google-api-python-client==2.146.0 +google-auth==2.34.0 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.1 +google-cloud-bigquery[pandas]==3.25.0 +google-cloud-core==2.4.1 +google-cloud-dataproc==5.12.0 +google-cloud-storage==2.18.2 +google-crc32c==1.6.0 +google-resumable-media==2.7.2 +googleapis-common-protos[grpc]==1.65.0 +greenlet==3.1.0; (platform_machine == "win32" or platform_machine == "WIN32" or platform_machine == "AMD64" or platform_machine == "amd64" or platform_machine == "x86_64" or platform_machine == "ppc64le" or platform_machine == "aarch64") and python_version < "3.13" +grpc-google-iam-v1==0.13.1 +grpcio==1.66.1 +grpcio-health-checking==1.62.3 +grpcio-status==1.62.2 +gspread==6.1.2 +httplib2==0.22.0 +humanfriendly==10.0 +idna==3.10 +importlib-metadata==6.11.0 +isodate==0.6.1 +jinja2==3.1.4 +joblib==1.4.2 +jsonschema==4.23.0 +jsonschema-specifications==2023.12.1 +kubernetes==30.1.0 +ldap3==2.9.1 +leather==0.4.0 +logbook==1.5.3 +mako==1.3.5 +markdown-it-py==3.0.0 +markupsafe==2.1.5 +mashumaro[msgpack]==3.13.1 +mdurl==0.1.2 +memoization==0.4.0 +minimal-snowplow-tracker==0.0.2 +more-itertools==10.5.0 +msgpack==1.0.8 +networkx==3.3 +numpy==2.1.1 +oauth2client==4.1.3 +oauthlib==3.2.2 +oracledb==2.4.1 +ordered-set==4.1.0 +orjson==3.10.7 +packaging==24.1 +pandas==2.2.2 +paramiko==3.5.0 +parsedatetime==2.6 +pathspec==0.12.1 +pendulum==3.0.0 +pex==2.19.1 +prompt-toolkit==3.0.36 +proto-plus==1.24.0 +protobuf==4.25.5 +psutil==6.0.0; platform_system == "Windows" +py-avro-schema==3.8.2 +pyarrow==17.0.0 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 +pycparser==2.22 +pycryptodome==3.20.0 +pydantic==2.9.2 +pydantic-core==2.23.4 +pygments==2.18.0 +pyjwt[crypto]==2.9.0 +pynacl==1.5.0 +pyparsing==3.1.4; python_version > "3.0" +pyreadline3==3.5.3; sys_platform == "win32" and python_version >= "3.8" +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +python-slugify==8.0.4 +pytimeparse==1.1.8 +pytz==2024.2 +pywin32==306; platform_system == "Windows" +pyyaml==6.0.2 +questionary==2.0.1 +referencing==0.35.1 +requests==2.32.3 +requests-oauthlib==2.0.0 +rich==13.8.1 +rpds-py==0.20.0 +rsa==4.9 +scikit-learn==1.5.2 +scipy==1.14.1 +setuptools==75.1.0 +shellingham==1.5.4 +six==1.16.0 +soupsieve==2.6 +sqlalchemy==2.0.35 +sqlglot[rs]==25.21.3 +sqlglotrs==0.2.12 +sqlparse==0.5.1 +sshtunnel @ git+https://github.com/pahaz/sshtunnel.git@aa7342e55e49529fea020a09ba8f1eed33eb896f +structlog==24.4.0 +tableauserverclient==0.33 +tabulate==0.9.0 +tenacity==9.0.0 +text-unidecode==1.3 +threadpoolctl==3.5.0 +time-machine==2.15.0; implementation_name != "pypy" +tomli==2.0.1 +toposort==1.10 +tqdm==4.66.5 +typeguard==4.3.0 +typer==0.12.5 +typing-extensions==4.12.2 +tzdata==2024.1 +universal-pathlib==0.2.5; python_version >= "3.12" +uritemplate==4.1.1 +urllib3==2.2.2 +watchdog==5.0.2 +wcwidth==0.2.13 +websocket-client==1.8.0 +zipp==3.20.2 diff --git a/src/teamster/libraries/ssh/sshtunnel.py b/src/teamster/libraries/ssh/sshtunnel.py index 093bd5803e..d279d37a4c 100644 --- a/src/teamster/libraries/ssh/sshtunnel.py +++ b/src/teamster/libraries/ssh/sshtunnel.py @@ -1,4 +1,9 @@ -# trunk-ignore-all(pyright) +# forked from https://github.com/pahaz/sshtunnel/blob/master/sshtunnel.py + +# trunk-ignore-all(pyright/reportAttributeAccessIssue) +# trunk-ignore-all(pyright/reportGeneralTypeIssues) +# trunk-ignore-all(pyright/reportCallIssue) +# trunk-ignore-all(pyright/reportArgumentType) import getpass import logging @@ -15,21 +20,14 @@ from select import select import paramiko - -string_types = str -input_ = input - - -__version__ = "0.4.0" -__author__ = "pahaz" - +from paramiko import PKey, ProxyCommand #: Timeout (seconds) for transport socket (``socket.settimeout``) SSH_TIMEOUT = 0.1 # ``None`` may cause a block of transport thread + #: Timeout (seconds) for tunnel connection (open_channel timeout) TUNNEL_TIMEOUT = 10.0 -_DAEMON = True #: Use daemon threads in connections _DEPRECATIONS = { "ssh_address": "ssh_address_or_host", "ssh_host": "ssh_address_or_host", @@ -38,14 +36,8 @@ } # logging -DEFAULT_LOGLEVEL = logging.ERROR #: default level if no logger passed (ERROR) TRACE_LEVEL = 1 logging.addLevelName(TRACE_LEVEL, "TRACE") -DEFAULT_SSH_DIRECTORY = "~/.ssh" - -_StreamServer = ( - socketserver.UnixStreamServer if os.name == "posix" else socketserver.TCPServer -) #: Path of optional ssh configuration file DEFAULT_SSH_DIRECTORY = "~/.ssh" @@ -59,9 +51,7 @@ def check_host(host): - assert isinstance(host, string_types), "IP is not a string ({0})".format( - type(host).__name__ - ) + assert isinstance(host, str), "IP is not a string ({0})".format(type(host).__name__) def check_port(port): @@ -91,9 +81,7 @@ def check_address(address): if isinstance(address, tuple): check_host(address[0]) check_port(address[1]) - elif isinstance(address, string_types): - if os.name != "posix": - raise ValueError("Platform does not support UNIX domain sockets") + elif isinstance(address, str): if not ( os.path.exists(address) or os.access(os.path.dirname(address), os.W_OK) ): @@ -134,8 +122,9 @@ def check_addresses(address_list, is_remote=False): >>> check_addresses([('127.0.0.1', 22), ('127.0.0.1', 2222)]) """ - assert all(isinstance(x, (tuple, string_types)) for x in address_list) - if is_remote and any(isinstance(x, string_types) for x in address_list): + assert all(isinstance(x, (tuple, str)) for x in address_list) + + if is_remote and any(isinstance(x, str) for x in address_list): raise AssertionError("UNIX domain sockets not allowed for remote" "addresses") for address in address_list: @@ -177,12 +166,15 @@ def create_logger( :class:`logging.Logger` """ logger = logger or logging.getLogger("sshtunnel.SSHTunnelForwarder") + if not any(isinstance(x, logging.Handler) for x in logger.handlers): - logger.setLevel(loglevel or DEFAULT_LOGLEVEL) + logger.setLevel(loglevel or logging.ERROR) console_handler = logging.StreamHandler() + _add_handler( - logger, handler=console_handler, loglevel=loglevel or DEFAULT_LOGLEVEL + logger, handler=console_handler, loglevel=loglevel or logging.ERROR ) + if loglevel: # override if loglevel was set logger.setLevel(loglevel) for handler in logger.handlers: @@ -194,15 +186,18 @@ def create_logger( if capture_warnings and sys.version_info >= (2, 7): logging.captureWarnings(True) pywarnings = logging.getLogger("py.warnings") + pywarnings.handlers.extend(logger.handlers) + return logger -def _add_handler(logger, handler=None, loglevel=None): +def _add_handler(logger: logging.Logger, handler: logging.Handler, loglevel=None): """ Add a handler to an existing logging.Logger object """ - handler.setLevel(loglevel or DEFAULT_LOGLEVEL) + handler.setLevel(loglevel or logging.ERROR) + if handler.level <= logging.DEBUG: _fmt = ( "%(asctime)s| %(levelname)-4.3s|%(threadName)10.9s/" @@ -213,6 +208,7 @@ def _add_handler(logger, handler=None, loglevel=None): handler.setFormatter( logging.Formatter("%(asctime)s| %(levelname)-8s| %(message)s") ) + logger.addHandler(handler) @@ -221,17 +217,20 @@ def _check_paramiko_handlers(logger=None): Add a console handler for paramiko.transport's logger if not present """ paramiko_logger = logging.getLogger("paramiko.transport") + if not paramiko_logger.handlers: if logger: paramiko_logger.handlers = logger.handlers else: console_handler = logging.StreamHandler() + console_handler.setFormatter( logging.Formatter( "%(asctime)s | %(levelname)-8s| PARAMIKO: " "%(lineno)03d@%(module)-10s| %(message)s" ) ) + paramiko_logger.addHandler(console_handler) @@ -284,21 +283,25 @@ class HandlerSSHTunnelForwarderError(BaseSSHTunnelForwarderError): class _ForwardHandler(socketserver.BaseRequestHandler): """Base handler for tunnel connections""" + logger: logging.Logger + ssh_transport: paramiko.Transport + remote_address = None - ssh_transport = None - logger = None info = None def _redirect(self, chan): while chan.active: rqst, _, _ = select([self.request, chan], [], [], 5) + if self.request in rqst: data = self.request.recv(16384) + if not data: self.logger.log( TRACE_LEVEL, ">>> OUT {0} recv empty data >>>".format(self.info) ) break + if self.logger.isEnabledFor(TRACE_LEVEL): self.logger.log( TRACE_LEVEL, @@ -306,7 +309,9 @@ def _redirect(self, chan): self.info, self.remote_address, hexlify(data) ), ) + chan.sendall(data) + if chan in rqst: # else if not chan.recv_ready(): self.logger.log( @@ -314,23 +319,28 @@ def _redirect(self, chan): "<<< IN {0} recv is not ready <<<".format(self.info), ) break + data = chan.recv(16384) + if self.logger.isEnabledFor(TRACE_LEVEL): hex_data = hexlify(data) self.logger.log( TRACE_LEVEL, "<<< IN {0} recv: {1} <<<".format(self.info, hex_data), ) + self.request.sendall(data) def handle(self): - uid = generate_random_string(5) self.info = "#{0} <-- {1}".format( - uid, self.client_address or self.server.local_address + generate_random_string(5), self.client_address or self.server.local_address ) + src_address = self.request.getpeername() + if not isinstance(src_address, tuple): src_address = ("dummy", 12345) + try: chan = self.ssh_transport.open_channel( kind="direct-tcpip", @@ -340,12 +350,16 @@ def handle(self): ) except Exception as e: # pragma: no cover msg_tupe = "ssh " if isinstance(e, paramiko.SSHException) else "" + exc_msg = "open new channel {0}error: {1}".format(msg_tupe, e) + log_msg = "{0} {1}".format(self.info, exc_msg) + self.logger.log(TRACE_LEVEL, log_msg) raise HandlerSSHTunnelForwarderError(exc_msg) from e self.logger.log(TRACE_LEVEL, "{0} connected".format(self.info)) + try: self._redirect(chan) except socket.error: @@ -379,15 +393,18 @@ def handle_error(self, request, client_address): (exc_class, exc, tb) = sys.exc_info() local_side = request.getsockname() remote_side = self.remote_address + self.logger.error( "Could not establish connection from local {0} " "to remote {1} side of the tunnel: {2}".format(local_side, remote_side, exc) ) + try: self.tunnel_ok.put(False, block=False, timeout=0.1) except queue.Full: # wait untill tunnel_ok.get is called pass + except exc: self.logger.error("unexpected internal error: {0}".format(exc)) @@ -405,14 +422,17 @@ def local_port(self): @property def remote_address(self): + # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address @property def remote_host(self): + # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address[0] @property def remote_port(self): + # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address[1] @@ -423,19 +443,21 @@ class _ThreadingForwardServer(socketserver.ThreadingMixIn, _ForwardServer): # If True, cleanly stop threads created by ThreadingMixIn when quitting # This value is overrides by SSHTunnelForwarder.daemon_forward_servers - daemon_threads = _DAEMON + daemon_threads = True -class _StreamForwardServer(_StreamServer): +class _StreamForwardServer(socketserver.UnixStreamServer): """ Serve over domain sockets (does not work on Windows) """ def __init__(self, *args, **kwargs): logger = kwargs.pop("logger", None) + self.logger = logger or create_logger() self.tunnel_ok = queue.Queue(1) - _StreamServer.__init__(self, *args, **kwargs) + + socketserver.UnixStreamServer.__init__(self, *args, **kwargs) @property def local_address(self): @@ -451,14 +473,17 @@ def local_port(self): @property def remote_address(self): + # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address @property def remote_host(self): + # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address[0] @property def remote_port(self): + # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address[1] @@ -469,7 +494,7 @@ class _ThreadingStreamForwardServer(socketserver.ThreadingMixIn, _StreamForwardS # If True, cleanly stop threads created by ThreadingMixIn when quitting # This value is overrides by SSHTunnelForwarder.daemon_forward_servers - daemon_threads = _DAEMON + daemon_threads = True class SSHTunnelForwarder(object): @@ -732,116 +757,9 @@ class SSHTunnelForwarder(object): skip_tunnel_checkup = True # This option affects the `ForwardServer` and all his threads - daemon_forward_servers = _DAEMON #: flag tunnel threads in daemon mode + daemon_forward_servers = True #: flag tunnel threads in daemon mode # This option affect only `Transport` thread - daemon_transport = _DAEMON #: flag SSH transport thread in daemon mode - - def __init__( - self, - ssh_address_or_host=None, - ssh_config_file=SSH_CONFIG_FILE, - ssh_host_key=None, - ssh_password=None, - ssh_pkey=None, - ssh_private_key_password=None, - ssh_proxy=None, - ssh_proxy_enabled=True, - ssh_username=None, - local_bind_address=None, - local_bind_addresses=None, - logger=None, - mute_exceptions=False, - remote_bind_address=None, - remote_bind_addresses=None, - set_keepalive=5.0, - threaded=True, # old version False - compression=None, - allow_agent=True, # look for keys from an SSH agent - host_pkey_directories=None, # look for keys in ~/.ssh - *args, - **kwargs, # for backwards compatibility - ): - self.logger = logger or create_logger() - - self.ssh_host_key = ssh_host_key - self.set_keepalive = set_keepalive - self._server_list = [] # reset server list - self.tunnel_is_up = {} # handle tunnel status - self._threaded = threaded - self.is_alive = False - - # Check if deprecated arguments ssh_address or ssh_host were used - for deprecated_argument in ["ssh_address", "ssh_host"]: - ssh_address_or_host = self._process_deprecated( - ssh_address_or_host, deprecated_argument, kwargs - ) - # other deprecated arguments - ssh_pkey = self._process_deprecated(ssh_pkey, "ssh_private_key", kwargs) - - self._raise_fwd_exc = ( - self._process_deprecated( - None, "raise_exception_if_any_forwarder_have_a_problem", kwargs - ) - or not mute_exceptions - ) - - if isinstance(ssh_address_or_host, tuple): - check_address(ssh_address_or_host) - (ssh_host, ssh_port) = ssh_address_or_host - else: - ssh_host = ssh_address_or_host - ssh_port = kwargs.pop("ssh_port", None) - - if kwargs: - raise ValueError("Unknown arguments: {0}".format(kwargs)) - - # remote binds - self._remote_binds = self._get_binds( - remote_bind_address, remote_bind_addresses, is_remote=True - ) - # local binds - self._local_binds = self._get_binds(local_bind_address, local_bind_addresses) - self._local_binds = self._consolidate_binds( - self._local_binds, self._remote_binds - ) - - ( - self.ssh_host, - self.ssh_username, - ssh_pkey, # still needs to go through _consolidate_auth - self.ssh_port, - self.ssh_proxy, - self.compression, - ) = self._read_ssh_config( - ssh_host, - ssh_config_file, - ssh_username, - ssh_pkey, - ssh_port, - ssh_proxy if ssh_proxy_enabled else None, - compression, - self.logger, - ) - - (self.ssh_password, self.ssh_pkeys) = self._consolidate_auth( - ssh_password=ssh_password, - ssh_pkey=ssh_pkey, - ssh_pkey_password=ssh_private_key_password, - allow_agent=allow_agent, - host_pkey_directories=host_pkey_directories, - logger=self.logger, - ) - - check_host(self.ssh_host) - check_port(self.ssh_port) - - self.logger.info( - "Connecting to gateway: {0}:{1} as user '{2}'".format( - self.ssh_host, self.ssh_port, self.ssh_username - ) - ) - - self.logger.debug("Concurrent connections allowed: {0}".format(self._threaded)) + daemon_transport = True #: flag SSH transport thread in daemon mode def local_is_up(self, target): """ @@ -892,7 +810,7 @@ def _check_tunnel(self, _srv): self.tunnel_is_up[_srv.local_address] = True return self.logger.info("Checking tunnel to: {0}".format(_srv.remote_address)) - if isinstance(_srv.local_address, string_types): # UNIX stream + if isinstance(_srv.local_address, str): # UNIX stream s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) else: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -945,7 +863,7 @@ def _make_ssh_forward_server(self, remote_address, local_bind_address): try: forward_maker_class = ( self._make_stream_ssh_forward_server_class - if isinstance(local_bind_address, string_types) + if isinstance(local_bind_address, str) else self._make_ssh_forward_server_class ) _Server = forward_maker_class(remote_address) @@ -978,6 +896,112 @@ def _make_ssh_forward_server(self, remote_address, local_bind_address): ), ) + def __init__( + self, + ssh_address_or_host=None, + ssh_config_file=SSH_CONFIG_FILE, + ssh_host_key=None, + ssh_password=None, + ssh_pkey=None, + ssh_private_key_password=None, + ssh_proxy=None, + ssh_proxy_enabled=True, + ssh_username=None, + local_bind_address=None, + local_bind_addresses=None, + logger=None, + mute_exceptions=False, + remote_bind_address=None, + remote_bind_addresses=None, + set_keepalive=5.0, + threaded=True, # old version False + compression=None, + allow_agent=True, # look for keys from an SSH agent + host_pkey_directories=None, # look for keys in ~/.ssh + *args, + **kwargs, # for backwards compatibility + ): + self.logger = logger or create_logger() + + self.ssh_host_key = ssh_host_key + self.set_keepalive = set_keepalive + self._server_list = [] # reset server list + self.tunnel_is_up = {} # handle tunnel status + self._threaded = threaded + self.is_alive = False + # Check if deprecated arguments ssh_address or ssh_host were used + for deprecated_argument in ["ssh_address", "ssh_host"]: + ssh_address_or_host = self._process_deprecated( + ssh_address_or_host, deprecated_argument, kwargs + ) + # other deprecated arguments + ssh_pkey = self._process_deprecated(ssh_pkey, "ssh_private_key", kwargs) + + self._raise_fwd_exc = ( + self._process_deprecated( + None, "raise_exception_if_any_forwarder_have_a_problem", kwargs + ) + or not mute_exceptions + ) + + if isinstance(ssh_address_or_host, tuple): + check_address(ssh_address_or_host) + (ssh_host, ssh_port) = ssh_address_or_host + else: + ssh_host = ssh_address_or_host + ssh_port = kwargs.pop("ssh_port", None) + + if kwargs: + raise ValueError("Unknown arguments: {0}".format(kwargs)) + + # remote binds + self._remote_binds = self._get_binds( + remote_bind_address, remote_bind_addresses, is_remote=True + ) + # local binds + self._local_binds = self._get_binds(local_bind_address, local_bind_addresses) + self._local_binds = self._consolidate_binds( + self._local_binds, self._remote_binds + ) + + ( + self.ssh_host, + self.ssh_username, + ssh_pkey, # still needs to go through _consolidate_auth + self.ssh_port, + self.ssh_proxy, + self.compression, + ) = self._read_ssh_config( + ssh_host, + ssh_config_file, + ssh_username, + ssh_pkey, + ssh_port, + ssh_proxy if ssh_proxy_enabled else None, + compression, + self.logger, + ) + + (self.ssh_password, self.ssh_pkeys) = self._consolidate_auth( + ssh_password=ssh_password, + ssh_pkey=ssh_pkey, + ssh_pkey_password=ssh_private_key_password, + allow_agent=allow_agent, + host_pkey_directories=host_pkey_directories, + logger=self.logger, + ) + + check_host(self.ssh_host) + check_port(self.ssh_port) + + self.logger.info( + "Connecting to gateway: {0}:{1} as user '{2}'".format( + self.ssh_host, self.ssh_port, self.ssh_username + ) + ) + + self.logger.debug("Concurrent connections allowed: {0}".format(self._threaded)) + @staticmethod def _read_ssh_config( ssh_host, @@ -1001,6 +1025,7 @@ def _read_ssh_config( # Try to read SSH_CONFIG_FILE try: # open the ssh config file + with open(os.path.expanduser(ssh_config_file), "r") as f: ssh_config.parse(f) # looks for information for the destination system @@ -1086,14 +1111,17 @@ def get_keys(logger=None, host_pkey_directories=None, allow_agent=False): "dsa": paramiko.DSSKey, "ecdsa": paramiko.ECDSAKey, } + if hasattr(paramiko, "Ed25519Key"): # NOQA: new in paramiko>=2.2: http://docs.paramiko.org/en/stable/api/keys.html#module-paramiko.ed25519key paramiko_key_types["ed25519"] = paramiko.Ed25519Key + for directory in host_pkey_directories: for keytype in paramiko_key_types.keys(): ssh_pkey_expanded = os.path.expanduser( os.path.join(directory, "id_{}".format(keytype)) ) + try: if os.path.isfile(ssh_pkey_expanded): ssh_pkey = SSHTunnelForwarder.read_private_key_file( @@ -1101,6 +1129,7 @@ def get_keys(logger=None, host_pkey_directories=None, allow_agent=False): logger=logger, key_type=paramiko_key_types[keytype], ) + if ssh_pkey: keys.append(ssh_pkey) except OSError as exc: @@ -1112,6 +1141,7 @@ def get_keys(logger=None, host_pkey_directories=None, allow_agent=False): ) if logger: logger.info("{0} key(s) loaded".format(len(keys))) + return keys @staticmethod @@ -1152,7 +1182,7 @@ def _consolidate_auth( allow_agent=allow_agent, ) - if isinstance(ssh_pkey, string_types): + if isinstance(ssh_pkey, str): ssh_pkey_expanded = os.path.expanduser(ssh_pkey) if os.path.exists(ssh_pkey_expanded): ssh_pkey = SSHTunnelForwarder.read_private_key_file( @@ -1162,7 +1192,8 @@ def _consolidate_auth( ) elif logger: logger.warning("Private key file not found: {0}".format(ssh_pkey)) - if isinstance(ssh_pkey, paramiko.pkey.PKey): + + if isinstance(ssh_pkey, PKey): ssh_loaded_pkeys.insert(0, ssh_pkey) if not ssh_password and not ssh_loaded_pkeys: @@ -1178,24 +1209,31 @@ def _raise(self, exception=BaseSSHTunnelForwarderError, reason=None): def _get_transport(self): """Return the SSH transport to the remote gateway""" if self.ssh_proxy: - if isinstance(self.ssh_proxy, paramiko.proxy.ProxyCommand): + if isinstance(self.ssh_proxy, ProxyCommand): proxy_repr = repr(self.ssh_proxy.cmd[1]) else: proxy_repr = repr(self.ssh_proxy) + self.logger.debug("Connecting via proxy: {0}".format(proxy_repr)) _socket = self.ssh_proxy else: _socket = (self.ssh_host, self.ssh_port) + if isinstance(_socket, socket.socket): _socket.settimeout(SSH_TIMEOUT) _socket.connect((self.ssh_host, self.ssh_port)) + transport = paramiko.Transport(_socket) + sock = transport.sock + if isinstance(sock, socket.socket): sock.settimeout(SSH_TIMEOUT) - transport.set_keepalive(self.set_keepalive) + + transport.set_keepalive(int(self.set_keepalive)) transport.use_compression(compress=self.compression) transport.daemon = self.daemon_transport + # try to solve https://github.com/paramiko/paramiko/issues/1181 # transport.banner_timeout = 200 if isinstance(sock, socket.socket): @@ -1215,26 +1253,23 @@ def _create_tunnels(self): if not self.is_active: try: self._connect_to_gateway() - except socket.gaierror as e: # raised by paramiko.Transport - self.logger.error( - msg=f"Could not resolve IP address for {self.ssh_host}, aborting!" + except socket.gaierror: # raised by paramiko.Transport + msg = "Could not resolve IP address for {0}, aborting!".format( + self.ssh_host ) - raise e + self.logger.error(msg) + return except (paramiko.SSHException, socket.error) as e: - self.logger.error( - msg=( - "Could not connect to gateway " - f"{self.ssh_host}:{self.ssh_port} : {e.args[0]}" - ) - ) - raise e - + template = "Could not connect to gateway {0}:{1} : {2}" + msg = template.format(self.ssh_host, self.ssh_port, e.args[0]) + self.logger.error(msg) + return for rem, loc in zip(self._remote_binds, self._local_binds): try: self._make_ssh_forward_server(rem, loc) except BaseSSHTunnelForwarderError as e: - self.logger.error(f"Problem setting SSH Forwarder up: {e.value}") - raise e + msg = "Problem setting SSH Forwarder up: {0}".format(e.value) + self.logger.error(msg) @staticmethod def _get_binds(bind_address, bind_addresses, is_remote=False): @@ -1255,18 +1290,14 @@ def _get_binds(bind_address, bind_addresses, is_remote=False): "'{0}_bind_addresses' arguments. Use one of " "them.".format(addr_kind) ) - if bind_address: bind_addresses = [bind_address] - if not is_remote: # Add random port if missing in local bind for i, local_bind in enumerate(bind_addresses): if isinstance(local_bind, tuple) and len(local_bind) == 1: bind_addresses[i] = (local_bind[0], 0) - check_addresses(bind_addresses, is_remote) - return bind_addresses @staticmethod @@ -1280,10 +1311,10 @@ def _process_deprecated(attrib, deprecated_attrib, kwargs): ) if deprecated_attrib in kwargs: warnings.warn( - message="'{0}' is DEPRECATED use '{1}' instead".format( + "'{0}' is DEPRECATED use '{1}' instead".format( deprecated_attrib, _DEPRECATIONS[deprecated_attrib] ), - category=DeprecationWarning, + DeprecationWarning, stacklevel=2, ) if attrib: @@ -1348,29 +1379,22 @@ def start(self): if self.is_alive: self.logger.warning("Already started!") return - self._create_tunnels() - if not self.is_active: self._raise( BaseSSHTunnelForwarderError, reason="Could not establish session to SSH gateway", ) - for _srv in self._server_list: thread = threading.Thread( target=self._serve_forever_wrapper, args=(_srv,), name="Srv-{0}".format(address_to_str(_srv.local_port)), ) - thread.daemon = self.daemon_forward_servers thread.start() - self._check_tunnel(_srv) - self.is_alive = any(self.tunnel_is_up.values()) - if not self.is_alive: self._raise( HandlerSSHTunnelForwarderError, @@ -1409,7 +1433,6 @@ def stop(self, force=False): ", ".join((address_to_str(k.local_address) for k in self._server_list)) or "None" ) - self.logger.debug("Listening tunnels: " + opened_address_text) self._stop_transport(force=force) self._server_list = [] # reset server list @@ -1442,9 +1465,10 @@ def _connect_to_gateway(self): ) if self._transport.is_alive: return - except paramiko.AuthenticationException: + except paramiko.AuthenticationException as e: self.logger.debug("Authentication error") self._stop_transport() + raise e if self.ssh_password: # avoid conflict using both pass and pkey self.logger.debug( @@ -1461,9 +1485,10 @@ def _connect_to_gateway(self): ) if self._transport.is_alive: return - except paramiko.AuthenticationException: + except paramiko.AuthenticationException as e: self.logger.debug("Authentication error") self._stop_transport() + raise e self.logger.error("Could not open connection to gateway") @@ -1499,6 +1524,7 @@ def _stop_transport(self, force=False): for _srv in self._server_list: status = "up" if self.tunnel_is_up[_srv.local_address] else "down" + self.logger.info( "Shutting down tunnel: {0} <> {1} ({2})".format( address_to_str(_srv.local_address), @@ -1506,8 +1532,10 @@ def _stop_transport(self, force=False): status, ) ) + _srv.shutdown() _srv.server_close() + # clean up the UNIX domain socket if we're using one if isinstance(_srv, _StreamForwardServer): try: @@ -1518,11 +1546,14 @@ def _stop_transport(self, force=False): _srv.local_address, repr(e) ) ) + self.is_alive = False + if self.is_active: self.logger.info("Closing ssh transport") self._transport.close() self._transport.stop_thread() + self.logger.debug("Transport is closed") @property From 3bca7dbc2ff91212d0b262914928968d0d75c81d Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 18 Sep 2024 23:01:08 +0000 Subject: [PATCH 4/4] fix --- pdm.lock | 9 +- pyproject.toml | 1 - requirements.txt | 2 +- src/teamster/libraries/ssh/resources.py | 51 ++++- src/teamster/libraries/ssh/sshtunnel.py | 244 ++++++++++++++---------- 5 files changed, 201 insertions(+), 106 deletions(-) diff --git a/pdm.lock b/pdm.lock index 5571508303..e0597d8e24 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:aa47064038fa484701b55e9c2e38f38f06c6db3d24edf4b4ff85612482e64701" +content_hash = "sha256:6e9918eacbdc2a671a475e04067e90a39719f3dea733072e14d48a3f6417fc20" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -2830,14 +2830,15 @@ files = [ [[package]] name = "sshtunnel" version = "0.4.0" -git = "https://github.com/pahaz/sshtunnel.git" -ref = "master" -revision = "aa7342e55e49529fea020a09ba8f1eed33eb896f" summary = "Pure python SSH tunnels" groups = ["default"] dependencies = [ "paramiko>=2.7.2", ] +files = [ + {file = "sshtunnel-0.4.0-py2.py3-none-any.whl", hash = "sha256:98e54c26f726ab8bd42b47a3a21fca5c3e60f58956f0f70de2fb8ab0046d0606"}, + {file = "sshtunnel-0.4.0.tar.gz", hash = "sha256:e7cb0ea774db81bf91844db22de72a40aae8f7b0f9bb9ba0f666d474ef6bf9fc"}, +] [[package]] name = "starlette" diff --git a/pyproject.toml b/pyproject.toml index 8c0be6b898..4b5632056f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ dependencies = [ "tableauserverclient>=0.25", "tenacity>=8.2.3", "pendulum>=3.0.0", - "sshtunnel @ git+https://github.com/pahaz/sshtunnel.git@master", ] requires-python = ">=3.12,<3.13" license = { text = "GPL-3.0-or-later" } diff --git a/requirements.txt b/requirements.txt index eea077ab55..47a5be4817 100644 --- a/requirements.txt +++ b/requirements.txt @@ -141,7 +141,7 @@ sqlalchemy==2.0.35 sqlglot[rs]==25.21.3 sqlglotrs==0.2.12 sqlparse==0.5.1 -sshtunnel @ git+https://github.com/pahaz/sshtunnel.git@aa7342e55e49529fea020a09ba8f1eed33eb896f +sshtunnel==0.4.0 structlog==24.4.0 tableauserverclient==0.33 tabulate==0.9.0 diff --git a/src/teamster/libraries/ssh/resources.py b/src/teamster/libraries/ssh/resources.py index 89cfbada6d..c940972b71 100644 --- a/src/teamster/libraries/ssh/resources.py +++ b/src/teamster/libraries/ssh/resources.py @@ -4,7 +4,8 @@ from dagster import _check from dagster_ssh import SSHResource as DagsterSSHResource from paramiko import AutoAddPolicy, SFTPAttributes, SFTPClient, SSHClient -from sshtunnel import SSHTunnelForwarder + +from teamster.libraries.ssh.sshtunnel import SSHTunnelForwarder class SSHResource(DagsterSSHResource): @@ -75,15 +76,57 @@ def get_connection(self) -> SSHClient: return client + # trunk-ignore(pyright/reportIncompatibleMethodOverride) def get_tunnel( self, remote_port, remote_host="localhost", local_port=None ) -> SSHTunnelForwarder: if self.tunnel_remote_host is not None: remote_host = self.tunnel_remote_host - return super().get_tunnel( - remote_port=remote_port, remote_host=remote_host, local_port=local_port - ) + _check.int_param(obj=remote_port, param_name="remote_port") + _check.str_param(obj=remote_host, param_name="remote_host") + _check.opt_int_param(obj=local_port, param_name="local_port") + + if local_port is not None: + local_bind_address = ("localhost", local_port) + else: + local_bind_address = ("localhost",) + + # Will prefer key string if specified, otherwise use the key file + if self._key_obj and self.key_file: + self.log.warning( + "SSHResource: key_string and key_file both specified as config. " + "Using key_string." + ) + + pkey = self._key_obj if self._key_obj else self.key_file + + if self.password and self.password.strip(): + client = SSHTunnelForwarder( + self.remote_host, + ssh_port=self.remote_port, + ssh_username=self.username, + ssh_password=self.password, + ssh_pkey=pkey, + ssh_proxy=self._host_proxy, + local_bind_address=local_bind_address, + remote_bind_address=(remote_host, remote_port), + logger=self._logger, + ) + else: + client = SSHTunnelForwarder( + self.remote_host, + ssh_port=self.remote_port, + ssh_username=self.username, + ssh_pkey=pkey, + ssh_proxy=self._host_proxy, + local_bind_address=local_bind_address, + remote_bind_address=(remote_host, remote_port), + host_pkey_directories=[], + logger=self._logger, + ) + + return client def listdir_attr_r( self, remote_dir: str = ".", exclude_dirs: list[str] | None = None diff --git a/src/teamster/libraries/ssh/sshtunnel.py b/src/teamster/libraries/ssh/sshtunnel.py index d279d37a4c..c9ef97cc8e 100644 --- a/src/teamster/libraries/ssh/sshtunnel.py +++ b/src/teamster/libraries/ssh/sshtunnel.py @@ -1,17 +1,24 @@ -# forked from https://github.com/pahaz/sshtunnel/blob/master/sshtunnel.py +# https://github.com/pahaz/sshtunnel/blob/master/sshtunnel.py +# trunk-ignore-all(pyright) -# trunk-ignore-all(pyright/reportAttributeAccessIssue) -# trunk-ignore-all(pyright/reportGeneralTypeIssues) -# trunk-ignore-all(pyright/reportCallIssue) -# trunk-ignore-all(pyright/reportArgumentType) +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +*sshtunnel* - Initiate SSH tunnels via a remote gateway. + +``sshtunnel`` works by opening a port forwarding SSH connection in the +background, using threads. + +The connection(s) are closed when explicitly calling the +:meth:`SSHTunnelForwarder.stop` method or using it as a context. + +""" import getpass import logging import os -import queue import random import socket -import socketserver import string import sys import threading @@ -20,14 +27,32 @@ from select import select import paramiko -from paramiko import PKey, ProxyCommand + +if sys.version_info[0] < 3: # pragma: no cover + import Queue as queue + import SocketServer as socketserver + + string_types = (basestring,) # noqa + input_ = raw_input # noqa +else: # pragma: no cover + import queue + import socketserver + + string_types = str + input_ = input + + +__version__ = "0.4.0" +__author__ = "pahaz" + #: Timeout (seconds) for transport socket (``socket.settimeout``) SSH_TIMEOUT = 0.1 # ``None`` may cause a block of transport thread - #: Timeout (seconds) for tunnel connection (open_channel timeout) TUNNEL_TIMEOUT = 10.0 +_DAEMON = True #: Use daemon threads in connections +_CONNECTION_COUNTER = 1 _DEPRECATIONS = { "ssh_address": "ssh_address_or_host", "ssh_host": "ssh_address_or_host", @@ -36,8 +61,14 @@ } # logging +DEFAULT_LOGLEVEL = logging.ERROR #: default level if no logger passed (ERROR) TRACE_LEVEL = 1 logging.addLevelName(TRACE_LEVEL, "TRACE") +DEFAULT_SSH_DIRECTORY = "~/.ssh" + +_StreamServer = ( + socketserver.UnixStreamServer if os.name == "posix" else socketserver.TCPServer +) #: Path of optional ssh configuration file DEFAULT_SSH_DIRECTORY = "~/.ssh" @@ -51,7 +82,9 @@ def check_host(host): - assert isinstance(host, str), "IP is not a string ({0})".format(type(host).__name__) + assert isinstance(host, string_types), "IP is not a string ({0})".format( + type(host).__name__ + ) def check_port(port): @@ -81,7 +114,9 @@ def check_address(address): if isinstance(address, tuple): check_host(address[0]) check_port(address[1]) - elif isinstance(address, str): + elif isinstance(address, string_types): + if os.name != "posix": + raise ValueError("Platform does not support UNIX domain sockets") if not ( os.path.exists(address) or os.access(os.path.dirname(address), os.W_OK) ): @@ -122,9 +157,8 @@ def check_addresses(address_list, is_remote=False): >>> check_addresses([('127.0.0.1', 22), ('127.0.0.1', 2222)]) """ - assert all(isinstance(x, (tuple, str)) for x in address_list) - - if is_remote and any(isinstance(x, str) for x in address_list): + assert all(isinstance(x, (tuple, string_types)) for x in address_list) + if is_remote and any(isinstance(x, string_types) for x in address_list): raise AssertionError("UNIX domain sockets not allowed for remote" "addresses") for address in address_list: @@ -166,15 +200,12 @@ def create_logger( :class:`logging.Logger` """ logger = logger or logging.getLogger("sshtunnel.SSHTunnelForwarder") - if not any(isinstance(x, logging.Handler) for x in logger.handlers): - logger.setLevel(loglevel or logging.ERROR) + logger.setLevel(loglevel or DEFAULT_LOGLEVEL) console_handler = logging.StreamHandler() - _add_handler( - logger, handler=console_handler, loglevel=loglevel or logging.ERROR + logger, handler=console_handler, loglevel=loglevel or DEFAULT_LOGLEVEL ) - if loglevel: # override if loglevel was set logger.setLevel(loglevel) for handler in logger.handlers: @@ -186,18 +217,16 @@ def create_logger( if capture_warnings and sys.version_info >= (2, 7): logging.captureWarnings(True) pywarnings = logging.getLogger("py.warnings") - pywarnings.handlers.extend(logger.handlers) - return logger -def _add_handler(logger: logging.Logger, handler: logging.Handler, loglevel=None): +def _add_handler(logger, handler=None, loglevel=None): """ Add a handler to an existing logging.Logger object """ - handler.setLevel(loglevel or logging.ERROR) + handler.setLevel(loglevel or DEFAULT_LOGLEVEL) if handler.level <= logging.DEBUG: _fmt = ( "%(asctime)s| %(levelname)-4.3s|%(threadName)10.9s/" @@ -208,7 +237,6 @@ def _add_handler(logger: logging.Logger, handler: logging.Handler, loglevel=None handler.setFormatter( logging.Formatter("%(asctime)s| %(levelname)-8s| %(message)s") ) - logger.addHandler(handler) @@ -217,20 +245,17 @@ def _check_paramiko_handlers(logger=None): Add a console handler for paramiko.transport's logger if not present """ paramiko_logger = logging.getLogger("paramiko.transport") - if not paramiko_logger.handlers: if logger: paramiko_logger.handlers = logger.handlers else: console_handler = logging.StreamHandler() - console_handler.setFormatter( logging.Formatter( "%(asctime)s | %(levelname)-8s| PARAMIKO: " "%(lineno)03d@%(module)-10s| %(message)s" ) ) - paramiko_logger.addHandler(console_handler) @@ -283,25 +308,21 @@ class HandlerSSHTunnelForwarderError(BaseSSHTunnelForwarderError): class _ForwardHandler(socketserver.BaseRequestHandler): """Base handler for tunnel connections""" - logger: logging.Logger - ssh_transport: paramiko.Transport - remote_address = None + ssh_transport = None + logger = None info = None def _redirect(self, chan): while chan.active: rqst, _, _ = select([self.request, chan], [], [], 5) - if self.request in rqst: data = self.request.recv(16384) - if not data: self.logger.log( TRACE_LEVEL, ">>> OUT {0} recv empty data >>>".format(self.info) ) break - if self.logger.isEnabledFor(TRACE_LEVEL): self.logger.log( TRACE_LEVEL, @@ -309,9 +330,7 @@ def _redirect(self, chan): self.info, self.remote_address, hexlify(data) ), ) - chan.sendall(data) - if chan in rqst: # else if not chan.recv_ready(): self.logger.log( @@ -319,28 +338,23 @@ def _redirect(self, chan): "<<< IN {0} recv is not ready <<<".format(self.info), ) break - data = chan.recv(16384) - if self.logger.isEnabledFor(TRACE_LEVEL): hex_data = hexlify(data) self.logger.log( TRACE_LEVEL, "<<< IN {0} recv: {1} <<<".format(self.info, hex_data), ) - self.request.sendall(data) def handle(self): + uid = generate_random_string(5) self.info = "#{0} <-- {1}".format( - generate_random_string(5), self.client_address or self.server.local_address + uid, self.client_address or self.server.local_address ) - src_address = self.request.getpeername() - if not isinstance(src_address, tuple): src_address = ("dummy", 12345) - try: chan = self.ssh_transport.open_channel( kind="direct-tcpip", @@ -350,16 +364,12 @@ def handle(self): ) except Exception as e: # pragma: no cover msg_tupe = "ssh " if isinstance(e, paramiko.SSHException) else "" - exc_msg = "open new channel {0}error: {1}".format(msg_tupe, e) - log_msg = "{0} {1}".format(self.info, exc_msg) - self.logger.log(TRACE_LEVEL, log_msg) raise HandlerSSHTunnelForwarderError(exc_msg) from e self.logger.log(TRACE_LEVEL, "{0} connected".format(self.info)) - try: self._redirect(chan) except socket.error: @@ -393,18 +403,15 @@ def handle_error(self, request, client_address): (exc_class, exc, tb) = sys.exc_info() local_side = request.getsockname() remote_side = self.remote_address - self.logger.error( "Could not establish connection from local {0} " "to remote {1} side of the tunnel: {2}".format(local_side, remote_side, exc) ) - try: self.tunnel_ok.put(False, block=False, timeout=0.1) except queue.Full: # wait untill tunnel_ok.get is called pass - except exc: self.logger.error("unexpected internal error: {0}".format(exc)) @@ -422,17 +429,14 @@ def local_port(self): @property def remote_address(self): - # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address @property def remote_host(self): - # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address[0] @property def remote_port(self): - # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address[1] @@ -443,21 +447,19 @@ class _ThreadingForwardServer(socketserver.ThreadingMixIn, _ForwardServer): # If True, cleanly stop threads created by ThreadingMixIn when quitting # This value is overrides by SSHTunnelForwarder.daemon_forward_servers - daemon_threads = True + daemon_threads = _DAEMON -class _StreamForwardServer(socketserver.UnixStreamServer): +class _StreamForwardServer(_StreamServer): """ Serve over domain sockets (does not work on Windows) """ def __init__(self, *args, **kwargs): logger = kwargs.pop("logger", None) - self.logger = logger or create_logger() self.tunnel_ok = queue.Queue(1) - - socketserver.UnixStreamServer.__init__(self, *args, **kwargs) + _StreamServer.__init__(self, *args, **kwargs) @property def local_address(self): @@ -473,17 +475,14 @@ def local_port(self): @property def remote_address(self): - # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address @property def remote_host(self): - # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address[0] @property def remote_port(self): - # trunk-ignore(pyright/reportFunctionMemberAccess) return self.RequestHandlerClass.remote_address[1] @@ -494,7 +493,7 @@ class _ThreadingStreamForwardServer(socketserver.ThreadingMixIn, _StreamForwardS # If True, cleanly stop threads created by ThreadingMixIn when quitting # This value is overrides by SSHTunnelForwarder.daemon_forward_servers - daemon_threads = True + daemon_threads = _DAEMON class SSHTunnelForwarder(object): @@ -757,9 +756,9 @@ class SSHTunnelForwarder(object): skip_tunnel_checkup = True # This option affects the `ForwardServer` and all his threads - daemon_forward_servers = True #: flag tunnel threads in daemon mode + daemon_forward_servers = _DAEMON #: flag tunnel threads in daemon mode # This option affect only `Transport` thread - daemon_transport = True #: flag SSH transport thread in daemon mode + daemon_transport = _DAEMON #: flag SSH transport thread in daemon mode def local_is_up(self, target): """ @@ -810,7 +809,7 @@ def _check_tunnel(self, _srv): self.tunnel_is_up[_srv.local_address] = True return self.logger.info("Checking tunnel to: {0}".format(_srv.remote_address)) - if isinstance(_srv.local_address, str): # UNIX stream + if isinstance(_srv.local_address, string_types): # UNIX stream s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) else: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -863,7 +862,7 @@ def _make_ssh_forward_server(self, remote_address, local_bind_address): try: forward_maker_class = ( self._make_stream_ssh_forward_server_class - if isinstance(local_bind_address, str) + if isinstance(local_bind_address, string_types) else self._make_ssh_forward_server_class ) _Server = forward_maker_class(remote_address) @@ -1025,7 +1024,6 @@ def _read_ssh_config( # Try to read SSH_CONFIG_FILE try: # open the ssh config file - with open(os.path.expanduser(ssh_config_file), "r") as f: ssh_config.parse(f) # looks for information for the destination system @@ -1111,17 +1109,14 @@ def get_keys(logger=None, host_pkey_directories=None, allow_agent=False): "dsa": paramiko.DSSKey, "ecdsa": paramiko.ECDSAKey, } - if hasattr(paramiko, "Ed25519Key"): # NOQA: new in paramiko>=2.2: http://docs.paramiko.org/en/stable/api/keys.html#module-paramiko.ed25519key paramiko_key_types["ed25519"] = paramiko.Ed25519Key - for directory in host_pkey_directories: for keytype in paramiko_key_types.keys(): ssh_pkey_expanded = os.path.expanduser( os.path.join(directory, "id_{}".format(keytype)) ) - try: if os.path.isfile(ssh_pkey_expanded): ssh_pkey = SSHTunnelForwarder.read_private_key_file( @@ -1129,7 +1124,6 @@ def get_keys(logger=None, host_pkey_directories=None, allow_agent=False): logger=logger, key_type=paramiko_key_types[keytype], ) - if ssh_pkey: keys.append(ssh_pkey) except OSError as exc: @@ -1141,7 +1135,6 @@ def get_keys(logger=None, host_pkey_directories=None, allow_agent=False): ) if logger: logger.info("{0} key(s) loaded".format(len(keys))) - return keys @staticmethod @@ -1182,7 +1175,7 @@ def _consolidate_auth( allow_agent=allow_agent, ) - if isinstance(ssh_pkey, str): + if isinstance(ssh_pkey, string_types): ssh_pkey_expanded = os.path.expanduser(ssh_pkey) if os.path.exists(ssh_pkey_expanded): ssh_pkey = SSHTunnelForwarder.read_private_key_file( @@ -1192,8 +1185,7 @@ def _consolidate_auth( ) elif logger: logger.warning("Private key file not found: {0}".format(ssh_pkey)) - - if isinstance(ssh_pkey, PKey): + if isinstance(ssh_pkey, paramiko.pkey.PKey): ssh_loaded_pkeys.insert(0, ssh_pkey) if not ssh_password and not ssh_loaded_pkeys: @@ -1209,31 +1201,24 @@ def _raise(self, exception=BaseSSHTunnelForwarderError, reason=None): def _get_transport(self): """Return the SSH transport to the remote gateway""" if self.ssh_proxy: - if isinstance(self.ssh_proxy, ProxyCommand): + if isinstance(self.ssh_proxy, paramiko.proxy.ProxyCommand): proxy_repr = repr(self.ssh_proxy.cmd[1]) else: proxy_repr = repr(self.ssh_proxy) - self.logger.debug("Connecting via proxy: {0}".format(proxy_repr)) _socket = self.ssh_proxy else: _socket = (self.ssh_host, self.ssh_port) - if isinstance(_socket, socket.socket): _socket.settimeout(SSH_TIMEOUT) _socket.connect((self.ssh_host, self.ssh_port)) - transport = paramiko.Transport(_socket) - sock = transport.sock - if isinstance(sock, socket.socket): sock.settimeout(SSH_TIMEOUT) - - transport.set_keepalive(int(self.set_keepalive)) + transport.set_keepalive(self.set_keepalive) transport.use_compression(compress=self.compression) transport.daemon = self.daemon_transport - # try to solve https://github.com/paramiko/paramiko/issues/1181 # transport.banner_timeout = 200 if isinstance(sock, socket.socket): @@ -1315,7 +1300,7 @@ def _process_deprecated(attrib, deprecated_attrib, kwargs): deprecated_attrib, _DEPRECATIONS[deprecated_attrib] ), DeprecationWarning, - stacklevel=2, + stacklevel=TRACE_LEVEL, ) if attrib: raise ValueError( @@ -1465,10 +1450,9 @@ def _connect_to_gateway(self): ) if self._transport.is_alive: return - except paramiko.AuthenticationException as e: + except paramiko.AuthenticationException: self.logger.debug("Authentication error") self._stop_transport() - raise e if self.ssh_password: # avoid conflict using both pass and pkey self.logger.debug( @@ -1485,10 +1469,9 @@ def _connect_to_gateway(self): ) if self._transport.is_alive: return - except paramiko.AuthenticationException as e: + except paramiko.AuthenticationException: self.logger.debug("Authentication error") self._stop_transport() - raise e self.logger.error("Could not open connection to gateway") @@ -1515,16 +1498,13 @@ def _stop_transport(self, force=False): self._check_is_started() except (BaseSSHTunnelForwarderError, HandlerSSHTunnelForwarderError) as e: self.logger.warning(e) - if force and self.is_active: # don't wait connections self.logger.info("Closing ssh transport") self._transport.close() self._transport.stop_thread() - for _srv in self._server_list: status = "up" if self.tunnel_is_up[_srv.local_address] else "down" - self.logger.info( "Shutting down tunnel: {0} <> {1} ({2})".format( address_to_str(_srv.local_address), @@ -1532,10 +1512,8 @@ def _stop_transport(self, force=False): status, ) ) - _srv.shutdown() _srv.server_close() - # clean up the UNIX domain socket if we're using one if isinstance(_srv, _StreamForwardServer): try: @@ -1546,14 +1524,11 @@ def _stop_transport(self, force=False): _srv.local_address, repr(e) ) ) - self.is_alive = False - if self.is_active: self.logger.info("Closing ssh transport") self._transport.close() self._transport.stop_thread() - self.logger.debug("Transport is closed") @property @@ -1714,3 +1689,80 @@ def __del__(self): "the garbage collector! Running .stop(force=True)" ) self.stop(force=True) + + +def open_tunnel(*args, **kwargs): + """ + Open an SSH Tunnel, wrapper for :class:`SSHTunnelForwarder` + + Arguments: + destination (Optional[tuple]): + SSH server's IP address and port in the format + (``ssh_address``, ``ssh_port``) + + Keyword Arguments: + debug_level (Optional[int or str]): + log level for :class:`logging.Logger` instance, i.e. ``DEBUG`` + + skip_tunnel_checkup (boolean): + Enable/disable the local side check and populate + :attr:`~SSHTunnelForwarder.tunnel_is_up` + + Default: True + + .. versionadded:: 0.1.0 + + .. note:: + A value of ``debug_level`` set to 1 == ``TRACE`` enables tracing mode + .. note:: + See :class:`SSHTunnelForwarder` for keyword arguments + + **Example**:: + + from sshtunnel import open_tunnel + + with open_tunnel(SERVER, + ssh_username=SSH_USER, + ssh_port=22, + ssh_password=SSH_PASSWORD, + remote_bind_address=(REMOTE_HOST, REMOTE_PORT), + local_bind_address=('', LOCAL_PORT)) as server: + def do_something(port): + pass + + print("LOCAL PORTS:", server.local_bind_port) + + do_something(server.local_bind_port) + """ + # Attach a console handler to the logger or create one if not passed + loglevel = kwargs.pop("debug_level", None) + logger = kwargs.get("logger", None) or create_logger(loglevel=loglevel) + kwargs["logger"] = logger + + ssh_address_or_host = kwargs.pop("ssh_address_or_host", None) + # Check if deprecated arguments ssh_address or ssh_host were used + for deprecated_argument in ["ssh_address", "ssh_host"]: + ssh_address_or_host = SSHTunnelForwarder._process_deprecated( + ssh_address_or_host, deprecated_argument, kwargs + ) + + ssh_port = kwargs.pop("ssh_port", 22) + skip_tunnel_checkup = kwargs.pop("skip_tunnel_checkup", True) + block_on_close = kwargs.pop("block_on_close", None) + if block_on_close: + warnings.warn( + "'block_on_close' is DEPRECATED. You should use either" + " .stop() or .stop(force=True), depends on what you do" + " with the active connections. This option has no" + " affect since 0.3.0", + DeprecationWarning, + stacklevel=TRACE_LEVEL, + ) + if not args: + if isinstance(ssh_address_or_host, tuple): + args = (ssh_address_or_host,) + else: + args = ((ssh_address_or_host, ssh_port),) + forwarder = SSHTunnelForwarder(*args, **kwargs) + forwarder.skip_tunnel_checkup = skip_tunnel_checkup + return forwarder