From 358bc561f4c5d865081493894040b47bd99376fa Mon Sep 17 00:00:00 2001 From: Eshin Kunishima Date: Thu, 3 Sep 2020 17:56:51 +0900 Subject: [PATCH 1/3] Add support for authenticated proxies --- .travis.s3cfg | 2 ++ S3/Config.py | 2 ++ S3/ConnMan.py | 7 ++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.s3cfg b/.travis.s3cfg index 91797afe2..b1b8e89d8 100644 --- a/.travis.s3cfg +++ b/.travis.s3cfg @@ -49,6 +49,8 @@ preserve_attrs = True progress_meter = True proxy_host = proxy_port = 0 +proxy_username = +proxy_password = put_continue = False recursive = False recv_chunk = 65536 diff --git a/S3/Config.py b/S3/Config.py index adbab442d..641b15a7e 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -153,6 +153,8 @@ class Config(object): acl_revokes = [] proxy_host = u"" proxy_port = 3128 + proxy_username = u"" + proxy_password = u"" encrypt = False dry_run = False add_encoding_exts = u"" diff --git a/S3/ConnMan.py b/S3/ConnMan.py index 2ec0cafa5..b75b2b839 100644 --- a/S3/ConnMan.py +++ b/S3/ConnMan.py @@ -15,6 +15,7 @@ from .Custom_httplib27 import httplib import ssl +from base64 import b64encode from logging import debug from threading import Semaphore from time import time @@ -229,11 +230,15 @@ def __init__(self, id, hostname, ssl, cfg): self.c = httplib.HTTPConnection(self.hostname, self.port) debug(u'non-proxied HTTPConnection(%s, %s)', self.hostname, self.port) else: + headers = {} + if cfg.proxy_username and cfg.proxy_password: + headers['Proxy-Authorization'] = 'Basic ' + \ + b64encode(('%s:%s' % (cfg.proxy_username, cfg.proxy_password)).encode('utf-8')).decode('ascii') if ssl: self.c = http_connection._https_connection(cfg.proxy_host, cfg.proxy_port) debug(u'proxied HTTPSConnection(%s, %s)', cfg.proxy_host, cfg.proxy_port) port = self.port and self.port or 443 - self.c.set_tunnel(self.hostname, port) + self.c.set_tunnel(self.hostname, port, headers) debug(u'tunnel to %s, %s', self.hostname, port) else: self.c = httplib.HTTPConnection(cfg.proxy_host, cfg.proxy_port) From 136856282e55a60974d8d980fee17588788218ed Mon Sep 17 00:00:00 2001 From: Eshin Kunishima Date: Tue, 15 Sep 2020 18:15:52 +0900 Subject: [PATCH 2/3] retrieve credential from proxy_host --- .travis.s3cfg | 4 +--- S3/Config.py | 2 -- S3/ConnMan.py | 14 +++++++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.s3cfg b/.travis.s3cfg index b1b8e89d8..f59baa153 100644 --- a/.travis.s3cfg +++ b/.travis.s3cfg @@ -47,10 +47,8 @@ multipart_chunk_size_mb = 15 multipart_max_chunks = 10000 preserve_attrs = True progress_meter = True -proxy_host = +proxy_host = proxy_port = 0 -proxy_username = -proxy_password = put_continue = False recursive = False recv_chunk = 65536 diff --git a/S3/Config.py b/S3/Config.py index 641b15a7e..adbab442d 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -153,8 +153,6 @@ class Config(object): acl_revokes = [] proxy_host = u"" proxy_port = 3128 - proxy_username = u"" - proxy_password = u"" encrypt = False dry_run = False add_encoding_exts = u"" diff --git a/S3/ConnMan.py b/S3/ConnMan.py index b75b2b839..cebbece54 100644 --- a/S3/ConnMan.py +++ b/S3/ConnMan.py @@ -27,7 +27,7 @@ from .Config import Config from .Exceptions import ParameterError, S3SSLCertificateError -from .Utils import getBucketFromHostname +from .Utils import getBucketFromHostname, deunicodise_s, unicodise_s @@ -231,17 +231,21 @@ def __init__(self, id, hostname, ssl, cfg): debug(u'non-proxied HTTPConnection(%s, %s)', self.hostname, self.port) else: headers = {} - if cfg.proxy_username and cfg.proxy_password: + proxy_hostname = cfg.proxy_host + if '@' in cfg.proxy_host: + credential, proxy_hostname = cfg.proxy_host.split('@') + # FIXME: Following line can't handle username or password including colon + proxy_username, proxy_password = credential.split(':') headers['Proxy-Authorization'] = 'Basic ' + \ - b64encode(('%s:%s' % (cfg.proxy_username, cfg.proxy_password)).encode('utf-8')).decode('ascii') + unicodise_s(b64encode(deunicodise_s(('%s:%s' % (proxy_username, proxy_password))))) if ssl: - self.c = http_connection._https_connection(cfg.proxy_host, cfg.proxy_port) + self.c = http_connection._https_connection(proxy_hostname, cfg.proxy_port) debug(u'proxied HTTPSConnection(%s, %s)', cfg.proxy_host, cfg.proxy_port) port = self.port and self.port or 443 self.c.set_tunnel(self.hostname, port, headers) debug(u'tunnel to %s, %s', self.hostname, port) else: - self.c = httplib.HTTPConnection(cfg.proxy_host, cfg.proxy_port) + self.c = httplib.HTTPConnection(proxy_hostname, cfg.proxy_port) debug(u'proxied HTTPConnection(%s, %s)', cfg.proxy_host, cfg.proxy_port) # No tunnel here for the moment From 31980971c5a028a0010a1b749d225a6f49d139c7 Mon Sep 17 00:00:00 2001 From: Eshin Kunishima Date: Tue, 15 Sep 2020 18:20:17 +0900 Subject: [PATCH 3/3] add http(|s)_proxy env support to config tool --- s3cmd | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/s3cmd b/s3cmd index f79557d85..132e37d8b 100755 --- a/s3cmd +++ b/s3cmd @@ -68,6 +68,12 @@ except ImportError: # python2 fallback code from distutils.spawn import find_executable as which +try: + # python 3 support + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + def output(message): sys.stdout.write(message + "\n") @@ -2342,21 +2348,13 @@ def run_configure(config_file, args): ("gpg_passphrase", "Encryption password", "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3"), ("gpg_command", "Path to GPG program"), ("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP, and can only be proxied with Python 2.7 or newer"), - ("proxy_host", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't connect to S3 directly"), + ("proxy_host", "username:password@example.com", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't connect to S3 directly"), ("proxy_port", "HTTP Proxy server port"), ] ## Option-specfic defaults if getattr(cfg, "gpg_command") == "": setattr(cfg, "gpg_command", which("gpg")) - if getattr(cfg, "proxy_host") == "" and os.getenv("http_proxy"): - autodetected_encoding = locale.getpreferredencoding() or "UTF-8" - re_match=re.match(r"(http://)?([^:]+):(\d+)", - unicodise_s(os.getenv("http_proxy"), autodetected_encoding)) - if re_match: - setattr(cfg, "proxy_host", re_match.groups()[1]) - setattr(cfg, "proxy_port", re_match.groups()[2]) - try: # Support for python3 # raw_input only exists in py2 and was renamed to input in py3 @@ -2372,12 +2370,16 @@ def run_configure(config_file, args): for option in options: prompt = option[1] ## Option-specific handling - if option[0] == 'proxy_host' and getattr(cfg, 'use_https') == True and sys.hexversion < 0x02070000: - setattr(cfg, option[0], "") - continue - if option[0] == 'proxy_port' and getattr(cfg, 'proxy_host') == "": - setattr(cfg, option[0], 0) - continue + if option[0] == "proxy_host": + autodetected_encoding = locale.getpreferredencoding() or "UTF-8" + key = "https_proxy" if getattr(cfg, 'use_https') else "http_proxy" + url = urlparse(unicodise_s(os.getenv(key), autodetected_encoding)) + if url.port is None: + port = 80 if url.scheme == "http" else 443 # httplib supports HTTP(S) proxy only + else: + port = url.port + setattr(cfg, "proxy_host", "%s:%s@%s" % (url.username, url.password, url.hostname)) + setattr(cfg, "proxy_port", port) try: val = getattr(cfg, option[0])