forked from wheever/ProxHTTPSProxyMII
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathProxHTTPSProxy.py
executable file
·407 lines (357 loc) · 18.5 KB
/
ProxHTTPSProxy.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"A Proxomitron Helper Program"
_name = 'ProxHTTPSProxyMII'
__author__ = 'phoenix'
__version__ = 'v1.4'
CONFIG = "config.ini"
CA_CERTS = "cacert.pem"
import os
import time
import configparser
import fnmatch
import logging
import threading
import ssl
import urllib3
from urllib3.contrib.socks import SOCKSProxyManager
#https://urllib3.readthedocs.org/en/latest/security.html#insecurerequestwarning
urllib3.disable_warnings()
from socketserver import ThreadingMixIn
from http.server import HTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse
from ProxyTool import ProxyRequestHandler, get_cert, counter
from colorama import init, Fore, Back, Style
init(autoreset=True)
class LoadConfig:
def __init__(self, configfile):
self.config = configparser.ConfigParser(allow_no_value=True,
inline_comment_prefixes=('#',))
self.config.read(configfile)
self.PROXADDR = self.config['GENERAL'].get('ProxAddr')
self.FRONTPORT = int(self.config['GENERAL'].get('FrontPort'))
self.REARPORT = int(self.config['GENERAL'].get('RearPort'))
self.DEFAULTPROXY = self.config['GENERAL'].get('DefaultProxy')
self.LOGLEVEL = self.config['GENERAL'].get('LogLevel')
class ConnectionPools:
"""
self.pools is a list of {'proxy': 'http://127.0.0.1:8080',
'pool': urllib3.ProxyManager() object,
'patterns': ['ab.com', 'bc.net', ...]}
self.getpool() is a method that returns pool based on host matching
"""
# Windows default CA certificates are incomplete
# See: http://bugs.python.org/issue20916
# cacert.pem sources:
# - http://curl.haxx.se/docs/caextract.html
# - http://certifi.io/en/latest/
# ssl_version="TLSv1" to specific version
sslparams = dict(cert_reqs="REQUIRED", ca_certs=CA_CERTS)
# IE: http://support2.microsoft.com/kb/181050/en-us
# Firefox about:config
# network.http.connection-timeout 90
# network.http.response.timeout 300
timeout = urllib3.util.timeout.Timeout(connect=90.0, read=300.0)
def __init__(self, config):
self.file = config
self.file_timestamp = os.path.getmtime(config)
self.loadConfig()
def loadConfig(self):
# self.conf has to be inited each time for reloading
self.conf = configparser.ConfigParser(allow_no_value=True, delimiters=('=',),
inline_comment_prefixes=('#',))
self.conf.read(self.file)
self.pools = []
proxy_sections = [section for section in self.conf.sections()
if section.startswith('PROXY')]
for section in proxy_sections:
proxy = section.split()[1]
self.pools.append(dict(proxy=proxy,
pool=self.setProxyPool(proxy),
patterns=list(self.conf[section].keys())))
default_proxy = self.conf['GENERAL'].get('DefaultProxy')
default_pool = (self.setProxyPool(default_proxy) if default_proxy else
[urllib3.PoolManager(num_pools=10, maxsize=8, timeout=self.timeout, **self.sslparams),
urllib3.PoolManager(num_pools=10, maxsize=8, timeout=self.timeout)])
self.pools.append({'proxy': default_proxy, 'pool': default_pool, 'patterns': '*'})
self.noverifylist = list(self.conf['SSL No-Verify'].keys())
self.blacklist = list(self.conf['BLACKLIST'].keys())
self.sslpasslist = list(self.conf['SSL Pass-Thru'].keys())
self.bypasslist = list(self.conf['BYPASS URL'].keys())
def reloadConfig(self):
while True:
mtime = os.path.getmtime(self.file)
if mtime > self.file_timestamp:
self.file_timestamp = mtime
self.loadConfig()
logger.info(Fore.RED + Style.BRIGHT
+ "*" * 20 + " CONFIG RELOADED " + "*" * 20)
time.sleep(1)
def getpool(self, host, httpmode=False):
noverify = True if httpmode or any((fnmatch.fnmatch(host, pattern) for pattern in self.noverifylist)) else False
for pool in self.pools:
if any((fnmatch.fnmatch(host, pattern) for pattern in pool['patterns'])):
return pool['proxy'], pool['pool'][noverify], noverify
def setProxyPool(self, proxy):
scheme = proxy.split(':')[0]
if scheme in ('http', 'https'):
ProxyManager = urllib3.ProxyManager
elif scheme in ('socks4', 'socks5'):
ProxyManager = SOCKSProxyManager
else:
print("Wrong Proxy Format: " + proxy)
print("Proxy should start with http/https/socks4/socks5 .")
input()
raise SystemExit
# maxsize is the max. number of connections to the same server
return [ProxyManager(proxy, num_pools=10, maxsize=8, timeout=self.timeout, **self.sslparams),
ProxyManager(proxy, num_pools=10, maxsize=8, timeout=self.timeout)]
class FrontServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
pass
class RearServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
pass
class FrontRequestHandler(ProxyRequestHandler):
"""
Sit between the client and Proxomitron
Convert https request to http
"""
server_version = "%s FrontProxy/%s" % (_name, __version__)
def do_CONNECT(self):
"Descrypt https request and dispatch to http handler"
# request line: CONNECT www.example.com:443 HTTP/1.1
self.host, self.port = self.path.split(":")
self.proxy, self.pool, self.noverify = pools.getpool(self.host)
if any((fnmatch.fnmatch(self.host, pattern) for pattern in pools.blacklist)):
# BLACK LIST
self.deny_request()
logger.info("%03d " % self.reqNum + Fore.CYAN + 'Denied by blacklist: %s' % self.host)
elif any((fnmatch.fnmatch(self.host, pattern) for pattern in pools.sslpasslist)):
# SSL Pass-Thru
if self.proxy and self.proxy.startswith('https'):
self.forward_to_https_proxy()
elif self.proxy and self.proxy.startswith('socks5'):
self.forward_to_socks5_proxy()
else:
self.tunnel_traffic()
# Upstream server or proxy of the tunnel is closed explictly, so we close the local connection too
self.close_connection = 1
else:
# SSL MITM
self.wfile.write(("HTTP/1.1 200 Connection established\r\n" +
"Proxy-agent: %s\r\n" % self.version_string() +
"\r\n").encode('ascii'))
commonname = '.' + self.host.partition('.')[-1] if self.host.count('.') >= 2 else self.host
dummycert = get_cert(commonname)
# set a flag for do_METHOD
self.ssltunnel = True
ssl_sock = ssl.wrap_socket(self.connection, keyfile=dummycert, certfile=dummycert, server_side=True)
# Ref: Lib/socketserver.py#StreamRequestHandler.setup()
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.rbufsize)
self.wfile = self.connection.makefile('wb', self.wbufsize)
# dispatch to do_METHOD()
self.handle_one_request()
def do_METHOD(self):
"Forward request to Proxomitron"
counter.increment_and_set(self, 'reqNum')
if self.ssltunnel:
# https request
host = self.host if self.port == '443' else "%s:%s" % (self.host, self.port)
url = "https://%s%s" % (host, self.path)
self.bypass = any((fnmatch.fnmatch(url, pattern) for pattern in pools.bypasslist))
if not self.bypass:
url = "http://%s%s" % (host, self.path)
# Tag the request so Proxomitron can recognize it
self.headers["Tagged"] = self.version_string() + ":%d" % self.reqNum
else:
# http request
self.host = urlparse(self.path).hostname
if any((fnmatch.fnmatch(self.host, pattern) for pattern in pools.blacklist)):
# BLACK LIST
self.deny_request()
logger.info("%03d " % self.reqNum + Fore.CYAN + 'Denied by blacklist: %s' % self.host)
return
host = urlparse(self.path).netloc
self.proxy, self.pool, self.noverify = pools.getpool(self.host, httpmode=True)
self.bypass = any((fnmatch.fnmatch('http://' + host + urlparse(self.path).path, pattern) for pattern in pools.bypasslist))
url = self.path
self.url = url
pool = self.pool if self.bypass else proxpool
data_length = self.headers.get("Content-Length")
self.postdata = self.rfile.read(int(data_length)) if data_length and int(data_length) > 0 else None
if self.command == "POST" and "Content-Length" not in self.headers:
buffer = self.rfile.read()
if buffer:
logger.warning("%03d " % self.reqNum + Fore.RED +
'POST w/o "Content-Length" header (Bytes: %d | Transfer-Encoding: %s | HTTPS: %s',
len(buffer), "Transfer-Encoding" in self.headers, self.ssltunnel)
# Remove hop-by-hop headers
self.purge_headers(self.headers)
r = None
# Below code in connectionpool.py expect the headers to has a copy() and update() method
# That's why we can't use self.headers directly when call pool.urlopen()
#
# Merge the proxy headers. Only do this in HTTP. We have to copy the
# headers dict so we can safely change it without those changes being
# reflected in anyone else's copy.
# if self.scheme == 'http':
# headers = headers.copy()
# headers.update(self.proxy_headers)
headers = urllib3._collections.HTTPHeaderDict(self.headers)
try:
# Sometimes 302 redirect would fail with "BadStatusLine" exception, and IE11 doesn't restart the request.
# retries=1 instead of retries=False fixes it.
#! Retry may cause the requests with the same reqNum appear in the log window
r = pool.urlopen(self.command, url, body=self.postdata, headers=headers,
retries=1, redirect=False, preload_content=False, decode_content=False)
if not self.ssltunnel:
if self.bypass:
prefix = '[BP]' if self.proxy else '[BD]'
else:
prefix = '[D]'
if self.command in ("GET", "HEAD"):
logger.info("%03d " % self.reqNum + Fore.MAGENTA + '%s "%s %s" %s %s' %
(prefix, self.command, url, r.status, r.getheader('Content-Length', '-')))
else:
logger.info("%03d " % self.reqNum + Fore.MAGENTA + '%s "%s %s %s" %s %s' %
(prefix, self.command, url, data_length, r.status, r.getheader('Content-Length', '-')))
self.send_response_only(r.status, r.reason)
# HTTPResponse.msg is easier to handle than urllib3._collections.HTTPHeaderDict
r.headers = r._original_response.msg
self.purge_write_headers(r.headers)
if self.command == 'HEAD' or r.status in (100, 101, 204, 304) or r.getheader("Content-Length") == '0':
written = None
else:
written = self.stream_to_client(r)
if "Content-Length" not in r.headers and 'Transfer-Encoding' not in r.headers:
self.close_connection = 1
# Intend to catch regular http and bypass http/https requests exceptions
# Regular https request exceptions should be handled by rear server
except urllib3.exceptions.TimeoutError as e:
self.sendout_error(url, 504, message="Timeout", explain=e)
logger.warning("%03d " % self.reqNum + Fore.YELLOW + '[F] %s on "%s %s"', e, self.command, url)
except (urllib3.exceptions.HTTPError,) as e:
self.sendout_error(url, 502, message="HTTPError", explain=e)
logger.warning("%03d " % self.reqNum + Fore.YELLOW + '[F] %s on "%s %s"', e, self.command, url)
finally:
if r:
# Release the connection back into the pool
r.release_conn()
do_GET = do_POST = do_HEAD = do_PUT = do_DELETE = do_OPTIONS = do_METHOD
class RearRequestHandler(ProxyRequestHandler):
"""
Supposed to be the parent proxy for Proxomitron for tagged requests
Convert http request to https
"""
server_version = "%s RearProxy/%s" % (_name, __version__)
def do_METHOD(self):
"Convert http request to https"
if self.headers.get("Tagged") and self.headers["Tagged"].startswith(_name):
self.reqNum = int(self.headers["Tagged"].split(":")[1])
# Remove the tag
del self.headers["Tagged"]
else:
self.sendout_error(self.path, 400,
explain="The proxy setting of the client is misconfigured.\n\n" +
"Please set the HTTPS proxy port to %s " % config.FRONTPORT +
"and check the Docs for other settings.")
logger.error(Fore.RED + Style.BRIGHT + "[Misconfigured HTTPS proxy port] " + self.path)
return
# request line: GET http://somehost.com/path?attr=value HTTP/1.1
url = "https" + self.path[4:]
self.host = urlparse(self.path).hostname
proxy, pool, noverify = pools.getpool(self.host)
prefix = '[P]' if proxy else '[D]'
data_length = self.headers.get("Content-Length")
self.postdata = self.rfile.read(int(data_length)) if data_length else None
self.purge_headers(self.headers)
r = None
# Below code in connectionpool.py expect the headers to has a copy() and update() method
# That's why we can't use self.headers directly when call pool.urlopen()
#
# Merge the proxy headers. Only do this in HTTP. We have to copy the
# headers dict so we can safely change it without those changes being
# reflected in anyone else's copy.
# if self.scheme == 'http':
# headers = headers.copy()
# headers.update(self.proxy_headers)
headers = urllib3._collections.HTTPHeaderDict(self.headers)
try:
r = pool.urlopen(self.command, url, body=self.postdata, headers=headers,
retries=1, redirect=False, preload_content=False, decode_content=False)
if proxy:
logger.debug('Using Proxy - %s' % proxy)
color = Fore.RED if noverify else Fore.GREEN
if self.command in ("GET", "HEAD"):
logger.info("%03d " % self.reqNum + color + '%s "%s %s" %s %s' %
(prefix, self.command, url, r.status, r.getheader('Content-Length', '-')))
else:
logger.info("%03d " % self.reqNum + color + '%s "%s %s %s" %s %s' %
(prefix, self.command, url, data_length, r.status, r.getheader('Content-Length', '-')))
self.send_response_only(r.status, r.reason)
# HTTPResponse.msg is easier to handle than urllib3._collections.HTTPHeaderDict
r.headers = r._original_response.msg
self.purge_write_headers(r.headers)
if self.command == 'HEAD' or r.status in (100, 101, 204, 304) or r.getheader("Content-Length") == '0':
written = None
else:
written = self.stream_to_client(r)
if "Content-Length" not in r.headers and 'Transfer-Encoding' not in r.headers:
self.close_connection = 1
except urllib3.exceptions.SSLError as e:
self.sendout_error(url, 417, message="SSL Certificate Failed", explain=e)
logger.error("%03d " % self.reqNum + Fore.RED + Style.BRIGHT + "[SSL Certificate Error] " + url)
except urllib3.exceptions.TimeoutError as e:
self.sendout_error(url, 504, message="Timeout", explain=e)
logger.warning("%03d " % self.reqNum + Fore.YELLOW + '[R]%s "%s %s" %s', prefix, self.command, url, e)
except (urllib3.exceptions.HTTPError,) as e:
self.sendout_error(url, 502, message="HTTPError", explain=e)
logger.warning("%03d " % self.reqNum + Fore.YELLOW + '[R]%s "%s %s" %s', prefix, self.command, url, e)
finally:
if r:
# Release the connection back into the pool
r.release_conn()
do_GET = do_POST = do_HEAD = do_PUT = do_DELETE = do_OPTIONS = do_METHOD
"""
#Information#
* Python default ciphers: http://bugs.python.org/issue20995
* SSL Cipher Suite Details of Your Browser: https://cc.dcsec.uni-hannover.de/
* https://wiki.mozilla.org/Security/Server_Side_TLS
"""
try:
if os.name == 'nt':
import ctypes
ctypes.windll.kernel32.SetConsoleTitleW('%s %s' % (_name, __version__))
config = LoadConfig(CONFIG)
logger = logging.getLogger(__name__)
logger.setLevel(getattr(logging, config.LOGLEVEL, logging.INFO))
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(message)s', datefmt='[%H:%M]')
handler.setFormatter(formatter)
logger.addHandler(handler)
pools = ConnectionPools(CONFIG)
proxpool = urllib3.ProxyManager(config.PROXADDR, num_pools=10, maxsize=8,
# A little longer than timeout of rear pool
# to avoid trigger front server exception handler
timeout=urllib3.util.timeout.Timeout(connect=90.0, read=310.0))
frontserver = FrontServer(('', config.FRONTPORT), FrontRequestHandler)
rearserver = RearServer(('', config.REARPORT), RearRequestHandler)
for worker in (frontserver.serve_forever, rearserver.serve_forever,
pools.reloadConfig):
thread = threading.Thread(target=worker)
thread.daemon = True
thread.start()
print("=" * 76)
print('%s %s (urllib3/%s)' % (_name, __version__, urllib3.__version__))
print()
print(' FrontServer : localhost:%s' % config.FRONTPORT)
print(' RearServer : localhost:%s' % config.REARPORT)
print(' ParentServer : %s' % config.DEFAULTPROXY)
print(' Proxomitron : ' + config.PROXADDR)
print("=" * 76)
while True:
time.sleep(1)
except KeyboardInterrupt:
print("Quitting...")