From d2f0e5c222c9db7c2790a48e0fe5d2566cdd96ea Mon Sep 17 00:00:00 2001 From: Rob Glew Date: Wed, 30 Mar 2016 14:37:50 -0500 Subject: [PATCH] Version 0.2.10 --- README.md | 14 + pappyproxy/__init__.py | 2 +- pappyproxy/http.py | 19 +- pappyproxy/plugin.py | 15 +- pappyproxy/plugins/filter.py | 9 +- pappyproxy/plugins/macrocmds.py | 49 +- pappyproxy/plugins/misc.py | 5 + pappyproxy/plugins/tagcmds.py | 6 +- pappyproxy/plugins/view.py | 31 +- pappyproxy/proxy.py | 91 +- pappyproxy/session.py | 115 ++- pappyproxy/templates/macro.py.template | 5 +- pappyproxy/tests/test_mangle.py | 209 ----- pappyproxy/tests/test_proxy.py | 1119 +++++++----------------- pappyproxy/tests/testutil.py | 4 +- pappyproxy/util.py | 22 +- setup.py | 1 + 17 files changed, 640 insertions(+), 1076 deletions(-) delete mode 100644 pappyproxy/tests/test_mangle.py diff --git a/README.md b/README.md index 53f0933..6fbdb9a 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,8 @@ Configuration for each project is done in the `config.json` file. The file is a | `debug_dir` (optional) | Where connection debug info should be stored. If not present, debug info is not saved to a file. | | `cert_dir` | Where the CA cert and the private key for the CA cert are stored | | `proxy_listeners` | A list of dicts which describe which ports the proxy will listen on. Each item is a dict with "port" and "interface" values which determine which port and interface to listen on. For example, if port=8000 and the interface is 127.0.0.1, the proxy will only accept connections from localhost on port 8000. To accept connections from anywhere, set the interface to 0.0.0.0. | +| `socks_proxy` | A dictionary with details on how to connect to an upstream SOCKS proxy to send all in-scope requests through. See the secion on upstream SOCKS proxies for more information. | +| `http_proxy` | A dictionary with details on how to connect to an upstream http proxy to send all in-scope requests through. See the section on upstream http proxies for more information. | The following tokens will also be replaced with values: @@ -379,6 +381,11 @@ Some arguments can take multiple IDs for an argument. To pass multiple IDs to a * `viq 1,2,u3` View information about requests 1, 2, and the unmangled version of 3 * `gma foo 4,5,6` Generate a macro with definitions for requests 4, 5, and 6 +In addition, you can pass in a wildcard to include all in context requests. + +* `viq *` View information about all in-context requests +* `dump_response *` Dump the responses of all in-context requests (will overwrite duplicates) + Context ------- The context is a set of filters that define which requests are considered "active". Only requests in the current context are displayed with `ls`. By default, the context includes every single request that passes through the proxy. You can limit down the current context by applying filters. Filters apply rules such as "the response code must equal 500" or "the host must contain google.com". Once you apply one or more filters, only requests/responses which pass every active filter will be a part of the current context. @@ -1123,6 +1130,13 @@ Changelog --------- The boring part of the readme +* 0.2.10 + * Add wildcard support for requests that can take in multiple request ids + * Update dump_response to dump multiple requests at the same time + * More autocompleters (macro commands, field for filters) + * Add non-async function to get in-context request IDs. Now macros can scan over all in-context stuff and do things with them. + * Improve sessions to be used to maintain state with macros + * Bugfixes * 0.2.9 * Fix bugs/clean up some code * 0.2.8 diff --git a/pappyproxy/__init__.py b/pappyproxy/__init__.py index cd9b137..e913eb4 100644 --- a/pappyproxy/__init__.py +++ b/pappyproxy/__init__.py @@ -1 +1 @@ -__version__ = '0.2.9' +__version__ = '0.2.10' diff --git a/pappyproxy/http.py b/pappyproxy/http.py index 703ef2f..89ceb48 100644 --- a/pappyproxy/http.py +++ b/pappyproxy/http.py @@ -127,7 +127,7 @@ def repeatable_parse_qs(s): @crochet.wait_for(timeout=180.0) @defer.inlineCallbacks def request_by_id(reqid): - req = Request.load_request(str(reqid)) + req = yield Request.load_request(str(reqid)) defer.returnValue(req) ########## @@ -2133,7 +2133,9 @@ class Request(HTTPMessage): def submit_request(request, save_request=False, intercepting_macros={}, - stream_transport=None): + stream_transport=None, + _factory_string_transport=False, + _conn_info=None): """ submit_request(request, save_request=False, intercepting_macros={}, stream_transport=None) @@ -2152,6 +2154,9 @@ class Request(HTTPMessage): from .proxy import ProxyClientFactory, get_next_connection_id, get_endpoint from .pappy import session + from .tests.testutil import TLSStringTransport + + # _factory__string_transport, _conn_classes are only for unit tests. Do not use. factory = None if stream_transport is None: @@ -2164,6 +2169,16 @@ class Request(HTTPMessage): save_all=save_request, stream_response=True, return_transport=stream_transport) + + # Set up stuff for unit test if needed + if _factory_string_transport: + factory._use_string_transport = True + if _conn_info is not None: + # Pass factory back to unit test + _conn_info['factory'] = factory + factory._conn_info = _conn_info + + # Set up factory settings factory.intercepting_macros = intercepting_macros factory.connection_id = get_next_connection_id() factory.connect() diff --git a/pappyproxy/plugin.py b/pappyproxy/plugin.py index 4325d35..4b9fadc 100644 --- a/pappyproxy/plugin.py +++ b/pappyproxy/plugin.py @@ -10,7 +10,9 @@ import imp import os import pappyproxy import stat +import crochet +from twisted.internet import defer from .proxy import add_intercepting_macro as proxy_add_intercepting_macro from .proxy import remove_intercepting_macro as proxy_remove_intercepting_macro from .colors import Colors @@ -146,7 +148,7 @@ def req_history(num=-1, ids=None, include_unmangled=False): """ return pappyproxy.Request.cache.req_it(num=num, ids=ids, include_unmangled=include_unmangled) -def main_context_ids(n=-1): +def async_main_context_ids(n=-1): """ Returns a deferred that resolves into a list of up to ``n`` of the most recent requests in the main context. You can then use @@ -156,6 +158,17 @@ def main_context_ids(n=-1): """ return pappyproxy.pappy.main_context.get_reqs(n) +@crochet.wait_for(timeout=None) +@defer.inlineCallbacks +def main_context_ids(*args, **kwargs): + """ + Same as :func:`pappyproxy.plugin.async_main_context_ids` but can be called + from macros and other non-async only functions. Cannot be called in async + functions. + """ + ret = yield async_main_context_ids(*args, **kwargs) + defer.returnValue(ret) + def run_cmd(cmd): """ Run a command as if you typed it into the console. Try and use diff --git a/pappyproxy/plugins/filter.py b/pappyproxy/plugins/filter.py index 4d8d185..104be5f 100644 --- a/pappyproxy/plugins/filter.py +++ b/pappyproxy/plugins/filter.py @@ -1,7 +1,7 @@ import crochet import pappyproxy -from pappyproxy.util import PappyException, confirm +from pappyproxy.util import PappyException, confirm, autocomplete_startswith from pappyproxy.http import Request from twisted.internet import defer @@ -40,6 +40,11 @@ class BuiltinFilters(object): return pappyproxy.context.Filter(BuiltinFilters._filters[name][1]) +def complete_filtercmd(text, line, begidx, endidx): + strs = [k for k, v in pappyproxy.context.Filter._filter_functions.iteritems()] + strs += [k for k, v in pappyproxy.context.Filter._async_filter_functions.iteritems()] + return autocomplete_startswith(text, strs) + @crochet.wait_for(timeout=None) @defer.inlineCallbacks def filtercmd(line): @@ -179,7 +184,7 @@ def load_cmds(cmd): 'filter_clear': (filter_clear, None), 'filter_up': (filter_up, None), 'builtin_filter': (builtin_filter, complete_builtin_filter), - 'filter': (filtercmd, None), + 'filter': (filtercmd, complete_filtercmd), }) cmd.add_aliases([ #('filter_prune', ''), diff --git a/pappyproxy/plugins/macrocmds.py b/pappyproxy/plugins/macrocmds.py index 427ad3d..8f1950a 100644 --- a/pappyproxy/plugins/macrocmds.py +++ b/pappyproxy/plugins/macrocmds.py @@ -4,7 +4,7 @@ import shlex from pappyproxy.plugin import active_intercepting_macros, add_intercepting_macro, remove_intercepting_macro from pappyproxy.macros import load_macros, macro_from_requests, gen_imacro -from pappyproxy.util import PappyException, load_reqlist +from pappyproxy.util import PappyException, load_reqlist, autocomplete_startswith from twisted.internet import defer loaded_macros = [] @@ -64,6 +64,11 @@ def load_macros_cmd(line): int_macro_dict[macro.short_name] = macro loaded_int_macros.append(macro) print 'Loaded "%s"' % macro + +def complete_run_macro(text, line, begidx, endidx): + global macro_dict + strs = [k for k,v in macro_dict.iteritems()] + return autocomplete_startswith(text, strs) def run_macro(line): """ @@ -81,6 +86,24 @@ def run_macro(line): macro = macro_dict[mname] macro.execute(args[1:]) +def complete_run_int_macro(text, line, begidx, endidx): + global int_macro_dict + global loaded_int_macros + running = [] + not_running = [] + for macro in loaded_int_macros: + if macro.name in [m.name for k, m in active_intercepting_macros().iteritems()]: + running.append(macro) + else: + not_running.append(macro) + strs = [] + for m in not_running: + strs.append(macro.name) + strs.append(macro.file_name) + if macro.short_name: + strs.append(macro.short_name) + return autocomplete_startswith(text, strs) + def run_int_macro(line): """ Activate an intercepting macro @@ -103,6 +126,24 @@ def run_int_macro(line): print 'Error initializing macro:' raise e +def complete_stop_int_macro(text, line, begidx, endidx): + global int_macro_dict + global loaded_int_macros + running = [] + not_running = [] + for macro in loaded_int_macros: + if macro.name in [m.name for k, m in active_intercepting_macros().iteritems()]: + running.append(macro) + else: + not_running.append(macro) + strs = [] + for m in running: + strs.append(macro.name) + strs.append(macro.file_name) + if macro.short_name: + strs.append(macro.short_name) + return autocomplete_startswith(text, strs) + def stop_int_macro(line): """ Stop a running intercepting macro @@ -201,9 +242,9 @@ def load_cmds(cmd): 'generate_int_macro': (generate_int_macro, None), 'generate_macro': (generate_macro, None), 'list_int_macros': (list_int_macros, None), - 'stop_int_macro': (stop_int_macro, None), - 'run_int_macro': (run_int_macro, None), - 'run_macro': (run_macro, None), + 'stop_int_macro': (stop_int_macro, complete_stop_int_macro), + 'run_int_macro': (run_int_macro, complete_run_int_macro), + 'run_macro': (run_macro, complete_run_macro), 'load_macros': (load_macros_cmd, None), }) cmd.add_aliases([ diff --git a/pappyproxy/plugins/misc.py b/pappyproxy/plugins/misc.py index 87934de..22fcbb8 100644 --- a/pappyproxy/plugins/misc.py +++ b/pappyproxy/plugins/misc.py @@ -186,6 +186,10 @@ def run_without_color(line): with Capturing() as output: session.cons.onecmd(line.strip()) print remove_color(output.val) + +def version(line): + import pappyproxy + print pappyproxy.__version__ def load_cmds(cmd): cmd.set_cmds({ @@ -197,6 +201,7 @@ def load_cmds(cmd): 'merge': (merge_datafile, None), 'nocolor': (run_without_color, None), 'watch': (watch_proxy, None), + 'version': (version, None), }) cmd.add_aliases([ #('rpy', ''), diff --git a/pappyproxy/plugins/tagcmds.py b/pappyproxy/plugins/tagcmds.py index c47e826..a679bf2 100644 --- a/pappyproxy/plugins/tagcmds.py +++ b/pappyproxy/plugins/tagcmds.py @@ -2,7 +2,7 @@ import crochet import pappyproxy import shlex -from pappyproxy.plugin import main_context_ids +from pappyproxy.plugin import async_main_context_ids from pappyproxy.util import PappyException, load_reqlist from twisted.internet import defer from pappyproxy.http import Request @@ -26,7 +26,7 @@ def tag(line): print 'Tagging %s with %s' % (', '.join(reqids), tag) else: print "Tagging all in-context requests with %s" % tag - reqids = yield main_context_ids() + reqids = yield async_main_context_ids() for reqid in reqids: req = yield Request.load_request(reqid) @@ -58,7 +58,7 @@ def untag(line): print 'Removing tag %s from %s' % (tag, ', '.join(reqids)) else: print "Removing tag %s from all in-context requests" % tag - reqids = yield main_context_ids() + reqids = yield async_main_context_ids() for reqid in reqids: req = yield Request.load_request(reqid) diff --git a/pappyproxy/plugins/view.py b/pappyproxy/plugins/view.py index 049627e..2da7575 100644 --- a/pappyproxy/plugins/view.py +++ b/pappyproxy/plugins/view.py @@ -10,7 +10,7 @@ import urllib from pappyproxy.util import PappyException, utc2local, load_reqlist, print_table, print_request_rows, get_req_data_row from pappyproxy.http import Request, repeatable_parse_qs from twisted.internet import defer -from pappyproxy.plugin import main_context_ids +from pappyproxy.plugin import async_main_context_ids from pappyproxy.colors import Colors, Styles, verb_color, scode_color, path_formatter, host_color from pygments.formatters import TerminalFormatter from pygments.lexers.data import JsonLexer @@ -255,7 +255,7 @@ def list_reqs(line): print_count = 25 rows = [] - ids = yield main_context_ids(print_count) + ids = yield async_main_context_ids(print_count) for i in ids: req = yield Request.load_request(i) rows.append(get_req_data_row(req)) @@ -477,7 +477,7 @@ def get_param_info(line): found_params = {} - ids = yield main_context_ids() + ids = yield async_main_context_ids() for i in ids: req = yield Request.load_request(i) for k, v in req.url_params.all_pairs(): @@ -501,17 +501,20 @@ def dump_response(line): """ # dump the data of a response args = shlex.split(line) - reqid = args[0] - req = yield Request.load_request(reqid) - rsp = req.response - if len(args) >= 2: - fname = args[1] - else: - fname = req.path.split('/')[-1] + reqs = yield load_reqlist(args[0]) + for req in reqs: + if req.response: + rsp = req.response + if len(args) >= 2: + fname = args[1] + else: + fname = req.path.split('/')[-1] - with open(fname, 'w') as f: - f.write(rsp.body) - print 'Response data written to %s' % fname + with open(fname, 'w') as f: + f.write(rsp.body) + print 'Response data written to %s' % fname + else: + print 'Request %s does not have a response' % req.reqid @crochet.wait_for(timeout=None) @defer.inlineCallbacks @@ -525,7 +528,7 @@ def site_map(line): paths = True else: paths = False - ids = yield main_context_ids() + ids = yield async_main_context_ids() paths_set = set() for reqid in ids: req = yield Request.load_request(reqid) diff --git a/pappyproxy/proxy.py b/pappyproxy/proxy.py index 1e86ce7..336c6a0 100644 --- a/pappyproxy/proxy.py +++ b/pappyproxy/proxy.py @@ -262,13 +262,17 @@ class ProxyClientFactory(ClientFactory): self.intercepting_macros = {} self.use_as_proxy = False self.sendback_function = None + self.dropped_request = False + + # Only used for unit tests. Do not use. + self._use_string_transport = False + self._conn_info = None def log(self, message, symbol='*', verbosity_level=1): log(message, id=self.connection_id, symbol=symbol, verbosity_level=verbosity_level) - def buildProtocol(self, addr, _do_callback=True): + def buildProtocol(self, addr): from pappyproxy.pappy import session - # _do_callback is intended to help with testing and should not be modified if self.use_as_proxy and context.in_scope(self.request): p = UpstreamHTTPProxyClient(self.request) if 'username' in session.config.http_proxy and 'password' in session.config.http_proxy: @@ -279,8 +283,7 @@ class ProxyClientFactory(ClientFactory): p = ProxyClient(self.request) p.factory = self self.log("Building protocol", verbosity_level=3) - if _do_callback: - p.data_defer.addCallback(self.return_request_pair) + p.data_defer.addCallback(self.return_request_pair) return p def clientConnectionFailed(self, connector, reason): @@ -310,17 +313,24 @@ class ProxyClientFactory(ClientFactory): else: yield self.request.async_deep_save() - (sendreq, mangled) = yield macros.mangle_request(sendreq, mangle_macros) - - if sendreq and mangled and self.save_all: - self.start_time = datetime.datetime.utcnow() - sendreq.time_start = self.start_time - yield sendreq.async_deep_save() - - if session.config.http_proxy: - self.use_as_proxy = True - if (not self.stream_response) and self.sendback_function: - self.data_defer.addCallback(self.sendback_function) + (mangreq, mangled) = yield macros.mangle_request(sendreq, mangle_macros) + if mangreq is None: + self.log("Request dropped. Closing connections.") + self.request.tags.add('dropped') + self.request.response = None + self.dropped_request = True + defer.returnValue(None) + else: + sendreq = mangreq + if sendreq and mangled and self.save_all: + self.start_time = datetime.datetime.utcnow() + sendreq.time_start = self.start_time + yield sendreq.async_deep_save() + + if session.config.http_proxy: + self.use_as_proxy = True + if (not self.stream_response) and self.sendback_function: + self.data_defer.addCallback(self.sendback_function) else: self.log("Request out of scope, passing along unmangled") self.request = sendreq @@ -337,7 +347,7 @@ class ProxyClientFactory(ClientFactory): from pappyproxy.pappy import session self.end_time = datetime.datetime.utcnow() - if session.config.debug_to_file or session.config.debug_verbosity > 0: + if session.config.debug_to_file or session.config.debug_verbosity > 0 and request.response: log_request(printable_data(request.response.full_response), id=self.connection_id, symbol=' 0): - log_request(printable_data(request.response.full_response), - id=self.connection_id, symbol='<', verbosity_level=3) + if request.response and (session.config.debug_to_file or session.config.debug_verbosity > 0): + log_request(printable_data(request.response.full_response), + id=self.connection_id, symbol='<', verbosity_level=3) else: self.log("Response out of scope, passing along unmangled") self.data_defer.callback(request) @@ -369,6 +380,10 @@ class ProxyClientFactory(ClientFactory): from pappyproxy.pappy import session yield self.prepare_request() + if self.dropped_request: + self.data_defer.callback(self.request) + defer.returnValue(None) + if context.in_scope(self.request): # Get connection using config endpoint = get_endpoint(self.request.host, @@ -380,12 +395,26 @@ class ProxyClientFactory(ClientFactory): # Just forward it normally endpoint = get_endpoint(self.request.host, self.request.port, - self.request.is_ssl) + self.request.is_ssl, + socks_config=None, + use_http_proxy=False) + + if self._use_string_transport: + from pappyproxy.tests.testutil import TLSStringTransport + # "Connect" via string transport + protocol = self.buildProtocol(('127.0.0.1', 0)) - # Connect via the endpoint - self.log("Accessing using endpoint") - yield endpoint.connect(self) - self.log("Connected") + # Pass the protocol back to the test + if self._conn_info: + self._conn_info['protocol'] = protocol + + tr = TLSStringTransport() + protocol.makeConnection(tr) + else: + # Connect via the endpoint + self.log("Accessing using endpoint") + yield endpoint.connect(self) + self.log("Connected") class ProxyServerFactory(ServerFactory): @@ -529,6 +558,14 @@ class ProxyServer(LineReceiver): def send_response_back(self, request): if request.response is not None: self.transport.write(request.response.full_response) + else: + droppedrsp = http.Response(('HTTP/1.1 200 OK\r\n' + 'Connection: close\r\n' + 'Cache-control: no-cache\r\n' + 'Pragma: no-cache\r\n' + 'Cache-control: no-store\r\n' + 'X-Frame-Options: DENY\r\n\r\n')) + self.transport.write(droppedrsp.full_message) self.log("Response sent back, losing connection") self.transport.loseConnection() diff --git a/pappyproxy/session.py b/pappyproxy/session.py index 5e03b9f..ac5b26b 100644 --- a/pappyproxy/session.py +++ b/pappyproxy/session.py @@ -1,9 +1,24 @@ from .http import ResponseCookie class Session(object): + """ + A class used to maintain a session over multiple requests. Can remember cookies + and apply a specific header to requests. It is also possible to give the session + a list of cookie names and it will only save those cookies. + """ def __init__(self, cookie_names=None, header_names=None, cookie_vals=None, header_vals=None): + """ + Session(self, cookie_names=None, header_names=None, cookie_vals=None, header_vals=None) + Constructor + + :param cookie_names: A whitelist for cookies that should be saved from :func:`~pappyproxy.session.Session.save_req` and :func:`~pappyproxy.session.Session.save_rsp` in the session. If no values are given, all cookies will be saved. + :param header_names: A whitelist for headers that should be saved from :func:`~pappyproxy.session.Session.save_req` in the session. If no values are given, no headers will be saved. + :param cookie_vals: A dictionary of cookies to populate the session session with. The key should be the cookie name, and the value can be either a string or a :class:`~pappyproxy.http.ResponseCookie`. If a :class:`~pappyproxy.http.ResponseCookie` is given, its flags will be used in :func:`~pappyproxy.session.Session.apply_rsp`. + :param header_vals: A dictionary of header values to populate the session with. The key should be the header name and the value should be a string which should be the header value. + """ + self.cookies = cookie_names or [] self.headers = header_names or [] self.cookie_vals = cookie_vals or {} @@ -19,25 +34,61 @@ class Session(object): if k not in self.headers: self.headers.append(k) + def _cookie_obj(k, v): + """ + Returns the value as a cookie object regardless of if the cookie is a string or a ResponseCookie. + """ + if isinstance(v, ResponseCookie): + return v + else: + cookie_str = '%s=%s' % (k, v) + return ResponseCookie(cookie_str) + + def _cookie_val(v): + """ + Returns the value of the cookie regardless of if the value is a string or a ResponseCookie + """ + if isinstance(v, ResponseCookie): + return v.val + else: + return v + def apply_req(self, req): + """ + apply_req(request) + + Apply saved headers and cookies to the request + """ + for k, v in self.cookie_vals.iteritems(): - if isinstance(v, ResponseCookie): - req.cookies[v.key] = v.val - else: - req.cookies[k] = v + req.cookies[k] = self._cookie_val(v) for k, v in self.header_vals.iteritems(): req.headers[k] = v def apply_rsp(self, rsp): + """ + apply_rsp(response) + + Will add a Set-Cookie header for each saved cookie. Will not + apply any saved headers. If the cookie was added from a call to + :func:`~pappyproxy.session.Session.save_rsp`, the Set-Cookie flags + will be the same as the original response. + """ + for k, v in self.cookie_vals.iteritems(): - if isinstance(v, ResponseCookie): - rsp.set_cookie(v) - else: - cookie_str = '%s=%s' % (k, v) - rsp.set_cookie(ResponseCookie(cookie_str)) + val = self._cookie_obj(v) + rsp.set_cookie(val) # Don't apply headers to responses - def get_req(self, req, cookies=None, headers=None): + def save_req(self, req, cookies=None, headers=None): + """ + save_req(req, cookies=None, headers=None) + + Updates the state of the session from the given request. + Cookie and headers can be added to their whitelists by passing in a list + for either ``cookies`` or ``headers``. + """ + if cookies: for c in cookies: if c not in self.cookies: @@ -64,7 +115,14 @@ class Session(object): if header in self.headers: self.header_vals[header] = req.headers[header] - def get_rsp(self, rsp, cookies=None): + def save_rsp(self, rsp, cookies=None): + """ + save_rsp(rsp, cookies=None) + + Update the state of the session from the response. Only cookies can be + updated from a response. Additional values can be added to the whitelist + by passing in a list of values for the ``cookies`` parameter. + """ if cookies: for c in cookies: if c not in self.cookies: @@ -80,3 +138,38 @@ class Session(object): for k, v in rsp.cookies.all_pairs(): if v.key in self.cookies: self.cookie_vals[v.key] = v + + def set_cookie(key, val): + """ + set_cookie(key, val) + + Set a cookie in the session. ``val`` can be either a string or a :class:`~pappyproxy.http.ResponseCookie`. + If a :class:`~pappyproxy.http.ResponseCookie` is used, make sure its ``key`` value is the same as + the key passed in to the function. + """ + self.cookie_vals[key] = val + + def get_cookie(key): + """ + get_cookie(key) + + Returns a string with the value of the cookie with the given string, even if the value is a :class:`~pappyproxy.http.ResponseCookie`. + If you want to get a :class:`~pappyproxy.http.ResponseCookie`, use :func:`~pappyproxy.session.Session.get_rsp_cookie`. + """ + if not key in self.cookie_vals: + raise KeyError('Cookie is not stored in session.') + v = self.cookie_vals[key] + return self._cookie_val(v) + + def get_rsp_cookie(key): + """ + get_rsp_cookie(key) + + Returns the :class:`~pappyproxy.http.ResponseCookie` associated with the key + regardless of if the value is stored as a string or a :class:`~pappyproxy.http.ResponseCookie`. + """ + if not key in self.cookie_vals: + raise KeyError('Cookie is not stored in session.') + v = self.cookie_vals[key] + return self._cookie_obj(v) + diff --git a/pappyproxy/templates/macro.py.template b/pappyproxy/templates/macro.py.template index 32fafe0..e280481 100644 --- a/pappyproxy/templates/macro.py.template +++ b/pappyproxy/templates/macro.py.template @@ -1,4 +1,5 @@ from pappyproxy.http import Request, get_request, post_request, request_by_id +from pappyproxy.plugin import main_context_ids from pappyproxy.context import set_tag from pappyproxy.iter import * @@ -12,7 +13,7 @@ from pappyproxy.iter import * MACRO_NAME = '{{macro_name}}' SHORT_NAME = '{{short_name}}' - +{% if req_lines %} ########### ## Requests # It's suggested that you call .copy() on these and then edit attributes @@ -23,7 +24,7 @@ SHORT_NAME = '{{short_name}}' req{{ count }} = Request(({% for line in lines %} '{{ line }}'{% endfor %}{% set count = count+1 %} ){{ params }}) -{% endfor %} +{% endfor %}{% endif %} def run_macro(args): # Example: diff --git a/pappyproxy/tests/test_mangle.py b/pappyproxy/tests/test_mangle.py deleted file mode 100644 index 72871b0..0000000 --- a/pappyproxy/tests/test_mangle.py +++ /dev/null @@ -1,209 +0,0 @@ -import pytest -import mock -import pappyproxy - -from pappyproxy.mangle import async_mangle_request, async_mangle_response -from pappyproxy.http import Request, Response -from testutil import no_tcp, no_database, func_deleted, mock_deferred, mock_deep_save, fake_saving - -def retf(r): - return False - -@pytest.fixture -def ignore_edit(mocker): - new_edit = mock.MagicMock() - new_edit.return_value = mock_deferred(None) - mocker.patch('pappyproxy.console.edit_file', new=new_edit) - -@pytest.fixture -def ignore_delete(mocker): - new_os_remove = mock.MagicMock() - mocker.patch('os.remove', new=new_os_remove) - return new_os_remove - -@pytest.fixture(autouse=True) -def no_logging(mocker): - mocker.patch('pappyproxy.proxy.log') - -@pytest.fixture -def req(): - r = Request() - r.status_line = 'GET / HTTP/1.1' - r.host = 'www.ffffff.eeeeee' - r.raw_data = 'AAAA' - return r - -@pytest.fixture -def req_w_rsp(req): - r = Response() - r.status_line = 'HTTP/1.1 200 OK' - r.headers['Test-Header'] = 'ABC123' - r.raw_data = 'AAAA' - req.response = r - return req - -@pytest.fixture -def mock_tempfile(mocker): - new_tfile_obj = mock.MagicMock() - tfile_instance = mock.MagicMock() - new_tfile_obj.return_value.__enter__.return_value = tfile_instance - - tfile_instance.name = 'mockTemporaryFile' - mocker.patch('tempfile.NamedTemporaryFile', new=new_tfile_obj) - - new_open = mock.MagicMock() - fake_file = mock.MagicMock(spec=file) - new_open.return_value.__enter__.return_value = fake_file - mocker.patch('__builtin__.open', new_open) - - return (new_tfile_obj, tfile_instance, new_open, fake_file) - - -######################## -## Test request mangling - -@pytest.inlineCallbacks -def test_mangle_request_edit(req, mock_deep_save, mock_tempfile, - ignore_edit, ignore_delete): - tfile_obj, tfile_instance, new_open, fake_file = mock_tempfile - r = req - new_contents = ('GET / HTTP/1.1\r\n' - 'Content-Length: 4\r\n\r\n' - 'BBBB') - fake_file.read.return_value = new_contents - new_req = yield async_mangle_request(r) - assert not mock_deep_save.called - assert tfile_obj.called - assert tfile_instance.write.called - assert tfile_instance.write.call_args == ((r.full_request,),) - assert new_open.called - assert fake_file.read.called - - assert new_req.full_request == new_contents - -@pytest.inlineCallbacks -def test_mangle_request_edit_newlines(req, mock_deep_save, mock_tempfile, - ignore_edit, ignore_delete): - # Intercepting is off, request in scope - tfile_obj, tfile_instance, new_open, fake_file = mock_tempfile - r = req - new_contents = ('GET / HTTP/1.1\r\n' - 'Test-Head: FOOBIE\n' - 'Content-Length: 4\n\r\n' - 'BBBB') - fake_file.read.return_value = new_contents - new_req = yield async_mangle_request(r) - - assert new_req.full_request == ('GET / HTTP/1.1\r\n' - 'Test-Head: FOOBIE\r\n' - 'Content-Length: 4\r\n\r\n' - 'BBBB') - assert new_req.headers['Test-Head'] == 'FOOBIE' - -@pytest.inlineCallbacks -def test_mangle_request_drop(req, mock_deep_save, mock_tempfile, - ignore_edit, ignore_delete): - # Intercepting is off, request in scope - tfile_obj, tfile_instance, new_open, fake_file = mock_tempfile - r = req - new_contents = '' - fake_file.read.return_value = new_contents - new_req = yield async_mangle_request(r) - - assert new_req is None - -@pytest.inlineCallbacks -def test_mangle_request_edit_len(req, mock_deep_save, mock_tempfile, - ignore_edit, ignore_delete): - # Intercepting is off, request in scope - tfile_obj, tfile_instance, new_open, fake_file = mock_tempfile - r = req - new_contents = ('GET / HTTP/1.1\r\n' - 'Test-Head: FOOBIE\n' - 'Content-Length: 4\n\r\n' - 'BBBBAAAA') - fake_file.read.return_value = new_contents - new_req = yield async_mangle_request(r) - - assert new_req.full_request == ('GET / HTTP/1.1\r\n' - 'Test-Head: FOOBIE\r\n' - 'Content-Length: 8\r\n\r\n' - 'BBBBAAAA') - - -######################### -## Test response mangling - -@pytest.inlineCallbacks -def test_mangle_response_edit(req_w_rsp, mock_deep_save, mock_tempfile, - ignore_edit, ignore_delete): - # Intercepting is on, edit - tfile_obj, tfile_instance, new_open, fake_file = mock_tempfile - r = req_w_rsp - old_rsp = r.response.full_response - new_contents = ('HTTP/1.1 403 NOTOKIEDOKIE\r\n' - 'Content-Length: 4\r\n' - 'Other-Header: foobles\r\n\r\n' - 'BBBB') - fake_file.read.return_value = new_contents - mangled_rsp = yield async_mangle_response(r) - assert not mock_deep_save.called - assert tfile_obj.called - assert tfile_instance.write.called - assert tfile_instance.write.call_args == ((old_rsp,),) - assert new_open.called - assert fake_file.read.called - - assert mangled_rsp.full_response == new_contents - -@pytest.inlineCallbacks -def test_mangle_response_newlines(req_w_rsp, mock_deep_save, mock_tempfile, - ignore_edit, ignore_delete): - # Intercepting is off, request in scope - tfile_obj, tfile_instance, new_open, fake_file = mock_tempfile - r = req_w_rsp - old_rsp = r.response.full_response - new_contents = ('HTTP/1.1 403 NOTOKIEDOKIE\n' - 'Content-Length: 4\n' - 'Other-Header: foobles\r\n\n' - 'BBBB') - fake_file.read.return_value = new_contents - mangled_rsp = yield async_mangle_response(r) - - assert mangled_rsp.full_response == ('HTTP/1.1 403 NOTOKIEDOKIE\r\n' - 'Content-Length: 4\r\n' - 'Other-Header: foobles\r\n\r\n' - 'BBBB') - assert mangled_rsp.headers['Other-Header'] == 'foobles' - -@pytest.inlineCallbacks -def test_mangle_response_drop(req_w_rsp, mock_deep_save, mock_tempfile, - ignore_edit, ignore_delete): - # Intercepting is off, request in scope - tfile_obj, tfile_instance, new_open, fake_file = mock_tempfile - r = req_w_rsp - old_rsp = r.response.full_response - new_contents = '' - fake_file.read.return_value = new_contents - mangled_rsp = yield async_mangle_response(r) - - assert mangled_rsp is None - -@pytest.inlineCallbacks -def test_mangle_response_new_len(req_w_rsp, mock_deep_save, mock_tempfile, - ignore_edit, ignore_delete): - # Intercepting is off, request in scope - tfile_obj, tfile_instance, new_open, fake_file = mock_tempfile - r = req_w_rsp - old_rsp = r.response.full_response - new_contents = ('HTTP/1.1 403 NOTOKIEDOKIE\n' - 'Content-Length: 4\n' - 'Other-Header: foobles\r\n\n' - 'BBBBAAAA') - fake_file.read.return_value = new_contents - mangled_rsp = yield async_mangle_response(r) - - assert mangled_rsp.full_response == ('HTTP/1.1 403 NOTOKIEDOKIE\r\n' - 'Content-Length: 8\r\n' - 'Other-Header: foobles\r\n\r\n' - 'BBBBAAAA') diff --git a/pappyproxy/tests/test_proxy.py b/pappyproxy/tests/test_proxy.py index 2da7f9a..4d2d9f6 100644 --- a/pappyproxy/tests/test_proxy.py +++ b/pappyproxy/tests/test_proxy.py @@ -4,63 +4,114 @@ import random import datetime import pappyproxy import base64 +import collections from pappyproxy import http from pappyproxy.proxy import ProxyClientFactory, ProxyServerFactory, UpstreamHTTPProxyClient +from pappyproxy.http import Request, Response +from pappyproxy.macros import InterceptMacro from testutil import mock_deferred, func_deleted, TLSStringTransport, freeze, mock_int_macro, no_tcp from twisted.internet import defer -@pytest.fixture(autouse=True) -def proxy_patches(mocker): - #mocker.patch("twisted.test.iosim.FakeTransport.startTLS") - mocker.patch("pappyproxy.proxy.load_certs_from_dir", new=mock_generate_cert) +class InterceptMacroTest(InterceptMacro): + + def __init__(self, new_req=None, new_rsp=None): + InterceptMacro.__init__(self) + + self.new_req = None + self.new_rsp = None + if new_req: + self.intercept_requests = True + self.new_req = new_req + if new_rsp: + self.intercept_responses = True + self.new_rsp = new_rsp + + def mangle_request(self, request): + if self.intercept_requests: + return self.new_req + else: + return request + + def mangle_response(self, request): + if self.intercept_responses: + return self.new_rsp + else: + return request.response + +class TestProxyConnection(object): + + @property + def client_protocol(self): + if 'protocol' not in self.conn_info: + raise Exception('Connection to server not made. Cannot write data as server.') + return self.conn_info['protocol'] + + @property + def client_factory(self): + if 'protocol' not in self.conn_info: + raise Exception('Connection to server not made. Cannot write data as server.') + return self.conn_info['factory'] -@pytest.fixture -def server_factory(): - return gen_server_factory() - -@pytest.fixture(autouse=True) -def mock_config(mocker): - c = pappyproxy.config.PappyConfig() - s = pappyproxy.pappy.PappySession(c) - mocker.patch.object(pappyproxy.pappy, 'session', new=s) - -def socks_config(mocker, config): - pappyproxy.pappy.session.config.socks_proxy = config - -def http_proxy_config(mocker, config): - pappyproxy.pappy.session.config.http_proxy = config - -def gen_server_factory(int_macros={}): - factory = ProxyServerFactory() - factory.save_all = True - factory.intercepting_macros = int_macros - return factory - -def gen_server_protocol(int_macros={}): - server_factory = gen_server_factory(int_macros=int_macros) - protocol = server_factory.buildProtocol(('127.0.0.1', 0)) - tr = TLSStringTransport() - protocol.makeConnection(tr) - return protocol - -@defer.inlineCallbacks -def gen_client_protocol(req, stream_response=False, save_all=True): - return_transport = TLSStringTransport() - factory = ProxyClientFactory(req, - save_all=save_all, - stream_response=stream_response, - return_transport=return_transport) - yield factory.prepare_request() - protocol = factory.buildProtocol(('127.0.0.1', 0), _do_callback=False) - tr = TLSStringTransport() - protocol.makeConnection(tr) - defer.returnValue(protocol) - -@pytest.fixture -def server_protocol(): - return gen_server_protocol() + def setUp(self, mocker, int_macros={}, socks_config=None, http_config=None, in_scope=True): + self.mocker = mocker + self.conn_info = {} + + # Mock config + self.mock_config = pappyproxy.config.PappyConfig() + self.mock_config.socks_proxy = socks_config + self.mock_config.http_proxy = http_config + self.mock_session = pappyproxy.pappy.PappySession(self.mock_config) + mocker.patch.object(pappyproxy.pappy, 'session', new=self.mock_session) + mocker.patch("pappyproxy.proxy.load_certs_from_dir", new=mock_generate_cert) + + # Listening server + self.server_factory = ProxyServerFactory() + self.server_factory.save_all = True + self.server_factory.intercepting_macros = int_macros + + self.server_protocol = self.server_factory.buildProtocol(('127.0.0.1', 0)) + self.server_transport = TLSStringTransport() + self.server_protocol.makeConnection(self.server_transport) + + # Other mocks + self.req_save = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + self.submit_request = mocker.patch('pappyproxy.http.Request.submit_request', + new=self.gen_mock_submit_request()) + self.get_endpoint = mocker.patch('pappyproxy.proxy.get_endpoint') + self.in_scope = mocker.patch('pappyproxy.context.in_scope').return_value = in_scope + + def gen_mock_submit_request(self): + orig = Request.submit_request + def f(request, save_request=False, intercepting_macros={}, stream_transport=None): + return orig(request, save_request=save_request, + intercepting_macros=intercepting_macros, + stream_transport=stream_transport, + _factory_string_transport=True, + _conn_info=self.conn_info) + return f + + def perform_connect_request(self): + self.write_as_browser('CONNECT https://www.AAAA.BBBB:443 HTTP/1.1\r\n\r\n') + assert self.read_as_browser() == 'HTTP/1.1 200 Connection established\r\n\r\n' + + def write_as_browser(self, data): + self.server_protocol.dataReceived(data) + + def read_as_browser(self): + s = self.server_protocol.transport.value() + self.server_protocol.transport.clear() + return s + + def write_as_server(self, data): + self.client_protocol.dataReceived(data) + def read_as_server(self): + s = self.client_protocol.transport.value() + self.client_protocol.transport.clear() + return s + + def mock_req_async_save(req): req.reqid = str(random.randint(1,1000000)) return mock_deferred() @@ -71,9 +122,6 @@ def mock_mangle_response_side_effect(new_rsp): return mock_deferred(True) return f -#################### -## Mock functions - def mock_generate_cert(cert_dir): private_key = ('-----BEGIN PRIVATE KEY-----\n' 'MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDAoClrYUEB7lM0\n' @@ -143,753 +191,232 @@ def test_no_tcp(): with pytest.raises(NotImplementedError): TLSWrapClientEndpoint('asdf.2341') -################ -### Proxy Server - -def test_proxy_server_connect(mocker, server_protocol): - mstarttls = mocker.patch('pappyproxy.tests.testutil.TLSStringTransport.startTLS') - server_protocol.dataReceived('CONNECT https://www.AAAA.BBBB:443 HTTP/1.1\r\n\r\n') - assert server_protocol.transport.value() == 'HTTP/1.1 200 Connection established\r\n\r\n' - assert mstarttls.called - -def test_proxy_server_forward_basic(mocker, server_protocol): - mforward = mocker.patch('pappyproxy.proxy.ProxyServer._generate_and_submit_client') - mreset = mocker.patch('pappyproxy.proxy.ProxyServer._reset') - +def test_proxy_server_connect(mocker): + proxy = TestProxyConnection() + proxy.setUp(mocker) + proxy.write_as_browser('CONNECT https://www.AAAA.BBBB:443 HTTP/1.1\r\n\r\n') + rsp = proxy.read_as_browser() + print rsp + assert rsp == 'HTTP/1.1 200 Connection established\r\n\r\n' + +def test_proxy_server_forward_basic(mocker): + proxy = TestProxyConnection() + proxy.setUp(mocker) req_contents = ('POST /fooo HTTP/1.1\r\n' 'Test-Header: foo\r\n' 'Content-Length: 4\r\n' + 'Host: www.AAAA.BBBB\r\n' '\r\n' 'ABCD') - server_protocol.dataReceived(req_contents) - - assert mforward.called - assert mreset.called - assert server_protocol._request_obj.full_message == req_contents - -def test_proxy_server_connect_uri(mocker, server_protocol): - mforward = mocker.patch('pappyproxy.proxy.ProxyServer._generate_and_submit_client') - server_protocol.dataReceived('CONNECT https://www.AAAA.BBBB:443 HTTP/1.1\r\n\r\n') - server_protocol.dataReceived('GET /fooo HTTP/1.1\r\nTest-Header: foo\r\n\r\n') - assert server_protocol._connect_uri == 'https://www.AAAA.BBBB' - assert server_protocol._request_obj.url == 'https://www.AAAA.BBBB' - assert server_protocol._request_obj.port == 443 - -## ProxyServer._generate_and_submit_client - -def test_proxy_server_create_client_factory(mocker, server_protocol): - mfactory = mock.MagicMock() - mfactory_class = mocker.patch('pappyproxy.proxy.ProxyClientFactory') - mfactory_class.return_value = mfactory - - mocker.patch('pappyproxy.proxy.ProxyServer._make_remote_connection') - - mfactory.prepare_request.return_value = mock_deferred(None) - full_req = ('POST /fooo HTTP/1.1\r\n' - 'Test-Header: foo\r\n' - 'Content-Length: 4\r\n' - '\r\n' - 'ABCD') - server_protocol.connection_id = 100 - - server_protocol.dataReceived(full_req) - # Make sure we created a ClientFactory with the right arguments - f_args, f_kwargs = mfactory_class.call_args - assert len(f_args) == 1 - - # Make sure the request got to the client class - req = f_args[0] - assert req.full_message == full_req - - # Make sure the correct settings got to the proxy - assert f_kwargs['stream_response'] == True - assert f_kwargs['save_all'] == True - - # Make sure we initialized the client factory - assert mfactory.prepare_request.called - assert mfactory.connection_id == 100 - assert server_protocol._make_remote_connection.called # should be immediately called because mock deferred - -def test_proxy_server_no_streaming_with_int_macros(mocker): - mfactory = mock.MagicMock() - mfactory_class = mocker.patch('pappyproxy.proxy.ProxyClientFactory') - mfactory_class.return_value = mfactory - - mocker.patch('pappyproxy.proxy.ProxyServer._make_remote_connection') - - mfactory.prepare_request.return_value = mock_deferred(None) - full_req = ('POST /fooo HTTP/1.1\r\n' - 'Test-Header: foo\r\n' - 'Content-Length: 4\r\n' - '\r\n' - 'ABCD') - - int_macros = [{'mockmacro': mock_int_macro(modified_req='GET / HTTP/1.1\r\n\r\n')}] - server_protocol = gen_server_protocol(int_macros=int_macros) - server_protocol.dataReceived(full_req) - f_args, f_kwargs = mfactory_class.call_args - assert f_kwargs['stream_response'] == False - -## ProxyServer._make_remote_connection - -@pytest.inlineCallbacks -def test_proxy_server_make_tcp_connection(mocker, server_protocol): - mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") - mtcpe_class.return_value = mtcpe = mock.MagicMock() - mtcpe.connect.return_value = mock_deferred() - - server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly - - req = http.Request("GET / HTTP/1.1\r\n\r\n") - req.host = 'Foo.Bar.Brazzers' - req.port = 80085 - server_protocol._request_obj = req - - yield server_protocol._make_remote_connection(req) - targs, tkwargs = mtcpe_class.call_args - assert targs[1] == 'Foo.Bar.Brazzers' - assert targs[2] == 80085 - assert tkwargs == {} - mtcpe.connect.assert_called_once_with(server_protocol._client_factory) - -@pytest.inlineCallbacks -def test_proxy_server_make_ssl_connection(mocker, server_protocol): - mssle_class = mocker.patch("twisted.internet.endpoints.SSL4ClientEndpoint") - mssle_class.return_value = mssle = mock.MagicMock() - mssle.connect.return_value = mock_deferred() - - server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly - - req = http.Request("GET / HTTP/1.1\r\n\r\n", is_ssl=True) - req.host = 'Foo.Bar.Brazzers' - req.port = 80085 - server_protocol._request_obj = req - - yield server_protocol._make_remote_connection(req) - targs, tkwargs = mssle_class.call_args - assert targs[1] == 'Foo.Bar.Brazzers' - assert targs[2] == 80085 - assert tkwargs == {} - mssle.connect.assert_called_once_with(server_protocol._client_factory) - -@pytest.inlineCallbacks -def test_proxy_server_make_tcp_connection_socks(mocker): - socks_config(mocker, {'host': '12345', 'port': 5555}) - - tls_wrap_class = mocker.patch("txsocksx.tls.TLSWrapClientEndpoint") - - mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") - mtcpe_class.return_value = mtcpe = mock.MagicMock() - - socks_class = mocker.patch("txsocksx.client.SOCKS5ClientEndpoint") - socks_class.return_value = sockse = mock.MagicMock() - - server_protocol = gen_server_protocol() - server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly - - req = http.Request("GET / HTTP/1.1\r\n\r\n") - req.host = 'Foo.Bar.Brazzers' - req.port = 80085 - server_protocol._request_obj = req - - yield server_protocol._make_remote_connection(req) - sargs, skwargs = socks_class.call_args - targs, tkwargs = mtcpe_class.call_args - assert targs[1] == '12345' - assert targs[2] == 5555 - assert sargs[0] == 'Foo.Bar.Brazzers' - assert sargs[1] == 80085 - assert sargs[2] == mtcpe - assert skwargs == {'methods': {'anonymous': ()}} - assert not tls_wrap_class.called - sockse.connect.assert_called_once_with(server_protocol._client_factory) - -@pytest.inlineCallbacks -def test_proxy_server_make_ssl_connection_socks(mocker): - socks_config(mocker, {'host': '12345', 'port': 5555}) - - tls_wrap_class = mocker.patch("txsocksx.tls.TLSWrapClientEndpoint") - tls_wrape = tls_wrap_class.return_value = mock.MagicMock() - - mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") - mtcpe_class.return_value = mtcpe = mock.MagicMock() - - socks_class = mocker.patch("txsocksx.client.SOCKS5ClientEndpoint") - socks_class.return_value = sockse = mock.MagicMock() - - server_protocol = gen_server_protocol() - server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly - - req = http.Request("GET / HTTP/1.1\r\n\r\n") - req.host = 'Foo.Bar.Brazzers' - req.port = 80085 - req.is_ssl = True - server_protocol._request_obj = req - - yield server_protocol._make_remote_connection(req) - sargs, skwargs = socks_class.call_args - targs, tkwargs = mtcpe_class.call_args - assert targs[1] == '12345' - assert targs[2] == 5555 - assert sargs[0] == 'Foo.Bar.Brazzers' - assert sargs[1] == 80085 - assert sargs[2] == mtcpe - assert skwargs == {'methods': {'anonymous': ()}} - assert not sockse.called - tls_wrape.connect.assert_called_once_with(server_protocol._client_factory) - -@pytest.inlineCallbacks -def test_proxy_server_make_ssl_connection_socks_username_only(mocker): - socks_config(mocker, {'host': '12345', 'port': 5555, 'username': 'foo'}) - - tls_wrap_class = mocker.patch("txsocksx.tls.TLSWrapClientEndpoint") - tls_wrape = tls_wrap_class.return_value = mock.MagicMock() - - mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") - mtcpe_class.return_value = mtcpe = mock.MagicMock() - - socks_class = mocker.patch("txsocksx.client.SOCKS5ClientEndpoint") - socks_class.return_value = sockse = mock.MagicMock() - - server_protocol = gen_server_protocol() - server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly - - req = http.Request("GET / HTTP/1.1\r\n\r\n") - req.host = 'Foo.Bar.Brazzers' - req.port = 80085 - req.is_ssl = True - server_protocol._request_obj = req - - yield server_protocol._make_remote_connection(req) - sargs, skwargs = socks_class.call_args - targs, tkwargs = mtcpe_class.call_args - assert targs[1] == '12345' - assert targs[2] == 5555 - assert sargs[0] == 'Foo.Bar.Brazzers' - assert sargs[1] == 80085 - assert sargs[2] == mtcpe - assert skwargs == {'methods': {'anonymous': ()}} - assert not sockse.called - tls_wrape.connect.assert_called_once_with(server_protocol._client_factory) - -@pytest.inlineCallbacks -def test_proxy_server_make_ssl_connection_socks_username_password(mocker): - socks_config(mocker, {'host': '12345', 'port': 5555, 'username': 'foo', 'password': 'password'}) - - tls_wrap_class = mocker.patch("txsocksx.tls.TLSWrapClientEndpoint") - tls_wrape = tls_wrap_class.return_value = mock.MagicMock() - - mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") - mtcpe_class.return_value = mtcpe = mock.MagicMock() - - socks_class = mocker.patch("txsocksx.client.SOCKS5ClientEndpoint") - socks_class.return_value = sockse = mock.MagicMock() - - server_protocol = gen_server_protocol() - server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly - - req = http.Request("GET / HTTP/1.1\r\n\r\n") - req.host = 'Foo.Bar.Brazzers' - req.port = 80085 - req.is_ssl = True - server_protocol._request_obj = req - - yield server_protocol._make_remote_connection(req) - sargs, skwargs = socks_class.call_args - targs, tkwargs = mtcpe_class.call_args - assert targs[1] == '12345' - assert targs[2] == 5555 - assert sargs[0] == 'Foo.Bar.Brazzers' - assert sargs[1] == 80085 - assert sargs[2] == mtcpe - assert skwargs == {'methods': {'login': ('foo','password'), 'anonymous': ()}} - assert not sockse.called - tls_wrape.connect.assert_called_once_with(server_protocol._client_factory) - - -######################## -### Proxy Client Factory - -@pytest.inlineCallbacks -def test_proxy_client_factory_prepare_reqs_simple(mocker, freeze): - import datetime - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - rsave.return_value = mock_deferred() - mocker.patch('pappyproxy.context.in_scope').return_value = True - mocker.patch('pappyproxy.macros.mangle_request').return_value = mock_deferred((req, False)) - - cf = ProxyClientFactory(req, - save_all=False, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) - assert req.reqid is None - assert not rsave.called - assert len(rsave.mock_calls) == 0 - -@pytest.inlineCallbacks -def test_proxy_client_factory_prepare_reqs_360_noscope(mocker, freeze): - import datetime - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - - rsave = mocker.patch('pappyproxy.http.Request.async_deep_save') - rsave.return_value = mock_deferred() - mocker.patch('pappyproxy.context.in_scope').return_value = False - mocker.patch('pappyproxy.macros.mangle_request', new=func_deleted) - - cf = ProxyClientFactory(req, - save_all=True, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - assert req.time_start == None - assert req.reqid is None - assert not rsave.called - assert len(rsave.mock_calls) == 0 - -@pytest.inlineCallbacks -def test_proxy_client_factory_prepare_reqs_save(mocker, freeze): - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - mocker.patch('pappyproxy.context.in_scope').return_value = True - mocker.patch('pappyproxy.macros.mangle_request').return_value = mock_deferred((req, False)) - - cf = ProxyClientFactory(req, - save_all=True, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) - assert req.reqid is not None - assert rsave.called - assert len(rsave.mock_calls) == 1 - -@pytest.inlineCallbacks -def test_proxy_client_factory_prepare_reqs_360_noscope_save(mocker, freeze): - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - mangreq = http.Request('BOOO / HTTP/1.1\r\n\r\n') - - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - mocker.patch('pappyproxy.context.in_scope').return_value = False - mocker.patch('pappyproxy.macros.mangle_request', side_effect=func_deleted) - - cf = ProxyClientFactory(req, - save_all=True, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - assert req.time_start == None - assert req.reqid is None - assert not rsave.called - assert len(rsave.mock_calls) == 0 - -@pytest.inlineCallbacks -def test_proxy_client_factory_prepare_mangle_req(mocker, freeze): - - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - mangreq = http.Request('BOOO / HTTP/1.1\r\n\r\n') - - def inc_day_mangle(x, y): - freeze.delta(days=1) - return mock_deferred((mangreq, True)) - - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - mocker.patch('pappyproxy.context.in_scope').return_value = True - mocker.patch('pappyproxy.macros.mangle_request', side_effect=inc_day_mangle) - - cf = ProxyClientFactory(req, - save_all=True, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - - assert cf.request == mangreq - assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) - assert cf.request.time_start == datetime.datetime(2015, 1, 2, 3, 30, 15, 50) - assert cf.request.reqid is not None - assert len(rsave.mock_calls) == 2 - -@pytest.inlineCallbacks -def test_proxy_client_factory_prepare_mangle_req_drop(mocker, freeze): - - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - - def inc_day_mangle(x, y): - freeze.delta(days=1) - return mock_deferred((None, True)) - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - mocker.patch('pappyproxy.context.in_scope').return_value = True - mocker.patch('pappyproxy.macros.mangle_request', side_effect=inc_day_mangle) - - cf = ProxyClientFactory(req, - save_all=True, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - - assert cf.request is None - assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) - assert len(rsave.mock_calls) == 1 - -@pytest.inlineCallbacks -def test_proxy_client_factory_prepare_mangle_req(mocker, freeze): - - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - mangreq = http.Request('BOOO / HTTP/1.1\r\n\r\n') - - def inc_day_mangle(x, y): - freeze.delta(days=1) - return mock_deferred((mangreq, True)) - - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - mocker.patch('pappyproxy.context.in_scope').return_value = True - mocker.patch('pappyproxy.macros.mangle_request', side_effect=inc_day_mangle) - - cf = ProxyClientFactory(req, - save_all=True, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - - assert cf.request == mangreq - assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) - assert cf.request.time_start == datetime.datetime(2015, 1, 2, 3, 30, 15, 50) - assert cf.request.reqid is not None - assert len(rsave.mock_calls) == 2 - -### return_request_pair - -@pytest.inlineCallbacks -def test_proxy_client_factory_return_request_pair_simple(mocker, freeze): - """ - Make sure the proxy doesn't do anything if the request is out of scope - """ - - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - mocker.patch('pappyproxy.context.in_scope').return_value = False - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - req.reqid = 1 - rsp = http.Response('HTTP/1.1 200 OK\r\n\r\n') - checkrsp = rsp.copy() - req.response = rsp - - mocker.patch('pappyproxy.macros.mangle_response').return_value = mock_deferred(False) - - cf = ProxyClientFactory(req, - save_all=False, - stream_response=False, - return_transport=None) - cf.start_time = datetime.datetime(2015, 1, 1, 3, 30, 14, 50) - cf.return_request_pair(req) - result = yield cf.data_defer - assert result == req - assert result.response == checkrsp - assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 14, 50) - assert req.time_end == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) - assert len(rsave.mock_calls) == 0 - -@pytest.inlineCallbacks -def test_proxy_client_factory_return_request_pair_mangle(mocker, freeze): - """ - Make one modification to the response - """ - - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - mocker.patch('pappyproxy.context.in_scope').return_value = True - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - req.reqid = 1 - rsp = http.Response('HTTP/1.1 200 OK\r\n\r\n') - req.response = rsp - - new_rsp = http.Response('HTTP/1.1 6969 LOLMANGLED\r\n\r\n') - checkrsp = new_rsp.copy() - - mocker.patch('pappyproxy.macros.mangle_response', - side_effect=mock_mangle_response_side_effect(new_rsp)) - - cf = ProxyClientFactory(req, - save_all=True, - stream_response=False, - return_transport=None) - cf.start_time = datetime.datetime(2015, 1, 1, 3, 30, 14, 50) - cf.return_request_pair(req) - result = yield cf.data_defer - assert result == req - assert result.response == checkrsp - assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 14, 50) - assert req.time_end == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) - assert len(rsave.mock_calls) == 2 - -@pytest.inlineCallbacks -def test_proxy_client_factory_return_request_pair_no_save_all(mocker, freeze): - """ - Make one modification to the response but don't save it - """ - - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - mocker.patch('pappyproxy.context.in_scope').return_value = True - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - req.reqid = 1 - rsp = http.Response('HTTP/1.1 200 OK\r\n\r\n') - req.response = rsp - - new_rsp = http.Response('HTTP/1.1 6969 LOLMANGLED\r\n\r\n') - checkrsp = new_rsp.copy() - - mocker.patch('pappyproxy.macros.mangle_response', - side_effect=mock_mangle_response_side_effect(new_rsp)).return_value = mock_deferred(True) - - cf = ProxyClientFactory(req, - save_all=False, - stream_response=False, - return_transport=None) - cf.start_time = datetime.datetime(2015, 1, 1, 3, 30, 14, 50) - cf.return_request_pair(req) - result = yield cf.data_defer - assert result == req - assert result.response == checkrsp - assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 14, 50) - assert req.time_end == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) - assert len(rsave.mock_calls) == 0 - -@pytest.inlineCallbacks -def test_proxy_client_factory_return_request_pair_save_all_no_mangle(mocker, freeze): - """ - Make one modification to the response but don't save it - """ - - freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - mocker.patch('pappyproxy.context.in_scope').return_value = True - - req = http.Request('GET / HTTP/1.1\r\n\r\n') - req.reqid = 1 - rsp = http.Response('HTTP/1.1 200 OK\r\n\r\n') - checkrsp = rsp.copy() - req.response = rsp - - mocker.patch('pappyproxy.macros.mangle_response').return_value = mock_deferred(False) - - cf = ProxyClientFactory(req, - save_all=True, - stream_response=False, - return_transport=None) - cf.start_time = datetime.datetime(2015, 1, 1, 3, 30, 14, 50) - cf.return_request_pair(req) - result = yield cf.data_defer - assert result == req - assert result.response == checkrsp - assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 14, 50) - assert req.time_end == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) - assert len(rsave.mock_calls) == 1 - -@pytest.inlineCallbacks -def test_proxy_client_factory_build_protocol_http_proxy(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345}) - - r = http.Request('GET / HTTP/1.1\r\n\r\n') - cf = ProxyClientFactory(r, - save_all=False, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - p = cf.buildProtocol('') - assert isinstance(p, UpstreamHTTPProxyClient) - assert p.creds is None - assert p.proxy_connected == False - -@pytest.inlineCallbacks -def test_proxy_client_factory_build_protocol_http_proxy_username_only(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345, 'username': 'foo'}) - - r = http.Request('GET / HTTP/1.1\r\n\r\n') - cf = ProxyClientFactory(r, - save_all=False, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - p = cf.buildProtocol('') - assert p.creds is None - -@pytest.inlineCallbacks -def test_proxy_client_factory_build_protocol_http_proxy_username_only(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345, 'username': 'foo', 'password': 'password'}) - - r = http.Request('GET / HTTP/1.1\r\n\r\n') - cf = ProxyClientFactory(r, - save_all=False, - stream_response=False, - return_transport=None) - yield cf.prepare_request() - p = cf.buildProtocol('') - assert p.creds == ('foo', 'password') - -@pytest.inlineCallbacks -def test_proxy_upstream_client_connection_made(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345}) - r = http.Request(('GET / HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n')) - p = yield gen_client_protocol(r, save_all=False) - assert isinstance(p, UpstreamHTTPProxyClient) - assert p.transport.value() == ('GET http://www.example.faketld/ HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n') - -@pytest.inlineCallbacks -def test_proxy_upstream_client_connection_made_creds(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345, 'username':'foo', 'password':'password'}) - r = http.Request(('GET / HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n')) - p = yield gen_client_protocol(r, save_all=False) - assert isinstance(p, UpstreamHTTPProxyClient) - assert p.transport.value() == ('GET http://www.example.faketld/ HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n' - 'Proxy-Authorization: Basic %s\r\n\r\n') % base64.b64encode('foo:password') - -@pytest.inlineCallbacks -def test_proxy_upstream_client_connection_made_ssl(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345}) - r = http.Request(('GET / HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n')) - r.is_ssl = True - p = yield gen_client_protocol(r, save_all=False) - assert isinstance(p, UpstreamHTTPProxyClient) - assert p.transport.value() == ('CONNECT www.example.faketld:443 HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n') - -@pytest.inlineCallbacks -def test_proxy_upstream_client_connection_made_ssl_creds(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345, 'username':'foo', 'password':'password'}) - r = http.Request(('GET / HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n')) - r.is_ssl = True - p = yield gen_client_protocol(r, save_all=False) - assert isinstance(p, UpstreamHTTPProxyClient) - assert p.transport.value() == ('CONNECT www.example.faketld:443 HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n' - 'Proxy-Authorization: Basic %s\r\n\r\n') % base64.b64encode('foo:password') - -@pytest.inlineCallbacks -def test_proxy_upstream_client_connection_made_ssl(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345}) - mstarttls = mocker.patch('pappyproxy.tests.testutil.TLSStringTransport.startTLS') - - r = http.Request(('GET / HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n')) - r.is_ssl = True - p = yield gen_client_protocol(r, save_all=False) - assert p.transport.value() == ('CONNECT www.example.faketld:443 HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n') - assert not mstarttls.called - p.transport.clear() - p.dataReceived('HTTP/1.1 200 OK\r\n\r\n') - assert mstarttls.called - assert p.transport.value() == ('GET / HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n') - -@pytest.inlineCallbacks -def test_proxy_upstream_client_connection_made_ssl_creds(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345, 'username':'foo', 'password':'password'}) - mstarttls = mocker.patch('pappyproxy.tests.testutil.TLSStringTransport.startTLS') - - r = http.Request(('GET / HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n')) - r.is_ssl = True - p = yield gen_client_protocol(r, save_all=False) - assert p.transport.value() == ('CONNECT www.example.faketld:443 HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n' - 'Proxy-Authorization: Basic %s\r\n\r\n') % base64.b64encode('foo:password') - assert not mstarttls.called - p.transport.clear() - p.dataReceived('HTTP/1.1 200 OK\r\n\r\n') - assert mstarttls.called - assert p.transport.value() == ('GET / HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n') - -@pytest.inlineCallbacks -def test_proxy_upstream_client_connection_incorrect_creds(mocker): - http_proxy_config(mocker, {'host': '12345', 'port': 12345, 'username':'foo', 'password':'password'}) - mstarttls = mocker.patch('pappyproxy.tests.testutil.TLSStringTransport.startTLS') - closed = mocker.patch('pappyproxy.tests.testutil.TLSStringTransport.loseConnection') - - r = http.Request(('GET / HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n\r\n')) - r.is_ssl = True - p = yield gen_client_protocol(r, save_all=False) - assert p.transport.value() == ('CONNECT www.example.faketld:443 HTTP/1.1\r\n' - 'Host: www.example.faketld\r\n' - 'Proxy-Authorization: Basic %s\r\n\r\n') % base64.b64encode('foo:password') - p.transport.clear() - p.dataReceived('HTTP/1.1 407 YOU DUN FUCKED UP\r\n\r\n') - assert not mstarttls.called - assert p.transport.value() == '' - assert closed.called - -### ProxyClient tests - -@pytest.inlineCallbacks -def test_proxy_client_simple(mocker): - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - req = http.Request('GET / HTTP/1.1\r\n\r\n') - client = yield gen_client_protocol(req, stream_response=False) - assert client.transport.value() == 'GET / HTTP/1.1\r\n\r\n' - client.transport.clear() - rsp = 'HTTP/1.1 200 OKILE DOKELY\r\n\r\n' - client.dataReceived(rsp) - retpair = yield client.data_defer - assert retpair.response.full_message == rsp + rsp_contents = ('HTTP/1.1 200 OK\r\n\r\n') + proxy.write_as_browser(req_contents) + assert proxy.read_as_server() == req_contents + proxy.write_as_server(rsp_contents) + assert proxy.read_as_browser() == rsp_contents + proxy.get_endpoint.assert_called_with('www.AAAA.BBBB', 80, False, socks_config=None, use_http_proxy=True) + assert proxy.req_save.called + +def test_proxy_server_forward_basic_ssl(mocker): + proxy = TestProxyConnection() + proxy.setUp(mocker) + proxy.perform_connect_request() + req_contents = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'ABCD') + rsp_contents = ('HTTP/1.1 200 OK\r\n\r\n') + proxy.write_as_browser(req_contents) + assert proxy.read_as_server() == req_contents + proxy.write_as_server(rsp_contents) + assert proxy.read_as_browser() == rsp_contents + assert proxy.req_save.called + proxy.get_endpoint.assert_called_with('www.AAAA.BBBB', 443, True, socks_config=None, use_http_proxy=True) + +def test_proxy_server_connect_uri(mocker): + proxy = TestProxyConnection() + proxy.setUp(mocker) + proxy.write_as_browser('CONNECT https://www.AAAA.BBBB:443 HTTP/1.1\r\n\r\n') + proxy.read_as_browser() + req_contents = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'ABCD') + proxy.write_as_browser(req_contents) + assert proxy.client_protocol.transport.startTLS.called + assert proxy.client_factory.request.host == 'www.AAAA.BBBB' + assert proxy.client_factory.request.port == 443 + assert proxy.client_factory.request.is_ssl == True + assert proxy.read_as_server() == req_contents + assert proxy.client_protocol.transport.startTLS.called + assert proxy.req_save.called + proxy.get_endpoint.assert_called_with('www.AAAA.BBBB', 443, True, socks_config=None, use_http_proxy=True) + +def test_proxy_server_connect_uri_alt_port(mocker): + proxy = TestProxyConnection() + proxy.setUp(mocker) + proxy.write_as_browser('CONNECT https://www.AAAA.BBBB:80085 HTTP/1.1\r\n\r\n') + proxy.read_as_browser() + req_contents = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'ABCD') + proxy.write_as_browser(req_contents) + assert proxy.client_factory.request.host == 'www.AAAA.BBBB' + assert proxy.client_factory.request.port == 80085 + assert proxy.client_factory.request.is_ssl == True + assert proxy.req_save.called + proxy.get_endpoint.assert_called_with('www.AAAA.BBBB', 80085, True, socks_config=None, use_http_proxy=True) + +def test_proxy_server_socks_basic(mocker): + proxy = TestProxyConnection() + proxy.setUp(mocker, socks_config={'host': 'www.banana.faketld', 'port': 1337}) + proxy.write_as_browser('CONNECT https://www.AAAA.BBBB:80085 HTTP/1.1\r\n\r\n') + proxy.read_as_browser() + req_contents = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'ABCD') + proxy.write_as_browser(req_contents) + proxy.get_endpoint.assert_called_with('www.AAAA.BBBB', 80085, True, + socks_config={'host':'www.banana.faketld', 'port':1337}, + use_http_proxy=True) + +def test_proxy_server_http_basic(mocker): + proxy = TestProxyConnection() + proxy.setUp(mocker, http_config={'host': 'www.banana.faketld', 'port': 1337}) + proxy.write_as_browser('CONNECT https://www.AAAA.BBBB:80085 HTTP/1.1\r\n\r\n') + proxy.read_as_browser() + req_contents = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'ABCD') + proxy.write_as_browser(req_contents) + assert proxy.req_save.called + proxy.get_endpoint.assert_called_with('www.AAAA.BBBB', 80085, True, + socks_config=None, + use_http_proxy=True) + +def test_proxy_server_360_noscope(mocker): + proxy = TestProxyConnection() + proxy.setUp(mocker, in_scope=False, socks_config={'host': 'www.banana.faketld', 'port': 1337}) + proxy.write_as_browser('CONNECT https://www.AAAA.BBBB:80085 HTTP/1.1\r\n\r\n') + proxy.read_as_browser() + req_contents = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'ABCD') + proxy.write_as_browser(req_contents) + assert not proxy.req_save.called + proxy.get_endpoint.assert_called_with('www.AAAA.BBBB', 80085, True, + socks_config=None, + use_http_proxy=False) - -@pytest.inlineCallbacks -def test_proxy_client_stream(mocker): - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - req = http.Request('GET / HTTP/1.1\r\n\r\n') - client = yield gen_client_protocol(req, stream_response=True) - client.transport.clear() - client.dataReceived('HTTP/1.1 404 GET FUCKE') - assert client.factory.return_transport.value() == 'HTTP/1.1 404 GET FUCKE' - client.factory.return_transport.clear() - client.dataReceived('D ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD') - assert client.factory.return_transport.value() == 'D ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD' - retpair = yield client.data_defer - assert retpair.response.full_message == 'HTTP/1.1 404 GET FUCKED ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD' - - -@pytest.inlineCallbacks -def test_proxy_client_nostream(mocker): - rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) - req = http.Request('GET / HTTP/1.1\r\n\r\n') - client = yield gen_client_protocol(req, stream_response=False) - client.transport.clear() - client.dataReceived('HTTP/1.1 404 GET FUCKE') - assert client.factory.return_transport.value() == '' - client.factory.return_transport.clear() - client.dataReceived('D ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD') - assert client.factory.return_transport.value() == '' - retpair = yield client.data_defer - assert retpair.response.full_message == 'HTTP/1.1 404 GET FUCKED ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD' - +def test_proxy_server_macro_simple(mocker): + proxy = TestProxyConnection() + + new_req_contents = 'GET / HTTP/1.1\r\nMangled: Very yes\r\n\r\n' + new_rsp_contents = 'HTTP/1.1 200 OKILIE DOKILIE\r\nMangled: Very yes\r\n\r\n' + new_req = Request(new_req_contents) + new_rsp = Response(new_rsp_contents) + test_macro = InterceptMacroTest(new_req=new_req, new_rsp=new_rsp) + proxy.setUp(mocker, int_macros={'test_macro': test_macro}) + proxy.write_as_browser('GET /serious.php HTTP/1.1\r\n\r\n') + assert proxy.read_as_server() == new_req_contents + proxy.write_as_server('HTTP/1.1 404 NOT FOUND\r\n\r\n') + assert proxy.read_as_browser() == new_rsp_contents + +def test_proxy_server_macro_multiple(mocker): + proxy = TestProxyConnection() + + new_req_contents1 = 'GET / HTTP/1.1\r\nMangled: Very yes\r\n\r\n' + new_rsp_contents1 = 'HTTP/1.1 200 OKILIE DOKILIE\r\nMangled: Very yes\r\n\r\n' + new_req1 = Request(new_req_contents1) + new_rsp1 = Response(new_rsp_contents1) + + new_req_contents2 = 'GET / HTTP/1.1\r\nMangled: Very very yes\r\n\r\n' + new_rsp_contents2 = 'HTTP/1.1 200 OKILIE DOKILIE\r\nMangled: Very very yes\r\n\r\n' + new_req2 = Request(new_req_contents2) + new_rsp2 = Response(new_rsp_contents2) + + test_macro1 = InterceptMacroTest(new_req=new_req1, new_rsp=new_rsp1) + test_macro2 = InterceptMacroTest(new_req=new_req2, new_rsp=new_rsp2) + + macros = collections.OrderedDict() + macros['macro1'] = test_macro1 + macros['macro2'] = test_macro2 + + proxy.setUp(mocker, int_macros=macros) + proxy.write_as_browser('GET /serious.php HTTP/1.1\r\n\r\n') + assert proxy.read_as_server() == new_req_contents2 + proxy.write_as_server('HTTP/1.1 404 NOT FOUND\r\n\r\n') + assert proxy.read_as_browser() == new_rsp_contents2 + +def test_proxy_server_macro_360_noscope(mocker): + proxy = TestProxyConnection() + + new_req_contents = 'GET / HTTP/1.1\r\nMangled: Very yes\r\n\r\n' + new_rsp_contents = 'HTTP/1.1 200 OKILIE DOKILIE\r\nMangled: Very yes\r\n\r\n' + new_req = Request(new_req_contents) + new_rsp = Response(new_rsp_contents) + test_macro = InterceptMacroTest(new_req=new_req, new_rsp=new_rsp) + proxy.setUp(mocker, int_macros={'test_macro': test_macro}, in_scope=False) + proxy.write_as_browser('GET /serious.php HTTP/1.1\r\n\r\n') + assert proxy.read_as_server() == 'GET /serious.php HTTP/1.1\r\n\r\n' + proxy.write_as_server('HTTP/1.1 404 NOT FOUND\r\n\r\n') + assert proxy.read_as_browser() == 'HTTP/1.1 404 NOT FOUND\r\n\r\n' + +def test_proxy_server_stream_simple(mocker): + proxy = TestProxyConnection() + proxy.setUp(mocker) + req_contents = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + 'Host: www.AAAA.BBBB\r\n' + '\r\n' + 'ABCD') + rsp_contents = ('HTTP/1.1 200 OK\r\n\r\n') + proxy.write_as_browser(req_contents) + assert proxy.read_as_server() == req_contents + proxy.write_as_server(rsp_contents[:20]) + assert proxy.read_as_browser() == rsp_contents[:20] + proxy.write_as_server(rsp_contents[20:]) + assert proxy.read_as_browser() == rsp_contents[20:] + +def test_proxy_server_macro_stream(mocker): + proxy = TestProxyConnection() + + new_req_contents = 'GET / HTTP/1.1\r\nMangled: Very yes\r\n\r\n' + new_rsp_contents = 'HTTP/1.1 200 OKILIE DOKILIE\r\nMangled: Very yes\r\n\r\n' + new_req = Request(new_req_contents) + new_rsp = Response(new_rsp_contents) + test_macro = InterceptMacroTest(new_req=new_req, new_rsp=new_rsp) + proxy.setUp(mocker, int_macros={'test_macro': test_macro}) + proxy.write_as_browser('GET /serious.php HTTP/1.1\r\n\r\n') + assert proxy.read_as_server() == new_req_contents + proxy.write_as_server('HTTP/1.1 404 ') + assert proxy.read_as_browser() == '' + proxy.write_as_server('NOT FOUND\r\n\r\n') + assert proxy.read_as_browser() == new_rsp_contents + +# It doesn't stream if out of scope and macros are active, but whatever. +# def test_proxy_server_macro_stream_360_noscope(mocker): +# proxy = TestProxyConnection() + +# new_req_contents = 'GET / HTTP/1.1\r\nMangled: Very yes\r\n\r\n' +# new_rsp_contents = 'HTTP/1.1 200 OKILIE DOKILIE\r\nMangled: Very yes\r\n\r\n' +# new_req = Request(new_req_contents) +# new_rsp = Response(new_rsp_contents) +# test_macro = InterceptMacroTest(new_req=new_req, new_rsp=new_rsp) +# proxy.setUp(mocker, int_macros={'test_macro': test_macro}, in_scope=False) +# proxy.write_as_browser('GET /serious.php HTTP/1.1\r\n\r\n') +# assert proxy.read_as_server() == 'GET /serious.php HTTP/1.1\r\n\r\n' +# proxy.write_as_server('HTTP/1.1 404 ') +# assert proxy.read_as_browser() == 'HTTP/1.1 404 ' +# proxy.write_as_server('NOT FOUND\r\n\r\n') +# assert proxy.read_as_browser() == 'NOT FOUND\r\n\r\n' diff --git a/pappyproxy/tests/testutil.py b/pappyproxy/tests/testutil.py index fd64e7d..1893d43 100644 --- a/pappyproxy/tests/testutil.py +++ b/pappyproxy/tests/testutil.py @@ -12,9 +12,7 @@ class ClassDeleted(): pass class TLSStringTransport(StringTransport): - - def startTLS(self, context, factory): - pass + startTLS = mock.MagicMock() class PappySession(object): diff --git a/pappyproxy/util.py b/pappyproxy/util.py index 74532ea..b761a1c 100644 --- a/pappyproxy/util.py +++ b/pappyproxy/util.py @@ -119,12 +119,21 @@ def load_reqlist(line, allow_special=True, ids_only=False): :Returns: Twisted deferred """ from .http import Request + from .plugin import async_main_context_ids # Parses a comma separated list of ids and returns a list of those requests # prints any errors if not line: raise PappyException('Request id(s) required') - ids = re.split(',\s*', line) reqs = [] + + if line == '*': + ids = yield async_main_context_ids() + for i in ids: + req = yield Request.load_request(i) + reqs.append(req) + defer.returnValue(reqs) + + ids = re.split(',\s*', line) if not ids_only: for reqid in ids: try: @@ -336,3 +345,14 @@ def copy_to_clipboard(text): def clipboard_contents(): return pyperclip.paste() + +def autocomplete_startswith(text, lst, allow_spaces=False): + ret = None + if not text: + ret = lst[:] + else: + ret = [n for n in lst if n.startswith(text)] + if not allow_spaces: + ret = [s for s in ret if ' ' not in s] + return ret + diff --git a/setup.py b/setup.py index 6fc998c..d621644 100755 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ setup(name='pappyproxy', 'cmd2>=0.6.8', 'crochet>=1.4.0', 'Jinja2>=2.8', + 'lxml>=3.6.0', 'pygments>=2.0.2', 'pyperclip>=1.5.26', 'pytest-cov>=2.2.0',