diff --git a/README.md b/README.md index 4c09c16..53f0933 100644 --- a/README.md +++ b/README.md @@ -1123,6 +1123,8 @@ Changelog --------- The boring part of the readme +* 0.2.9 + * Fix bugs/clean up some code * 0.2.8 * Upstream HTTP proxy support * Usability improvements diff --git a/pappyproxy/__init__.py b/pappyproxy/__init__.py index 14e974f..cd9b137 100644 --- a/pappyproxy/__init__.py +++ b/pappyproxy/__init__.py @@ -1 +1 @@ -__version__ = '0.2.8' +__version__ = '0.2.9' diff --git a/pappyproxy/comm.py b/pappyproxy/comm.py index c8be914..499ce76 100644 --- a/pappyproxy/comm.py +++ b/pappyproxy/comm.py @@ -31,21 +31,21 @@ class CommServer(LineReceiver): if line == '': return - try: - command_data = json.loads(line) - command = command_data['action'] - valid = False - if command in self.action_handlers: - valid = True - result = {'success': True} - func_defer = self.action_handlers[command](command_data) - func_defer.addCallback(self.action_result_handler, result) - func_defer.addErrback(self.action_error_handler, result) - if not valid: - raise PappyException('%s is an invalid command' % command_data['action']) - except PappyException as e: - return_data = {'success': False, 'message': str(e)} - self.sendLine(json.dumps(return_data)) + #try: + command_data = json.loads(line) + command = command_data['action'] + valid = False + if command in self.action_handlers: + valid = True + result = {'success': True} + func_defer = self.action_handlers[command](command_data) + func_defer.addCallback(self.action_result_handler, result) + func_defer.addErrback(self.action_error_handler, result) + if not valid: + raise PappyException('%s is an invalid command' % command_data['action']) + # except PappyException as e: + # return_data = {'success': False, 'message': str(e)} + # self.sendLine(json.dumps(return_data)) def action_result_handler(self, data, result): result.update(data) @@ -94,11 +94,15 @@ class CommServer(LineReceiver): @defer.inlineCallbacks def action_submit_request(self, data): from .http import Request + from .plugin import active_intercepting_macros message = base64.b64decode(data['full_message']) - try: - req = yield Request.submit_new(data['host'].encode('utf-8'), data['port'], data['is_ssl'], message) - except Exception: - raise PappyException('Error submitting request. Please make sure request is a valid HTTP message.') + req = Request(message) + req.host = data['host'].encode('utf-8') + req.port = data['port'] + req.is_ssl = data['is_ssl'] + yield Request.submit_request(req, + save_request=True, + intercepting_macros=active_intercepting_macros()) if 'tags' in data: req.tags = set(data['tags']) yield req.async_deep_save() diff --git a/pappyproxy/context.py b/pappyproxy/context.py index 8ba020d..c26eaf2 100644 --- a/pappyproxy/context.py +++ b/pappyproxy/context.py @@ -375,7 +375,7 @@ def gen_filter_by_before(args): defer.returnValue(f) @defer.inlineCallbacks -def gen_filter_by_after(reqid, negate=False): +def gen_filter_by_after(args, negate=False): if len(args) != 1: raise PappyException('Invalid number of arguments') r = yield Request.load_request(args[0]) diff --git a/pappyproxy/http.py b/pappyproxy/http.py index 23b31bf..703ef2f 100644 --- a/pappyproxy/http.py +++ b/pappyproxy/http.py @@ -2130,60 +2130,74 @@ class Request(HTTPMessage): @staticmethod @defer.inlineCallbacks - def submit_new(host, port, is_ssl, full_request): + def submit_request(request, + save_request=False, + intercepting_macros={}, + stream_transport=None): """ - submit_new(host, port, is_ssl, full_request) - Submits a request with the given parameters and returns a request object - with the response. + submit_request(request, save_request=False, intercepting_macros={}, stream_transport=None) - :param host: The host to submit to - :type host: string - :param port: The port to submit to - :type port: Integer - :type is_ssl: Whether to use SSL - :param full_request: The request data to send - :type full_request: string - :rtype: Twisted deferred that calls back with a Request + Submits the request then sets ``request.response``. Returns a deferred that + is called with the request that was submitted. + + :param request: The request to submit + :type host: Request + :param save_request: Whether to save the request to history + :type save_request: Bool + :param intercepting_macros: Dictionary of intercepting macros to be applied to the request + :type intercepting_macros: Dict or collections.OrderedDict + :param stream_transport: Return transport to stream to. Set to None to not stream the response. + :type stream_transport: twisted.internet.interfaces.ITransport """ + from .proxy import ProxyClientFactory, get_next_connection_id, get_endpoint from .pappy import session - new_req = Request(full_request) - new_req.is_ssl = is_ssl - new_req.port = port - new_req._host = host - - factory = ProxyClientFactory(new_req, save_all=False, stream_response=False, return_transport=None) - factory.intercepting_macros = {} + factory = None + if stream_transport is None: + factory = ProxyClientFactory(request, + save_all=save_request, + stream_response=False, + return_transport=None) + else: + factory = ProxyClientFactory(request, + save_all=save_request, + stream_response=True, + return_transport=stream_transport) + factory.intercepting_macros = intercepting_macros factory.connection_id = get_next_connection_id() - yield factory.prepare_request() - endpoint = get_endpoint(host, port, is_ssl, - socks_config=session.config.socks_proxy) - yield endpoint.connect(factory) + factory.connect() new_req = yield factory.data_defer - defer.returnValue(new_req) + request.response = new_req.response + defer.returnValue(request) @defer.inlineCallbacks - def async_submit(self): + def async_submit(self, mangle=False): """ async_submit() Same as :func:`~pappyproxy.http.Request.submit` but generates deferreds. Submits the request using its host, port, etc. and updates its response value to the resulting response. + :param mangle: Whether to pass the request through active intercepting macros. + :type mangle: Bool + :rtype: Twisted deferred """ - new_req = yield Request.submit_new(self.host, self.port, self.is_ssl, - self.full_request) - self.set_metadata(new_req.get_metadata()) - self.unmangled = new_req.unmangled - self.response = new_req.response - self.time_start = new_req.time_start - self.time_end = new_req.time_end + from pappyproxy.plugin import active_intercepting_macros + + if mangle: + int_macros = active_intercepting_macros() + else: + int_macros = None + yield Request.submit_request(self, + save_request=False, + intercepting_macros=int_macros, + stream_transport=None) @crochet.wait_for(timeout=180.0) @defer.inlineCallbacks - def submit(self): + def submit(self, mangle=False): """ submit() Submits the request using its host, port, etc. and updates its response value @@ -2191,7 +2205,7 @@ class Request(HTTPMessage): Cannot be called in async functions. This is what you should use to submit your requests in macros. """ - yield self.async_submit() + yield self.async_submit(mangle=mangle) class Response(HTTPMessage): diff --git a/pappyproxy/plugin.py b/pappyproxy/plugin.py index 0bbdb1e..4325d35 100644 --- a/pappyproxy/plugin.py +++ b/pappyproxy/plugin.py @@ -107,12 +107,13 @@ def remove_intercepting_macro(name): def active_intercepting_macros(): """ - Returns a list of the active intercepting macro objects. Modifying + Returns a dict of the active intercepting macro objects. Modifying this list will not affect which macros are active. """ - ret = [] + ret = {} for factory in pappyproxy.pappy.session.server_factories: - ret += [v for k, v in factory.intercepting_macros.iteritems() ] + for k, v in factory.intercepting_macros.iteritems(): + ret[k] = v return ret def in_memory_reqs(): diff --git a/pappyproxy/plugins/decode.py b/pappyproxy/plugins/decode.py index 9072d61..5479126 100644 --- a/pappyproxy/plugins/decode.py +++ b/pappyproxy/plugins/decode.py @@ -1,14 +1,13 @@ import HTMLParser import StringIO import base64 -import clipboard import datetime import gzip import shlex import string import urllib -from pappyproxy.util import PappyException, hexdump, printable_data +from pappyproxy.util import PappyException, hexdump, printable_data, copy_to_clipboard, clipboard_contents def print_maybe_bin(s): binary = False @@ -45,6 +44,18 @@ def gzip_decode_helper(s): dec_data = dec_data.read() return dec_data +def base64_decode_helper(s): + try: + return base64.b64decode(s) + except TypeError: + for i in range(1, 5): + try: + s_padded = base64.b64decode(s + '='*i) + return s_padded + except: + pass + raise PappyException("Unable to base64 decode string") + def html_encode_helper(s): return ''.join(['&#x{0:x};'.format(ord(c)) for c in s]) @@ -54,13 +65,13 @@ def html_decode_helper(s): def _code_helper(line, func, copy=True): args = shlex.split(line) if not args: - s = clipboard.paste() + s = clipboard_contents() print 'Will decode:' print printable_data(s) s = func(s) if copy: try: - clipboard.copy(s) + copy_to_clipboard(s) except: print 'Result cannot be copied to the clipboard. Result not copied.' return s @@ -68,7 +79,7 @@ def _code_helper(line, func, copy=True): s = func(args[0].strip()) if copy: try: - clipboard.copy(s) + copy_to_clipboard(s) except: print 'Result cannot be copied to the clipboard. Result not copied.' return s @@ -79,7 +90,7 @@ def base64_decode(line): If no string is given, will decode the contents of the clipboard. Results are copied to the clipboard. """ - print_maybe_bin(_code_helper(line, base64.b64decode)) + print_maybe_bin(_code_helper(line, base64_decode_helper)) def base64_encode(line): """ @@ -159,7 +170,7 @@ def base64_decode_raw(line): results will not be copied. It is suggested you redirect the output to a file. """ - print _code_helper(line, base64.b64decode, copy=False) + print _code_helper(line, base64_decode_helper, copy=False) def base64_encode_raw(line): """ diff --git a/pappyproxy/plugins/macrocmds.py b/pappyproxy/plugins/macrocmds.py index 0f0eda4..427ad3d 100644 --- a/pappyproxy/plugins/macrocmds.py +++ b/pappyproxy/plugins/macrocmds.py @@ -127,7 +127,7 @@ def list_int_macros(line): running = [] not_running = [] for macro in loaded_int_macros: - if macro.name in [m.name for m in active_intercepting_macros()]: + if macro.name in [m.name for k, m in active_intercepting_macros().iteritems()]: running.append(macro) else: not_running.append(macro) diff --git a/pappyproxy/plugins/misc.py b/pappyproxy/plugins/misc.py index 5f0bc32..87934de 100644 --- a/pappyproxy/plugins/misc.py +++ b/pappyproxy/plugins/misc.py @@ -1,6 +1,7 @@ import crochet import pappyproxy import shlex +import sys from pappyproxy.colors import Colors, Styles, path_formatter, host_color, scode_color, verb_color from pappyproxy.util import PappyException, remove_color, confirm, load_reqlist, Capturing @@ -34,6 +35,7 @@ class PrintStreamInterceptMacro(InterceptMacro): s += req.url_color s += ', len=' + str(len(req.body)) print s + sys.stdout.flush() @staticmethod def _print_response(req): @@ -47,6 +49,7 @@ class PrintStreamInterceptMacro(InterceptMacro): s += req.url_color s += ', len=' + str(len(req.response.body)) print s + sys.stdout.flush() def mangle_request(self, request): PrintStreamInterceptMacro._print_request(request) diff --git a/pappyproxy/proxy.py b/pappyproxy/proxy.py index ffa749d..1e86ce7 100644 --- a/pappyproxy/proxy.py +++ b/pappyproxy/proxy.py @@ -62,11 +62,20 @@ def log_request(request, id=None, symbol='*', verbosity_level=3): for l in r_split: log(l, id, symbol, verbosity_level) -def get_endpoint(target_host, target_port, target_ssl, socks_config=None): +def get_endpoint(target_host, target_port, target_ssl, socks_config=None, use_http_proxy=False, debugid=None): # Imports go here to allow mocking for tests from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint from txsocksx.client import SOCKS5ClientEndpoint from txsocksx.tls import TLSWrapClientEndpoint + from pappyproxy.pappy import session + + log("Getting endpoint for host '%s' on port %d ssl=%s, socks_config=%s, use_http_proxy=%s" % (target_host, target_port, target_ssl, str(socks_config), use_http_proxy), id=debugid, verbosity_level=3) + + if session.config.http_proxy and use_http_proxy: + target_host = session.config.http_proxy['host'] + target_port = session.config.http_proxy['port'] + target_ssl = False # We turn on ssl after CONNECT request if needed + log("Connecting to http proxy at %s:%d" % (target_host, target_port), id=debugid, verbosity_level=3) if socks_config is not None: sock_host = socks_config['host'] @@ -183,7 +192,7 @@ class UpstreamHTTPProxyClient(ProxyClient): sendreq.proxy_creds = self.creds lines = sendreq.full_request.splitlines() for l in lines: - self.log(l, symbol='>r', verbosity_level=3) + self.log(l, symbol='>rp', verbosity_level=3) self.transport.write(sendreq.full_message) def connectionMade(self): @@ -194,10 +203,16 @@ class UpstreamHTTPProxyClient(ProxyClient): self.connect_response = True if self.creds is not None: connreq.proxy_creds = self.creds + lines = connreq.full_message.splitlines() + for l in lines: + self.log(l, symbol='>p', verbosity_level=3) self.transport.write(connreq.full_message) else: self.proxy_connected = True self.stream_response = True + lines = self.request.full_message.splitlines() + for l in lines: + self.log(l, symbol='>p', verbosity_level=3) self.write_proxied_request(self.request) def handle_response_end(self, *args, **kwargs): @@ -211,6 +226,10 @@ class UpstreamHTTPProxyClient(ProxyClient): self.transport.loseConnection() assert self._response_obj.full_response self.data_defer.callback(self.request) + elif self._response_obj.response_code != 200: + print "Error establishing connection to proxy" + self.transport.loseConnection() + return elif self.connect_response: self.log("Response to CONNECT request recieved from http proxy", verbosity_level=3) self.proxy_connected = True @@ -220,10 +239,12 @@ class UpstreamHTTPProxyClient(ProxyClient): self.completed = False self._sent = False + self.log("Starting TLS", verbosity_level=3) self.transport.startTLS(ClientTLSContext()) + self.log("TLS started", verbosity_level=3) lines = self.request.full_message.splitlines() for l in lines: - self.log(l, symbol='>r', verbosity_level=3) + self.log(l, symbol='>rpr', verbosity_level=3) self.transport.write(self.request.full_message) class ProxyClientFactory(ClientFactory): @@ -240,6 +261,7 @@ class ProxyClientFactory(ClientFactory): self.return_transport = return_transport self.intercepting_macros = {} self.use_as_proxy = False + self.sendback_function = None def log(self, message, symbol='*', verbosity_level=1): log(message, id=self.connection_id, symbol=symbol, verbosity_level=verbosity_level) @@ -297,6 +319,8 @@ class ProxyClientFactory(ClientFactory): if session.config.http_proxy: self.use_as_proxy = True + if (not self.stream_response) and self.sendback_function: + self.data_defer.addCallback(self.sendback_function) else: self.log("Request out of scope, passing along unmangled") self.request = sendreq @@ -340,6 +364,29 @@ class ProxyClientFactory(ClientFactory): self.data_defer.callback(request) defer.returnValue(None) + @defer.inlineCallbacks + def connect(self): + from pappyproxy.pappy import session + + yield self.prepare_request() + if context.in_scope(self.request): + # Get connection using config + endpoint = get_endpoint(self.request.host, + self.request.port, + self.request.is_ssl, + socks_config=session.config.socks_proxy, + use_http_proxy=True) + else: + # Just forward it normally + endpoint = get_endpoint(self.request.host, + self.request.port, + self.request.is_ssl) + + # Connect via the endpoint + self.log("Accessing using endpoint") + yield endpoint.connect(self) + self.log("Connected") + class ProxyServerFactory(ServerFactory): def __init__(self, save_all=False): @@ -425,6 +472,8 @@ class ProxyServer(LineReceiver): @defer.inlineCallbacks def full_request_received(self): + from pappyproxy.http import Request + global cached_certs self.log('End of request', verbosity_level=3) @@ -447,8 +496,18 @@ class ProxyServer(LineReceiver): # if _request_obj.host is a listener, forward = False + if self.factory.intercepting_macros: + return_transport = None + else: + return_transport = self.transport + if forward: - self._generate_and_submit_client() + d = Request.submit_request(self._request_obj, + save_request=True, + intercepting_macros=self.factory.intercepting_macros, + stream_transport=return_transport) + if return_transport is None: + d.addCallback(self.send_response_back) self._reset() def _reset(self): @@ -467,73 +526,9 @@ class ProxyServer(LineReceiver): self._request_obj.port = self._connect_port self.setLineMode() - def _generate_and_submit_client(self): - """ - Sets up self._client_factory with self._request_obj then calls back to - submit the request - """ - - self.log("Forwarding to %s on %d" % (self._request_obj.host, self._request_obj.port)) - if self.factory.intercepting_macros: - stream = False - else: - stream = True - self.log('Creating client factory, stream=%s' % stream) - self._client_factory = ProxyClientFactory(self._request_obj, - save_all=self.factory.save_all, - stream_response=stream, - return_transport=self.transport) - self._client_factory.intercepting_macros = self.factory.intercepting_macros - self._client_factory.connection_id = self.connection_id - if not stream: - self._client_factory.data_defer.addCallback(self.send_response_back) - d = self._client_factory.prepare_request() - d.addCallback(self._make_remote_connection) - return d - - @defer.inlineCallbacks - def _make_remote_connection(self, req): - """ - Creates an endpoint to the target server using the given configuration - options then connects to the endpoint using self._client_factory - """ - from pappyproxy.pappy import session - - self._request_obj = req - - # If we have a socks proxy, wrap the endpoint in it - if context.in_scope(self._request_obj): - # Modify the request connection settings to match settings in the factory - if self.factory.force_ssl: - self._request_obj.is_ssl = True - if self.factory.forward_host: - self._request_obj.host = self.factory.forward_host - - usehost = self._request_obj.host - useport = self._request_obj.port - usessl = self._request_obj.is_ssl - if session.config.http_proxy: - usehost = session.config.http_proxy['host'] - useport = session.config.http_proxy['port'] - usessl = False # We turn on ssl after CONNECT request if needed - self.log("Connecting to http proxy at %s:%d" % (usehost, useport)) - - # Get connection from the request - endpoint = get_endpoint(usehost, useport, usessl, - socks_config=session.config.socks_proxy) - else: - endpoint = get_endpoint(self._request_obj.host, - self._request_obj.port, - self._request_obj.is_ssl) - - # Connect via the endpoint - self.log("Accessing using endpoint") - yield endpoint.connect(self._client_factory) - self.log("Connected") - - def send_response_back(self, response): - if response is not None: - self.transport.write(response.response.full_response) + def send_response_back(self, request): + if request.response is not None: + self.transport.write(request.response.full_response) self.log("Response sent back, losing connection") self.transport.loseConnection() diff --git a/pappyproxy/site/static/dickbutt.jpg b/pappyproxy/site/static/dickbutt.jpg new file mode 100644 index 0000000..3a3d983 Binary files /dev/null and b/pappyproxy/site/static/dickbutt.jpg differ diff --git a/pappyproxy/tests/test_comm.py b/pappyproxy/tests/test_comm.py index 3a046d8..c217358 100644 --- a/pappyproxy/tests/test_comm.py +++ b/pappyproxy/tests/test_comm.py @@ -10,6 +10,10 @@ from pappyproxy.comm import CommServer from pappyproxy.http import Request, Response from testutil import mock_deferred, func_deleted, TLSStringTransport, freeze, mock_int_macro, no_tcp +@pytest.fixture(autouse=True) +def no_int_macros(mocker): + mocker.patch('pappyproxy.plugin.active_intercepting_macros').return_value = {} + @pytest.fixture def http_request(): req = Request('GET / HTTP/1.1\r\n\r\n') @@ -42,6 +46,13 @@ def mock_loader(rsp): return rsp return classmethod(f) +def mock_submitter(rsp): + def f(_, req, *args, **kwargs): + req.response = rsp + req.reqid = 123 + return mock_deferred(req) + return classmethod(f) + def mock_loader_fail(): def f(*args, **kwargs): raise PappyException("lololo message don't exist dawg") @@ -80,7 +91,8 @@ def test_get_response_fail(mocker, http_request): assert 'message' in v def test_submit_request(mocker, http_request): - mocker.patch.object(pappyproxy.http.Request, 'submit_new', new=mock_loader(http_request)) + rsp = Response('HTTP/1.1 200 OK\r\n\r\n') + mocker.patch.object(pappyproxy.http.Request, 'submit_request', new=mock_submitter(rsp)) mocker.patch('pappyproxy.http.Request.async_deep_save').return_value = mock_deferred() comm_data = {"action": "submit"} @@ -92,14 +104,14 @@ def test_submit_request(mocker, http_request): v = perform_comm(json.dumps(comm_data)) expected_data = {} - expected_data['request'] = json.loads(http_request.to_json()) - expected_data['response'] = json.loads(http_request.response.to_json()) - expected_data['success'] = True - expected_data['request']['tags'] = ['footag'] + expected_data[u'request'] = json.loads(http_request.to_json()) + expected_data[u'response'] = json.loads(http_request.response.to_json()) + expected_data[u'success'] = True + expected_data[u'request'][u'tags'] = [u'footag'] assert json.loads(v) == expected_data def test_submit_request_fail(mocker, http_request): - mocker.patch.object(pappyproxy.http.Request, 'submit_new', new=mock_loader_fail()) + mocker.patch.object(pappyproxy.http.Request, 'submit_request', new=mock_loader_fail()) mocker.patch('pappyproxy.http.Request.async_deep_save').return_value = mock_deferred() comm_data = {"action": "submit"} diff --git a/pappyproxy/util.py b/pappyproxy/util.py index 8819ee9..74532ea 100644 --- a/pappyproxy/util.py +++ b/pappyproxy/util.py @@ -4,11 +4,20 @@ import re import string import sys import time +import pyperclip from .colors import Styles, Colors, verb_color, scode_color, path_formatter, host_color from twisted.internet import defer from twisted.test.proto_helpers import StringTransport +try: + # If you don't do this then pyperclip imports gtk, it blocks the twisted reactor. + # Dumb. I know. + import gtk + gtk.set_interactive(False) +except ImportError: + pass + class PappyException(Exception): """ The exception class for Pappy. If a plugin command raises one of these, the @@ -320,3 +329,10 @@ def confirm(message, default='n'): return True else: return False + + +def copy_to_clipboard(text): + pyperclip.copy(text) + +def clipboard_contents(): + return pyperclip.paste() diff --git a/setup.py b/setup.py index 56a3642..6fc998c 100755 --- a/setup.py +++ b/setup.py @@ -23,11 +23,11 @@ setup(name='pappyproxy', download_url='https://github.com/roglew/pappy-proxy/archive/%s.tar.gz'%VERSION, install_requires=[ 'beautifulsoup4>=4.4.1', - 'clipboard>=0.0.4', 'cmd2>=0.6.8', 'crochet>=1.4.0', 'Jinja2>=2.8', 'pygments>=2.0.2', + 'pyperclip>=1.5.26', 'pytest-cov>=2.2.0', 'pytest-mock>=0.9.0', 'pytest-twisted>=1.5',