Version 0.2.9

README.md

@@ -1123,6 +1123,8 @@ Changelog
 ---------
 The boring part of the readme
 
+* 0.2.9
+    * Fix bugs/clean up some code
 * 0.2.8
     * Upstream HTTP proxy support
     * Usability improvements

pappyproxy/__init__.py

@@ -1 +1 @@
-__version__ = '0.2.8'
+__version__ = '0.2.9'

pappyproxy/comm.py

@@ -31,7 +31,7 @@ class CommServer(LineReceiver):
 
         if line == '':
             return
-        try:
+        #try:
         command_data = json.loads(line)
         command = command_data['action']
         valid = False
@@ -43,9 +43,9 @@ class CommServer(LineReceiver):
             func_defer.addErrback(self.action_error_handler, result)
         if not valid:
             raise PappyException('%s is an invalid command' % command_data['action'])
-        except PappyException as e:
+        # except PappyException as e:
-            return_data = {'success': False, 'message': str(e)}
+        #     return_data = {'success': False, 'message': str(e)}
-            self.sendLine(json.dumps(return_data))
+        #     self.sendLine(json.dumps(return_data))
 
     def action_result_handler(self, data, result):
         result.update(data)
@@ -94,11 +94,15 @@ class CommServer(LineReceiver):
     @defer.inlineCallbacks
     def action_submit_request(self, data):
         from .http import Request
+        from .plugin import active_intercepting_macros
         message = base64.b64decode(data['full_message'])
-        try:
+        req = Request(message)
-            req = yield Request.submit_new(data['host'].encode('utf-8'), data['port'], data['is_ssl'], message)
+        req.host = data['host'].encode('utf-8')
-        except Exception:
+        req.port = data['port']
-            raise PappyException('Error submitting request. Please make sure request is a valid HTTP message.')
+        req.is_ssl = data['is_ssl']
+        yield Request.submit_request(req,
+                                     save_request=True,
+                                     intercepting_macros=active_intercepting_macros())
         if 'tags' in data:
             req.tags = set(data['tags'])
         yield req.async_deep_save()

pappyproxy/context.py

@@ -375,7 +375,7 @@ def gen_filter_by_before(args):
     defer.returnValue(f)
 
 @defer.inlineCallbacks
-def gen_filter_by_after(reqid, negate=False):
+def gen_filter_by_after(args, negate=False):
     if len(args) != 1:
         raise PappyException('Invalid number of arguments')
     r = yield Request.load_request(args[0])

pappyproxy/http.py

@@ -2130,60 +2130,74 @@ class Request(HTTPMessage):
         
     @staticmethod
     @defer.inlineCallbacks
-    def submit_new(host, port, is_ssl, full_request):
+    def submit_request(request,
+                       save_request=False,
+                       intercepting_macros={},
+                       stream_transport=None):
         """
-        submit_new(host, port, is_ssl, full_request)
+        submit_request(request, save_request=False, intercepting_macros={}, stream_transport=None)
-        Submits a request with the given parameters and returns a request object
-        with the response.
 
-        :param host: The host to submit to
+        Submits the request then sets ``request.response``. Returns a deferred that
-        :type host: string
+        is called with the request that was submitted.
-        :param port: The port to submit to
+
-        :type port: Integer
+        :param request: The request to submit
-        :type is_ssl: Whether to use SSL
+        :type host: Request
-        :param full_request: The request data to send
+        :param save_request: Whether to save the request to history
-        :type full_request: string
+        :type save_request: Bool
-        :rtype: Twisted deferred that calls back with a Request
+        :param intercepting_macros: Dictionary of intercepting macros to be applied to the request
+        :type intercepting_macros: Dict or collections.OrderedDict
+        :param stream_transport: Return transport to stream to. Set to None to not stream the response.
+        :type stream_transport: twisted.internet.interfaces.ITransport
         """
+
         from .proxy import ProxyClientFactory, get_next_connection_id, get_endpoint
         from .pappy import session
 
-        new_req = Request(full_request)
+        factory = None
-        new_req.is_ssl = is_ssl
+        if stream_transport is None:
-        new_req.port = port
+            factory = ProxyClientFactory(request,
-        new_req._host = host
+                                        save_all=save_request,
-
+                                        stream_response=False,
-        factory = ProxyClientFactory(new_req, save_all=False, stream_response=False, return_transport=None)
+                                        return_transport=None)
-        factory.intercepting_macros = {}
+        else:
+            factory = ProxyClientFactory(request,
+                                        save_all=save_request,
+                                        stream_response=True,
+                                        return_transport=stream_transport)
+        factory.intercepting_macros = intercepting_macros
         factory.connection_id = get_next_connection_id()
-        yield factory.prepare_request()
+        factory.connect()
-        endpoint = get_endpoint(host, port, is_ssl,
-                                socks_config=session.config.socks_proxy)
-        yield endpoint.connect(factory)
         new_req = yield factory.data_defer
-        defer.returnValue(new_req)
+        request.response = new_req.response
+        defer.returnValue(request)
 
     @defer.inlineCallbacks
-    def async_submit(self):
+    def async_submit(self, mangle=False):
         """
         async_submit()
         Same as :func:`~pappyproxy.http.Request.submit` but generates deferreds.
         Submits the request using its host, port, etc. and updates its response value
         to the resulting response.
 
+        :param mangle: Whether to pass the request through active intercepting macros.
+        :type mangle: Bool
+
         :rtype: Twisted deferred
         """
-        new_req = yield Request.submit_new(self.host, self.port, self.is_ssl,
+        from pappyproxy.plugin import active_intercepting_macros
-                                           self.full_request)
+        
-        self.set_metadata(new_req.get_metadata())
+        if mangle:
-        self.unmangled = new_req.unmangled
+            int_macros = active_intercepting_macros()
-        self.response = new_req.response
+        else:
-        self.time_start = new_req.time_start
+            int_macros = None
-        self.time_end = new_req.time_end
+        yield Request.submit_request(self,
+                                     save_request=False,
+                                     intercepting_macros=int_macros,
+                                     stream_transport=None)
 
     @crochet.wait_for(timeout=180.0)
     @defer.inlineCallbacks
-    def submit(self):
+    def submit(self, mangle=False):
         """
         submit()
         Submits the request using its host, port, etc. and updates its response value
@@ -2191,7 +2205,7 @@ class Request(HTTPMessage):
         Cannot be called in async functions.
         This is what you should use to submit your requests in macros.
         """
-        yield self.async_submit()
+        yield self.async_submit(mangle=mangle)
 
 
 class Response(HTTPMessage):

pappyproxy/plugin.py

@@ -107,12 +107,13 @@ def remove_intercepting_macro(name):
     
 def active_intercepting_macros():
     """
-    Returns a list of the active intercepting macro objects. Modifying
+    Returns a dict of the active intercepting macro objects. Modifying
     this list will not affect which macros are active.
     """
-    ret = []
+    ret = {}
     for factory in pappyproxy.pappy.session.server_factories:
-        ret += [v for k, v in factory.intercepting_macros.iteritems() ]
+        for k, v in factory.intercepting_macros.iteritems():
+            ret[k] = v
     return ret
 
 def in_memory_reqs():

pappyproxy/plugins/decode.py

@@ -1,14 +1,13 @@
 import HTMLParser
 import StringIO
 import base64
-import clipboard
 import datetime
 import gzip
 import shlex
 import string
 import urllib
 
-from pappyproxy.util import PappyException, hexdump, printable_data
+from pappyproxy.util import PappyException, hexdump, printable_data, copy_to_clipboard, clipboard_contents
 
 def print_maybe_bin(s):
     binary = False
@@ -45,6 +44,18 @@ def gzip_decode_helper(s):
     dec_data = dec_data.read()
     return dec_data
 
+def base64_decode_helper(s):
+    try:
+        return base64.b64decode(s)
+    except TypeError:
+        for i in range(1, 5):
+            try:
+                s_padded = base64.b64decode(s + '='*i)
+                return s_padded
+            except:
+                pass
+        raise PappyException("Unable to base64 decode string")
+
 def html_encode_helper(s):
     return ''.join(['&#x{0:x};'.format(ord(c)) for c in s])
 
@@ -54,13 +65,13 @@ def html_decode_helper(s):
 def _code_helper(line, func, copy=True):
     args = shlex.split(line)
     if not args:
-        s = clipboard.paste()
+        s = clipboard_contents()
         print 'Will decode:'
         print printable_data(s)
         s = func(s)
         if copy:
             try:
-                clipboard.copy(s)
+                copy_to_clipboard(s)
             except:
                 print 'Result cannot be copied to the clipboard. Result not copied.'
         return s
@@ -68,7 +79,7 @@ def _code_helper(line, func, copy=True):
         s = func(args[0].strip())
         if copy:
             try:
-                clipboard.copy(s)
+                copy_to_clipboard(s)
             except:
                 print 'Result cannot be copied to the clipboard. Result not copied.'
         return s
@@ -79,7 +90,7 @@ def base64_decode(line):
     If no string is given, will decode the contents of the clipboard.
     Results are copied to the clipboard.
     """
-    print_maybe_bin(_code_helper(line, base64.b64decode))
+    print_maybe_bin(_code_helper(line, base64_decode_helper))
 
 def base64_encode(line):
     """
@@ -159,7 +170,7 @@ def base64_decode_raw(line):
     results will not be copied. It is suggested you redirect the output
     to a file.
     """
-    print _code_helper(line, base64.b64decode, copy=False)
+    print _code_helper(line, base64_decode_helper, copy=False)
 
 def base64_encode_raw(line):
     """

pappyproxy/plugins/macrocmds.py

@@ -127,7 +127,7 @@ def list_int_macros(line):
     running = []
     not_running = []
     for macro in loaded_int_macros:
-        if macro.name in [m.name for m in active_intercepting_macros()]:
+        if macro.name in [m.name for k, m in active_intercepting_macros().iteritems()]:
             running.append(macro)
         else:
             not_running.append(macro)

pappyproxy/plugins/misc.py

@@ -1,6 +1,7 @@
 import crochet
 import pappyproxy
 import shlex
+import sys
 
 from pappyproxy.colors import Colors, Styles, path_formatter, host_color, scode_color, verb_color
 from pappyproxy.util import PappyException, remove_color, confirm, load_reqlist, Capturing
@@ -34,6 +35,7 @@ class PrintStreamInterceptMacro(InterceptMacro):
         s += req.url_color
         s += ', len=' + str(len(req.body))
         print s
+        sys.stdout.flush()
 
     @staticmethod
     def _print_response(req):
@@ -47,6 +49,7 @@ class PrintStreamInterceptMacro(InterceptMacro):
         s += req.url_color
         s += ', len=' + str(len(req.response.body))
         print s
+        sys.stdout.flush()
 
     def mangle_request(self, request):
         PrintStreamInterceptMacro._print_request(request)

pappyproxy/proxy.py

@@ -62,11 +62,20 @@ def log_request(request, id=None, symbol='*', verbosity_level=3):
         for l in r_split:
             log(l, id, symbol, verbosity_level)
             
-def get_endpoint(target_host, target_port, target_ssl, socks_config=None):
+def get_endpoint(target_host, target_port, target_ssl, socks_config=None, use_http_proxy=False, debugid=None):
     # Imports go here to allow mocking for tests
     from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint
     from txsocksx.client import SOCKS5ClientEndpoint
     from txsocksx.tls import TLSWrapClientEndpoint
+    from pappyproxy.pappy import session
+
+    log("Getting endpoint for host '%s' on port %d ssl=%s, socks_config=%s, use_http_proxy=%s" % (target_host, target_port, target_ssl, str(socks_config), use_http_proxy), id=debugid, verbosity_level=3)
+
+    if session.config.http_proxy and use_http_proxy:
+        target_host = session.config.http_proxy['host']
+        target_port = session.config.http_proxy['port']
+        target_ssl = False # We turn on ssl after CONNECT request if needed
+        log("Connecting to http proxy at %s:%d" % (target_host, target_port), id=debugid, verbosity_level=3)
 
     if socks_config is not None:
         sock_host = socks_config['host']
@@ -183,7 +192,7 @@ class UpstreamHTTPProxyClient(ProxyClient):
             sendreq.proxy_creds = self.creds
         lines = sendreq.full_request.splitlines()
         for l in lines:
-            self.log(l, symbol='>r', verbosity_level=3)
+            self.log(l, symbol='>rp', verbosity_level=3)
         self.transport.write(sendreq.full_message)
 
     def connectionMade(self):
@@ -194,10 +203,16 @@ class UpstreamHTTPProxyClient(ProxyClient):
                 self.connect_response = True
                 if self.creds is not None:
                     connreq.proxy_creds = self.creds
+                lines = connreq.full_message.splitlines()
+                for l in lines:
+                    self.log(l, symbol='>p', verbosity_level=3)
                 self.transport.write(connreq.full_message)
             else:
                 self.proxy_connected = True
                 self.stream_response = True
+                lines = self.request.full_message.splitlines()
+                for l in lines:
+                    self.log(l, symbol='>p', verbosity_level=3)
                 self.write_proxied_request(self.request)
 
     def handle_response_end(self, *args, **kwargs):
@@ -211,6 +226,10 @@ class UpstreamHTTPProxyClient(ProxyClient):
             self.transport.loseConnection()
             assert self._response_obj.full_response
             self.data_defer.callback(self.request)
+        elif self._response_obj.response_code != 200:
+            print "Error establishing connection to proxy"
+            self.transport.loseConnection()
+            return
         elif self.connect_response:
             self.log("Response to CONNECT request recieved from http proxy", verbosity_level=3)
             self.proxy_connected = True
@@ -220,10 +239,12 @@ class UpstreamHTTPProxyClient(ProxyClient):
             self.completed = False
             self._sent = False
 
+            self.log("Starting TLS", verbosity_level=3)
             self.transport.startTLS(ClientTLSContext())
+            self.log("TLS started", verbosity_level=3)
             lines = self.request.full_message.splitlines()
             for l in lines:
-                self.log(l, symbol='>r', verbosity_level=3)
+                self.log(l, symbol='>rpr', verbosity_level=3)
             self.transport.write(self.request.full_message)
 
 class ProxyClientFactory(ClientFactory):
@@ -240,6 +261,7 @@ class ProxyClientFactory(ClientFactory):
         self.return_transport = return_transport
         self.intercepting_macros = {}
         self.use_as_proxy = False
+        self.sendback_function = None
 
     def log(self, message, symbol='*', verbosity_level=1):
         log(message, id=self.connection_id, symbol=symbol, verbosity_level=verbosity_level)
@@ -297,6 +319,8 @@ class ProxyClientFactory(ClientFactory):
 
             if session.config.http_proxy:
                 self.use_as_proxy = True
+                if (not self.stream_response) and self.sendback_function:
+                    self.data_defer.addCallback(self.sendback_function)
         else:
             self.log("Request out of scope, passing along unmangled")
         self.request = sendreq
@@ -340,6 +364,29 @@ class ProxyClientFactory(ClientFactory):
         self.data_defer.callback(request)
         defer.returnValue(None)
 
+    @defer.inlineCallbacks
+    def connect(self):
+        from pappyproxy.pappy import session
+
+        yield self.prepare_request()
+        if context.in_scope(self.request):
+            # Get connection using config
+            endpoint = get_endpoint(self.request.host,
+                                    self.request.port,
+                                    self.request.is_ssl,
+                                    socks_config=session.config.socks_proxy,
+                                    use_http_proxy=True)
+        else:
+            # Just forward it normally
+            endpoint = get_endpoint(self.request.host,
+                                    self.request.port,
+                                    self.request.is_ssl)
+
+        # Connect via the endpoint
+        self.log("Accessing using endpoint")
+        yield endpoint.connect(self)
+        self.log("Connected")
+
 class ProxyServerFactory(ServerFactory):
 
     def __init__(self, save_all=False):
@@ -425,6 +472,8 @@ class ProxyServer(LineReceiver):
                 
     @defer.inlineCallbacks
     def full_request_received(self):
+        from pappyproxy.http import Request
+
         global cached_certs
         
         self.log('End of request', verbosity_level=3)
@@ -447,8 +496,18 @@ class ProxyServer(LineReceiver):
 
         # if _request_obj.host is a listener, forward = False
 
+        if self.factory.intercepting_macros:
+            return_transport = None
+        else:
+            return_transport = self.transport
+
         if forward:
-            self._generate_and_submit_client()
+            d = Request.submit_request(self._request_obj,
+                                       save_request=True,
+                                       intercepting_macros=self.factory.intercepting_macros,
+                                       stream_transport=return_transport)
+            if return_transport is None:
+                d.addCallback(self.send_response_back)
         self._reset()
         
     def _reset(self):
@@ -467,73 +526,9 @@ class ProxyServer(LineReceiver):
             self._request_obj.port = self._connect_port
         self.setLineMode()
 
-    def _generate_and_submit_client(self):
+    def send_response_back(self, request):
-        """
+        if request.response is not None:
-        Sets up self._client_factory with self._request_obj then calls back to
+            self.transport.write(request.response.full_response)
-        submit the request
-        """
-
-        self.log("Forwarding to %s on %d" % (self._request_obj.host, self._request_obj.port))
-        if self.factory.intercepting_macros:
-            stream = False
-        else:
-            stream = True
-        self.log('Creating client factory, stream=%s' % stream)
-        self._client_factory = ProxyClientFactory(self._request_obj,
-                                                  save_all=self.factory.save_all,
-                                                  stream_response=stream,
-                                                  return_transport=self.transport)
-        self._client_factory.intercepting_macros = self.factory.intercepting_macros
-        self._client_factory.connection_id = self.connection_id
-        if not stream:
-            self._client_factory.data_defer.addCallback(self.send_response_back)
-        d = self._client_factory.prepare_request()
-        d.addCallback(self._make_remote_connection)
-        return d
-
-    @defer.inlineCallbacks
-    def _make_remote_connection(self, req):
-        """
-        Creates an endpoint to the target server using the given configuration
-        options then connects to the endpoint using self._client_factory
-        """
-        from pappyproxy.pappy import session
-
-        self._request_obj = req
-
-        # If we have a socks proxy, wrap the endpoint in it
-        if context.in_scope(self._request_obj):
-            # Modify the request connection settings to match settings in the factory
-            if self.factory.force_ssl:
-                self._request_obj.is_ssl = True
-            if self.factory.forward_host:
-                self._request_obj.host = self.factory.forward_host
-
-            usehost = self._request_obj.host
-            useport = self._request_obj.port
-            usessl = self._request_obj.is_ssl
-            if session.config.http_proxy:
-                usehost = session.config.http_proxy['host']
-                useport = session.config.http_proxy['port']
-                usessl = False # We turn on ssl after CONNECT request if needed
-                self.log("Connecting to http proxy at %s:%d" % (usehost, useport))
-
-            # Get connection from the request
-            endpoint = get_endpoint(usehost, useport, usessl,
-                                    socks_config=session.config.socks_proxy)
-        else:
-            endpoint = get_endpoint(self._request_obj.host,
-                                    self._request_obj.port,
-                                    self._request_obj.is_ssl)
-
-        # Connect via the endpoint
-        self.log("Accessing using endpoint")
-        yield endpoint.connect(self._client_factory)
-        self.log("Connected")
-        
-    def send_response_back(self, response):
-        if response is not None:
-            self.transport.write(response.response.full_response)
         self.log("Response sent back, losing connection")
         self.transport.loseConnection()
         

pappyproxy/tests/test_comm.py

@@ -10,6 +10,10 @@ from pappyproxy.comm import CommServer
 from pappyproxy.http import Request, Response
 from testutil import mock_deferred, func_deleted, TLSStringTransport, freeze, mock_int_macro, no_tcp
 
+@pytest.fixture(autouse=True)
+def no_int_macros(mocker):
+    mocker.patch('pappyproxy.plugin.active_intercepting_macros').return_value = {}
+
 @pytest.fixture
 def http_request():
     req = Request('GET / HTTP/1.1\r\n\r\n')
@@ -42,6 +46,13 @@ def mock_loader(rsp):
         return rsp
     return classmethod(f)
 
+def mock_submitter(rsp):
+    def f(_, req, *args, **kwargs):
+        req.response = rsp
+        req.reqid = 123
+        return mock_deferred(req)
+    return classmethod(f)
+
 def mock_loader_fail():
     def f(*args, **kwargs):
         raise PappyException("lololo message don't exist dawg")
@@ -80,7 +91,8 @@ def test_get_response_fail(mocker, http_request):
     assert 'message' in v
 
 def test_submit_request(mocker, http_request):
-    mocker.patch.object(pappyproxy.http.Request, 'submit_new', new=mock_loader(http_request))
+    rsp = Response('HTTP/1.1 200 OK\r\n\r\n')
+    mocker.patch.object(pappyproxy.http.Request, 'submit_request', new=mock_submitter(rsp))
     mocker.patch('pappyproxy.http.Request.async_deep_save').return_value = mock_deferred()
 
     comm_data = {"action": "submit"}
@@ -92,14 +104,14 @@ def test_submit_request(mocker, http_request):
     v = perform_comm(json.dumps(comm_data))
 
     expected_data = {}
-    expected_data['request'] = json.loads(http_request.to_json())
+    expected_data[u'request'] = json.loads(http_request.to_json())
-    expected_data['response'] = json.loads(http_request.response.to_json())
+    expected_data[u'response'] = json.loads(http_request.response.to_json())
-    expected_data['success'] = True
+    expected_data[u'success'] = True
-    expected_data['request']['tags'] = ['footag']
+    expected_data[u'request'][u'tags'] = [u'footag']
     assert json.loads(v) == expected_data
 
 def test_submit_request_fail(mocker, http_request):
-    mocker.patch.object(pappyproxy.http.Request, 'submit_new', new=mock_loader_fail())
+    mocker.patch.object(pappyproxy.http.Request, 'submit_request', new=mock_loader_fail())
     mocker.patch('pappyproxy.http.Request.async_deep_save').return_value = mock_deferred()
 
     comm_data = {"action": "submit"}

pappyproxy/util.py

@@ -4,11 +4,20 @@ import re
 import string
 import sys
 import time
+import pyperclip
 
 from .colors import Styles, Colors, verb_color, scode_color, path_formatter, host_color
 from twisted.internet import defer
 from twisted.test.proto_helpers import StringTransport
 
+try:
+    # If you don't do this then pyperclip imports gtk, it blocks the twisted reactor.
+    # Dumb. I know.
+    import gtk
+    gtk.set_interactive(False)
+except ImportError:
+    pass
+
 class PappyException(Exception):
     """
     The exception class for Pappy. If a plugin command raises one of these, the
@@ -320,3 +329,10 @@ def confirm(message, default='n'):
         return True
     else:
         return False
+
+
+def copy_to_clipboard(text):
+    pyperclip.copy(text)
+    
+def clipboard_contents():
+    return pyperclip.paste()

setup.py

@@ -23,11 +23,11 @@ setup(name='pappyproxy',
       download_url='https://github.com/roglew/pappy-proxy/archive/%s.tar.gz'%VERSION,
       install_requires=[
           'beautifulsoup4>=4.4.1',
-          'clipboard>=0.0.4',
           'cmd2>=0.6.8',
           'crochet>=1.4.0',
           'Jinja2>=2.8',
           'pygments>=2.0.2',
+          'pyperclip>=1.5.26',
           'pytest-cov>=2.2.0',
           'pytest-mock>=0.9.0',
           'pytest-twisted>=1.5',