Release 0.0.1

See README diff for changes

pappyproxy/console.py

@@ -0,0 +1,792 @@
+import cmd2
+import crochet
+import curses
+import datetime
+import os
+import pappyproxy
+import pygments
+import shlex
+import string
+import subprocess
+import sys
+import termios
+import time
+
+from twisted.internet import defer, reactor
+from pappyproxy.util import PappyException
+from pygments.lexers import get_lexer_for_mimetype
+from pygments.formatters import TerminalFormatter
+
+"""
+console.py
+
+Functions and classes involved with interacting with console input and output
+"""
+
+# http://www.termsys.demon.co.uk/vtansi.htm#cursor
+SAVE_CURSOR = '\x1b[7'
+UNSAVE_CURSOR = '\x1b[8'
+LINE_UP = '\x1b[1A'
+LINE_ERASE = '\x1b[2K'
+PRINT_LINE = '\x1b[1i'
+
+edit_queue = []
+
+def print_pappy_errors(func):
+    def catch(*args, **kwargs):
+        try:
+            func(*args, **kwargs)
+        except PappyException as e:
+            print str(e)
+    return catch
+
+class ProxyCmd(cmd2.Cmd):
+
+    def __init__(self, *args, **kwargs):
+        self.alerts = []
+        self.prompt = 'itsPappyTime> '
+        self.debug = True
+        cmd2.Cmd.__init__(self, *args, **kwargs)
+        
+    def add_alert(self, alert):
+        self.alerts.append(alert)
+
+    def postcmd(self, stop, line):
+        for l in self.alerts:
+            print '[!] ', l
+        self.alerts = []
+        return stop
+
+    def help_view_request_headers(self):
+        print ("View information about request\n"
+               "Usage: view_request_info <reqid> [u]"
+               "If 'u' is given as an additional argument, the unmangled version "
+               "of the request will be displayed.")
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_view_request_info(self, line):
+        args = shlex.split(line)
+        try:
+            reqid = int(args[0])
+            showid = reqid
+        except:
+            raise PappyException("Enter a valid number for the request id")
+
+        req = yield pappyproxy.http.Request.load_request(reqid)
+        showreq = req
+
+        show_unmangled = False
+        if len(args) > 1 and args[1][0].lower() == 'u':
+            if not req.unmangled:
+                raise PappyException("Request was not mangled")
+            show_unmangled = True
+            showreq = req.unmangled
+
+        print ''
+        print_request_extended(showreq)
+        print ''
+
+    def help_view_request_headers(self):
+        print ("View the headers of the request\n"
+               "Usage: view_request_headers <reqid> [u]"
+               "If 'u' is given as an additional argument, the unmangled version "
+               "of the request will be displayed.")
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_view_request_headers(self, line):
+        args = shlex.split(line)
+        try:
+            reqid = int(args[0])
+            showid = reqid
+        except:
+            raise PappyException("Enter a valid number for the request id")
+
+        req = yield pappyproxy.http.Request.load_request(reqid)
+        showreq = req
+
+        show_unmangled = False
+        if len(args) > 1 and args[1][0].lower() == 'u':
+            if not req.unmangled:
+                raise PappyException("Request was not mangled")
+            show_unmangled = True
+            showreq = req.unmangled
+
+        if show_unmangled:
+            print 'UNMANGLED --------------------'
+        print ''
+        view_full_request(showreq, True)
+
+    def help_view_full_request(self):
+        print ("View the full data of the request\n"
+               "Usage: view_full_request <reqid> [u]\n"
+               "If 'u' is given as an additional argument, the unmangled version "
+               "of the request will be displayed.")
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_view_full_request(self, line):
+        args = shlex.split(line)
+        try:
+            reqid = int(args[0])
+            showid = reqid
+        except:
+            raise PappyException("Enter a valid number for the request id")
+
+        req = yield pappyproxy.http.Request.load_request(reqid)
+        showreq = req
+
+        show_unmangled = False
+        if len(args) > 1 and args[1][0].lower() == 'u':
+            if not req.unmangled:
+                raise PappyException("Request was not mangled")
+            show_unmangled = True
+            showreq = req.unmangled
+
+        if show_unmangled:
+            print 'UNMANGLED --------------------'
+        print ''
+        view_full_request(showreq)
+
+    def help_view_response_headers(self):
+        print ("View the headers of the response\n"
+               "Usage: view_response_headers <reqid>")
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_view_response_headers(self, line):
+        args = shlex.split(line)
+        try:
+            reqid = int(args[0])
+            showid = reqid
+        except:
+            raise PappyException("Enter a valid number for the request id")
+
+        req = yield pappyproxy.http.Request.load_request(reqid)
+        showrsp = req.response
+
+        show_unmangled = False
+        if len(args) > 1 and args[1][0].lower() == 'u':
+            if not req.response.unmangled:
+                raise PappyException("Response was not mangled")
+            show_unmangled = True
+            showrsp = req.response.unmangled
+
+        if show_unmangled:
+            print ''
+            print 'UNMANGLED --------------------'
+        print ''
+        view_full_response(showrsp, True)
+
+    def help_view_full_response(self):
+        print ("View the full data of the response associated with a request\n"
+               "Usage: view_full_response <reqid>")
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_view_full_response(self, line):
+        args = shlex.split(line)
+        try:
+            reqid = int(args[0])
+            showid = reqid
+        except:
+            raise PappyException("Enter a valid number for the request id")
+
+        req = yield pappyproxy.http.Request.load_request(reqid)
+        showrsp = req.response
+
+        show_unmangled = False
+        if len(args) > 1 and args[1][0].lower() == 'u':
+            if not req.response.unmangled:
+                raise PappyException("Response was not mangled")
+            show_unmangled = True
+            showrsp = req.response.unmangled
+
+        if show_unmangled:
+            print ''
+            print 'UNMANGLED --------------------'
+        print ''
+        view_full_response(showrsp)
+
+    def help_list(self):
+        print ("List request/response pairs in the current context\n"
+               "Usage: list")
+
+    @print_pappy_errors
+    def do_list(self, line):
+        args = shlex.split(line)
+        if len(args) > 0:
+            if args[0][0].lower() == 'a':
+                print_count = -1
+            else:
+                try:
+                    print_count = int(args[0])
+                except:
+                    print "Please enter a valid argument for list"
+                    return
+        else:
+            print_count = 25
+        
+        pappyproxy.context.sort()
+        if print_count > 0:
+            to_print = pappyproxy.context.active_requests[:]
+            to_print = sorted(to_print, key=lambda x: x.reqid, reverse=True)
+            to_print = to_print[:print_count]
+            print_requests(to_print)
+        else:
+            print_requests(pappyproxy.context.active_requests)
+
+    def help_filter(self):
+        print ("Apply a filter to the current context\n"
+               "Usage: filter <filter string>\n"
+               "See README.md for information on filter strings")
+
+    @print_pappy_errors
+    def do_filter(self, line):
+        if not line:
+            raise PappyException("Filter string required")
+        
+        filter_to_add = pappyproxy.context.Filter(line)
+        pappyproxy.context.add_filter(filter_to_add)
+
+    def help_filter_clear(self):
+        print ("Reset the context so that it contains no filters (ignores scope)\n"
+               "Usage: filter_clear")
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_filter_clear(self, line):
+        pappyproxy.context.active_filters = []
+        yield pappyproxy.context.reload_from_storage()
+
+    def help_filter_list(self):
+        print ("Print the filters that make up the current context\n"
+               "Usage: filter_list")
+
+    @print_pappy_errors
+    def do_filter_list(self, line):
+        for f in pappyproxy.context.active_filters:
+            print f.filter_string
+
+
+    def help_scope_save(self):
+        print ("Set the scope to be the current context. Saved between launches\n"
+               "Usage: scope_save")
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_scope_save(self, line):
+        pappyproxy.context.save_scope()
+        yield pappyproxy.context.store_scope(pappyproxy.http.dbpool)
+
+    def help_scope_reset(self):
+        print ("Set the context to be the scope (view in-scope items)\n"
+               "Usage: scope_reset")
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_scope_reset(self, line):
+        yield pappyproxy.context.reset_to_scope()
+
+    def help_scope_delete(self):
+        print ("Delete the scope so that it contains all request/response pairs\n"
+               "Usage: scope_delete")        
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_scope_delete(self, line):
+        pappyproxy.context.set_scope([])
+        yield pappyproxy.context.store_scope(pappyproxy.http.dbpool)
+
+    def help_scope_list(self):
+        print ("Print the filters that make up the scope\n"
+               "Usage: scope_list")
+
+    @print_pappy_errors
+    def do_scope_list(self, line):
+        pappyproxy.context.print_scope()
+
+    def help_repeater(self):
+        print ("Open a request in the repeater\n"
+               "Usage: repeater <reqid>")
+
+    @print_pappy_errors
+    def do_repeater(self, line):
+        args = shlex.split(line)
+        try:
+            reqid = int(args[0])
+        except:
+            raise PappyException("Enter a valid number for the request id")
+
+        repid = reqid
+        if len(args) > 1 and args[1][0].lower() == 'u':
+            umid = get_unmangled(reqid)
+            if umid is not None:
+                repid = umid
+        pappyproxy.repeater.start_editor(repid)
+
+    def help_submit(self):
+        print "Submit a request again (NOT IMPLEMENTED)"
+
+    @print_pappy_errors
+    @crochet.wait_for(timeout=30.0)
+    @defer.inlineCallbacks
+    def do_submit(self, line):
+        pass
+        # reqid = int(line)
+        # req = yield http.Request.load_request(reqid)
+        # rsp = yield req.submit()
+        # print printable_data(rsp.full_response)
+
+    def help_intercept(self):
+        print ("Intercept requests and/or responses and edit them with before passing them along\n"
+               "Usage: intercept <reqid>")
+
+    @print_pappy_errors
+    def do_intercept(self, line):
+        global edit_queue
+        args = shlex.split(line)
+        intercept_requests = False
+        intercept_responses = False
+
+        req_names = ('req', 'request', 'requests')
+        rsp_names = ('rsp', 'response', 'responses')
+
+        if any(a in req_names for a in args):
+            intercept_requests = True
+        if any(a in rsp_names for a in args):
+            intercept_responses = True
+
+        if intercept_requests and intercept_responses:
+            intercept_str = 'Requests and responses'
+        elif intercept_requests:
+            intercept_str = 'Requests'
+        elif intercept_responses:
+            intercept_str = 'Responses'
+        else:
+            intercept_str = 'NOTHING'
+
+        pappyproxy.mangle.set_intercept_requests(intercept_requests)
+        pappyproxy.mangle.set_intercept_responses(intercept_responses)
+
+        ## Interceptor loop
+        stdscr = curses.initscr()
+        curses.noecho()
+        curses.cbreak()
+
+        try:
+            editnext = False
+            stdscr.nodelay(True)
+            while True:
+                stdscr.addstr(0, 0, "Currently intercepting: %s" % intercept_str)
+                stdscr.clrtoeol()
+                stdscr.addstr(1, 0, "%d item(s) in queue." % len(edit_queue))
+                stdscr.clrtoeol()
+                if editnext:
+                    stdscr.addstr(2, 0, "Waiting for next item... Press 'q' to quit or 'b' to quit waiting")
+                else:
+                    stdscr.addstr(2, 0, "Press 'n' to edit the next item or 'q' to quit interceptor.")
+                stdscr.clrtoeol()
+
+                c = stdscr.getch()
+                if c == ord('q'):
+                    break
+                elif c == ord('n'):
+                    editnext = True
+                elif c == ord('b'):
+                    editnext = False
+
+                if editnext and edit_queue:
+                    editnext = False
+                    (to_edit, deferred) = edit_queue.pop(0)
+                    editor = 'vi'
+                    if 'EDITOR' in os.environ:
+                        editor = os.environ['EDITOR']
+                    subprocess.call([editor, to_edit])
+                    stdscr.clear()
+                    deferred.callback(None)
+        finally:
+            curses.nocbreak()
+            stdscr.keypad(0)
+            curses.echo()
+            curses.endwin()
+            pappyproxy.mangle.set_intercept_requests(False)
+            pappyproxy.mangle.set_intercept_responses(False)
+            # Send remaining requests along
+            while len(edit_queue) > 0:
+                (fname, deferred) = edit_queue.pop(0)
+                deferred.callback(None)
+
+    def help_gencerts(self):
+        print ("Generate CA cert and private CA file\n"
+               "Usage: gencerts [/path/to/put/certs/in]")
+
+    @print_pappy_errors
+    def do_gencerts(self, line):
+        dest_dir = line or pappyproxy.config.CERT_DIR
+        print "This will overwrite any existing certs in %s. Are you sure?" % dest_dir
+        print "(y/N)",
+        answer = raw_input()
+        if not answer or answer[0].lower() != 'y':
+            return False
+        print "Generating certs to %s" % dest_dir
+        pappyproxy.proxy.generate_ca_certs(dest_dir)
+
+    def help_log(self):
+        print ("View the log\n"
+               "Usage: log [verbosity (default is 1)]\n"
+               "verbosity=1: Show connections as they're made/lost, some additional info\n"
+               "verbosity=3: Show full requests/responses as they are processed by the proxy")
+
+    @print_pappy_errors
+    def do_log(self, line):
+        try:
+            verbosity = int(line.strip())
+        except:
+            verbosity = 1
+        pappyproxy.config.DEBUG_VERBOSITY = verbosity
+        raw_input()
+        pappyproxy.config.DEBUG_VERBOSITY = 0
+
+    @print_pappy_errors
+    def do_testerror(self, line):
+        raise PappyException("Test error")
+
+    @print_pappy_errors
+    def do_EOF(self):
+        print "EOF"
+        return True
+
+    ### ABBREVIATIONS
+    def help_ls(self):
+        self.help_list()
+
+    @print_pappy_errors
+    def do_ls(self, line):
+        self.onecmd('list %s' % line)
+
+    def help_sr(self):
+        self.help_scope_reset()
+
+    @print_pappy_errors
+    def do_sr(self, line):
+        self.onecmd('scope_reset %s' % line)
+
+    def help_sls(self):
+        self.help_scope_list()
+
+    @print_pappy_errors
+    def do_sls(self, line):
+        self.onecmd('scope_list %s' % line)
+
+    def help_viq(self):
+        self.help_view_request_info()
+
+    @print_pappy_errors
+    def do_viq(self, line):
+        self.onecmd('view_request_info %s' % line)
+
+    def help_vhq(self):
+        self.help_view_request_headers()
+
+    @print_pappy_errors
+    def do_vhq(self, line):
+        self.onecmd('view_request_headers %s' % line)
+
+    def help_vfq(self):
+        self.help_view_full_request()
+
+    @print_pappy_errors
+    def do_vfq(self, line):
+        self.onecmd('view_full_request %s' % line)
+
+    def help_vhs(self):
+        self.help_view_response_headers()
+
+    @print_pappy_errors
+    def do_vhs(self, line):
+        self.onecmd('view_response_headers %s' % line)
+
+    def help_vfs(self):
+        self.help_view_full_response()
+
+    @print_pappy_errors
+    def do_vfs(self, line):
+        self.onecmd('view_full_response %s' % line)
+
+    def help_fl(self):
+        self.help_filter()
+
+    @print_pappy_errors
+    def do_fl(self, line):
+        self.onecmd('filter %s' % line)
+
+    def help_f(self):
+        self.help_filter()
+
+    @print_pappy_errors
+    def do_f(self, line):
+        self.onecmd('filter %s' % line)
+
+    def help_fls(self):
+        self.help_filter_list()
+
+    @print_pappy_errors
+    def do_fls(self, line):
+        self.onecmd('filter_list %s' % line)
+
+    def help_fc(self):
+        self.help_filter_clear()
+
+    @print_pappy_errors
+    def do_fc(self, line):
+        self.onecmd('filter_clear %s' % line)
+
+    def help_rp(self):
+        self.help_repeater()
+
+    @print_pappy_errors
+    def do_rp(self, line):
+        self.onecmd('repeater %s' % line)
+
+    def help_ic(self):
+        self.help_intercept()
+
+    @print_pappy_errors
+    def do_ic(self, line):
+        self.onecmd('intercept %s' % line)
+
+
+    
+def cmd_failure(cmd):
+    print "FAILURE"
+
+def edit_file(fname, front=False):
+    global edit_queue
+    # Adds the filename to the edit queue. Returns a deferred that is fired once
+    # the file is edited and the editor is closed
+    d = defer.Deferred()
+    if front:
+        edit_queue = [(fname, d)] + edit_queue
+    else:
+        edit_queue.append((fname, d))
+    return d
+    
+def print_table(coldata, rows):
+    # Coldata: List of dicts with info on how to print the columns.
+    #  name: heading to give column
+    #  width: (optional) maximum width before truncating. 0 for unlimited
+    # Rows: List of tuples with the data to print
+
+    # Get the width of each column
+    widths = []
+    headers = []
+    for data in coldata:
+        if 'name' in data:
+            headers.append(data['name'])
+        else:
+            headers.append('')
+    empty_headers = True
+    for h in headers:
+        if h != '':
+            empty_headers = False
+    if not empty_headers:
+        rows = [headers] + rows
+
+    for i in range(len(coldata)):
+        col = coldata[i]
+        if 'width' in col and col['width'] > 0:
+            maxwidth = col['width']
+        else:
+            maxwidth = 0
+        colwidth = 0
+        for row in rows:
+            printstr = str(row[i])
+            if len(printstr) > colwidth:
+                colwidth = len(printstr)
+        if maxwidth > 0 and colwidth > maxwidth:
+            widths.append(maxwidth)
+        else:
+            widths.append(colwidth)
+
+    # Print rows
+    padding = 2
+    for row in rows:
+        for (col, width) in zip(row, widths):
+            printstr = str(col)
+            if len(printstr) > width:
+                for i in range(len(printstr)-4, len(printstr)-1):
+                    printstr=printstr[:width]
+                    printstr=printstr[:-3]+'...'
+            sys.stdout.write(printstr)
+            sys.stdout.write(' '*(width-len(printstr)))
+            sys.stdout.write(' '*padding)
+        sys.stdout.write('\n')
+        sys.stdout.flush()
+
+
+def printable_data(data):
+    chars = []
+    for c in data:
+        if c in string.printable:
+            chars += c
+        else:
+            chars += '.'
+    return ''.join(chars)
+
+@crochet.wait_for(timeout=30.0)
+@defer.inlineCallbacks
+def get_unmangled(reqid):
+    req = yield pappyproxy.http.Request.load_request(reqid)
+    if req.unmangled:
+        defer.returnValue(req.unmangled.reqid)
+    else:
+        defer.returnValue(None)
+
+    
+def view_full_request(request, headers_only=False):
+    if headers_only:
+        print printable_data(request.raw_headers)
+    else:
+        print printable_data(request.full_request)
+
+def view_full_response(response, headers_only=False):
+    def check_type(response, against):
+        if 'Content-Type' in response.headers and against in response.headers['Content-Type']:
+            return True
+        return False
+
+    if headers_only:
+        print printable_data(response.raw_headers)
+    else:
+        print response.raw_headers,
+        to_print = printable_data(response.raw_data)
+        if 'content-type' in response.headers:
+            try:
+                lexer = get_lexer_for_mimetype(response.headers['content-type'].split(';')[0])
+                to_print = pygments.highlight(to_print, lexer, TerminalFormatter())
+            except ClassNotFound:
+                pass
+
+        print to_print
+
+def print_requests(requests):
+    # Print a table with info on all the requests in the list
+    cols = [
+        {'name':'ID'},
+        {'name':'Verb'},
+        {'name': 'Host'},
+        {'name':'Path', 'width':40},
+        {'name':'S-Code'},
+        {'name':'Req Len'},
+        {'name':'Rsp Len'},
+        {'name':'Time'},
+        {'name':'Mngl'},
+    ]
+    rows = []
+    for request in requests:
+        rid = request.reqid
+        method = request.verb
+        host = request.headers['host']
+        path = request.full_path
+        reqlen = len(request.raw_data)
+        rsplen = 'N/A'
+        mangle_str = '--'
+
+        if request.unmangled:
+            mangle_str = 'q'
+            
+        if request.response:
+            response_code = str(request.response.response_code) + \
+                ' ' + request.response.response_text
+            rsplen = len(request.response.raw_data)
+            if request.response.unmangled:
+                if mangle_str == '--':
+                    mangle_str = 's'
+                else:
+                    mangle_str += '/s'
+        else:
+            response_code = ''
+
+        time_str = '--'
+        if request.time_start and request.time_end:
+            time_delt = request.time_end - request.time_start
+            time_str = "%.2f" % time_delt.total_seconds()
+
+        port = request.port
+        if request.is_ssl:
+            is_ssl = 'YES'
+        else:
+            is_ssl = 'NO'
+            
+        rows.append([rid, method, host, path, response_code,
+                     reqlen, rsplen, time_str, mangle_str])
+    print_table(cols, rows)
+    
+def print_request_extended(request):
+    # Prints extended info for the request
+    title = "Request Info (reqid=%d)" % request.reqid
+    print title
+    print '-'*len(title)
+    reqlen = len(request.raw_data)
+    reqlen = '%d bytes' % reqlen
+    rsplen = 'No response'
+
+    mangle_str = 'Nothing mangled'
+    if request.unmangled:
+        mangle_str = 'Request'
+
+    if request.response:
+        response_code = str(request.response.response_code) + \
+            ' ' + request.response.response_text
+        rsplen = len(request.response.raw_data)
+        rsplen = '%d bytes' % rsplen
+
+        if request.response.unmangled:
+            if mangle_str == 'Nothing mangled':
+                mangle_str = 'Response'
+            else:
+                mangle_str += ' and Response'
+    else:
+        response_code = ''
+
+    time_str = '--'
+    if request.time_start and request.time_end:
+        time_delt = request.time_end - request.time_start
+        time_str = "%.2f sec" % time_delt.total_seconds()
+
+    port = request.port
+    if request.is_ssl:
+        is_ssl = 'YES'
+    else:
+        is_ssl = 'NO'
+
+    if request.time_start:
+        time_made_str = request.time_start.strftime('%a, %b %d, %Y, %I:%M:%S %p')
+    else:
+        time_made_str = '--'
+    
+    print 'Made on %s' % time_made_str
+    print 'ID: %d' % request.reqid
+    print 'Verb: %s' % request.verb
+    print 'Host: %s' % request.host
+    print 'Path: %s' % request.full_path
+    print 'Status Code: %s' % response_code
+    print 'Request Length: %s' % reqlen
+    print 'Response Length: %s' % rsplen
+    if request.response.unmangled:
+        print 'Unmangled Response Length: %s bytes' % len(request.response.unmangled.full_response)
+    print 'Time: %s' % time_str
+    print 'Port: %s' % request.port
+    print 'SSL: %s' % is_ssl
+    print 'Mangled: %s' % mangle_str