From 1acaeccf5e644bb66f10a34d1703bd2525a5458f Mon Sep 17 00:00:00 2001 From: Rob Glew Date: Wed, 3 Feb 2016 10:28:22 -0600 Subject: [PATCH] Version 0.2.5 --- README.md | 15 +++++- docs/source/conf.py | 4 +- pappyproxy/comm.py | 2 + pappyproxy/console.py | 2 +- pappyproxy/http.py | 18 +++++-- pappyproxy/plugin.py | 2 +- pappyproxy/plugins/decode.py | 2 + pappyproxy/plugins/macrocmds.py | 2 +- pappyproxy/plugins/manglecmds.py | 6 ++- pappyproxy/plugins/view.py | 55 +++++++++++++++++++++ pappyproxy/plugins/vim_repeater/repeater.py | 1 + pappyproxy/templates/intmacro.py | 1 + pappyproxy/tests/test_http.py | 18 +++++++ setup.py | 2 +- 14 files changed, 119 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 03dfaee..29126ce 100644 --- a/README.md +++ b/README.md @@ -223,10 +223,18 @@ The following commands can be used to view requests and responses | `viq ` | view_request_info, viq | View additional information about requests. Includes the target port, if SSL was used, applied tags, and other information. | | `vfq ` | view_full_request, vfq | [V]iew [F]ull Re[Q]uest, prints the full request including headers and data. | | `vbq ` | view_request_bytes, vbq | [V]iew [B]ytes of Re[Q]uest, prints the full request including headers and data without coloring or additional newlines. Use this if you want to write a request to a file. | +| `ppq [format]` | pretty_print_request, ppq | Pretty print a request. If a format is given, it will try and print the body of the request with that format. Otherwise it will make a guess based off of the Content-Type header. | | `vhq ` | view_request_headers, vhq | [V]iew [H]eaders of a Re[Q]uest. Prints just the headers of a request. | | `vfs ` | view_full_response, vfs |[V]iew [F]ull Re[S]ponse, prints the full response associated with a request including headers and data. | -| `vbs ` | view_response_bytes, vbs | [V]iew [B]ytes of Re[S]ponse, prints the full response including headers and data without coloring or additional newlines. Use this if you want to write a response to a file. | | `vhs ` | view_response_headers, vhs | [V]iew [H]eaders of a Re[S]ponse. Prints just the headers of a response associated with a request. | +| `vbs ` | view_response_bytes, vbs | [V]iew [B]ytes of Re[S]ponse, prints the full response including headers and data without coloring or additional newlines. Use this if you want to write a response to a file. | +| `pps [format]` | pretty_print_response, pps | Pretty print a response. If a format is given, it will try and print the body of the response with that format. Otherwise it will make a guess based off of the Content-Type header. | + +Available formats for `ppq` and `pps` commands: + +| Format | Description | +|:-------|:------------| +| `json` | Print as JSON | The table shown by `ls` will have the following columns: @@ -944,6 +952,11 @@ Changelog --------- The boring part of the readme +* 0.2.5 + * Requests sent with repeater now are given `repeater` tag + * Add ppq and pps commands + * Look at the pretty prompt + * Bugfixes * 0.2.4 * Add command history saving between sessions * Add html encoder/decoder diff --git a/docs/source/conf.py b/docs/source/conf.py index 6ff9673..f4a8127 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -59,9 +59,9 @@ author = u'Rob Glew' # built documents. # # The short X.Y version. -version = u'0.2.4' +version = u'0.2.5' # The full version, including alpha/beta/rc tags. -release = u'0.2.4' +release = u'0.2.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pappyproxy/comm.py b/pappyproxy/comm.py index 5d220bd..90c0bf0 100644 --- a/pappyproxy/comm.py +++ b/pappyproxy/comm.py @@ -99,6 +99,8 @@ class CommServer(LineReceiver): def action_submit_request(self, data): message = base64.b64decode(data['full_message']) req = yield Request.submit_new(data['host'], data['port'], data['is_ssl'], message) + if 'tags' in data: + req.tags = data['tags'] yield req.async_deep_save() retdata = {} diff --git a/pappyproxy/console.py b/pappyproxy/console.py index 10efe2f..d922fd4 100644 --- a/pappyproxy/console.py +++ b/pappyproxy/console.py @@ -262,7 +262,7 @@ class ProxyCmd(cmd2.Cmd): """ def __init__(self, *args, **kwargs): - self.prompt = 'pappy> ' + self.prompt = 'pappy' + Colors.YELLOW + '> ' + Colors.ENDC self.debug = True self._cmds = {} diff --git a/pappyproxy/http.py b/pappyproxy/http.py index 09ea1a5..424eb6d 100644 --- a/pappyproxy/http.py +++ b/pappyproxy/http.py @@ -803,7 +803,17 @@ class HTTPMessage(object): stripped = True elif key.lower() == 'content-length': # We use our own content length - self._data_obj = LengthData(int(val)) + if self._data_obj and self._data_obj.complete: + # We're regenerating or something so we want to base this header + # off our existing body + val = self._data_obj.body + self._data_obj = LengthData(len(val)) + if len(val) > 0: + self._data_obj.add_data(val) + self._encoding_type = ENCODE_NONE + self.complete = True + else: + self._data_obj = LengthData(int(val)) return (not stripped) @@ -1972,8 +1982,10 @@ class Request(HTTPMessage): """ from .proxy import ProxyClientFactory, get_next_connection_id, ClientTLSContext - new_obj = Request(full_request) - factory = ProxyClientFactory(new_obj, save_all=False) + new_req = Request(full_request) + new_req.is_ssl = is_ssl + new_req.port = port + factory = ProxyClientFactory(new_req, save_all=False) factory.connection_id = get_next_connection_id() if is_ssl: reactor.connectSSL(host, port, factory, ClientTLSContext()) diff --git a/pappyproxy/plugin.py b/pappyproxy/plugin.py index 1615fa5..4787859 100644 --- a/pappyproxy/plugin.py +++ b/pappyproxy/plugin.py @@ -109,7 +109,7 @@ def active_intercepting_macros(): Returns a list of the active intercepting macro objects. Modifying this list will not affect which macros are active. """ - return pappyproxy.pappy.server_factory.intercepting_macros[:] + return [v for k, v in pappyproxy.pappy.server_factory.intercepting_macros.iteritems() ] def in_memory_reqs(): """ diff --git a/pappyproxy/plugins/decode.py b/pappyproxy/plugins/decode.py index 200a324..c348d53 100644 --- a/pappyproxy/plugins/decode.py +++ b/pappyproxy/plugins/decode.py @@ -54,6 +54,8 @@ def _code_helper(line, func, copy=True): args = shlex.split(line) if not args: s = clipboard.paste() + print 'Will decode:' + print printable_data(s) s = func(s) if copy: try: diff --git a/pappyproxy/plugins/macrocmds.py b/pappyproxy/plugins/macrocmds.py index 1a9bbfc..c2843d5 100644 --- a/pappyproxy/plugins/macrocmds.py +++ b/pappyproxy/plugins/macrocmds.py @@ -124,7 +124,7 @@ def list_int_macros(line): running = [] not_running = [] for macro in loaded_int_macros: - if macro.name in active_intercepting_macros(): + if macro.name in [m.name for m in active_intercepting_macros()]: running.append(macro) else: not_running.append(macro) diff --git a/pappyproxy/plugins/manglecmds.py b/pappyproxy/plugins/manglecmds.py index 8aa0398..40ec359 100644 --- a/pappyproxy/plugins/manglecmds.py +++ b/pappyproxy/plugins/manglecmds.py @@ -212,7 +212,11 @@ def intercept(line): editor = 'vi' if 'EDITOR' in os.environ: editor = os.environ['EDITOR'] - subprocess.call([editor, to_edit]) + additional_args = [] + if editor == 'vim': + # prevent adding additional newline + additional_args.append('-b') + subprocess.call([editor, to_edit] + additional_args) stdscr.clear() deferred.callback(None) finally: diff --git a/pappyproxy/plugins/view.py b/pappyproxy/plugins/view.py index eeb4e47..9f32c8a 100644 --- a/pappyproxy/plugins/view.py +++ b/pappyproxy/plugins/view.py @@ -1,6 +1,9 @@ import crochet import datetime +import json import pappyproxy +import pygments +import pprint import shlex from pappyproxy.console import load_reqlist, print_table, print_request_rows, get_req_data_row @@ -9,6 +12,8 @@ from pappyproxy.http import Request from twisted.internet import defer from pappyproxy.plugin import main_context_ids from pappyproxy.colors import Colors, Styles, verb_color, scode_color, path_formatter, host_color +from pygments.formatters import TerminalFormatter +from pygments.lexers.data import JsonLexer ################### ## Helper functions @@ -91,6 +96,17 @@ def print_tree(tree): # Prints a tree. Takes in a sorted list of path tuples _print_tree_helper(tree, 0, []) +def pretty_print_body(fmt, body): + if fmt.lower() == 'json': + try: + d = json.loads(body.strip()) + except: + raise PappyException('Body could not be parsed as JSON') + s = json.dumps(d, indent=4, sort_keys=True) + print pygments.highlight(s, JsonLexer(), TerminalFormatter()) + else: + raise PappyException('%s is not a valid format' % fmt) + def _get_tree_prefix(depth, print_bars, last): if depth == 0: return u'' @@ -247,6 +263,22 @@ def view_request_bytes(line): if len(reqs) > 1: print '-'*30 print '' + +@crochet.wait_for(timeout=None) +@defer.inlineCallbacks +def pretty_print_request(line): + """ + Print the body of the request pretty printed. + Usage: pretty_print_request + """ + args = shlex.split(line) + if len(args) < 2: + raise PappyException("Usage: pretty_print_request ") + reqids = args[1] + + reqs = yield load_reqlist(reqids) + for req in reqs: + pretty_print_body(args[0], req.body) @crochet.wait_for(timeout=None) @defer.inlineCallbacks @@ -297,6 +329,25 @@ def view_response_bytes(line): else: print "Request %s does not have a response" % req.reqid +@crochet.wait_for(timeout=None) +@defer.inlineCallbacks +def pretty_print_response(line): + """ + Print the body of the request pretty printed. + Usage: pretty_print_request + """ + args = shlex.split(line) + if len(args) < 2: + raise PappyException("Usage: pretty_print_request ") + reqids = args[1] + + reqs = yield load_reqlist(reqids) + for req in reqs: + if req.response: + pretty_print_body(args[0], req.response.body) + else: + print 'No response associated with request %s' % req.reqid + @crochet.wait_for(timeout=None) @defer.inlineCallbacks def dump_response(line): @@ -345,9 +396,11 @@ def load_cmds(cmd): 'view_request_headers': (view_request_headers, None), 'view_full_request': (view_full_request, None), 'view_request_bytes': (view_request_bytes, None), + 'pretty_print_request': (pretty_print_request, None), 'view_response_headers': (view_response_headers, None), 'view_full_response': (view_full_response, None), 'view_response_bytes': (view_response_bytes, None), + 'pretty_print_response': (pretty_print_response, None), 'site_map': (site_map, None), 'dump_response': (dump_response, None), }) @@ -357,9 +410,11 @@ def load_cmds(cmd): ('view_request_headers', 'vhq'), ('view_full_request', 'vfq'), ('view_request_bytes', 'vbq'), + ('pretty_print_request', 'ppq'), ('view_response_headers', 'vhs'), ('view_full_response', 'vfs'), ('view_response_bytes', 'vbs'), + ('pretty_print_response', 'pps'), ('site_map', 'sm'), #('dump_response', 'dr'), ]) diff --git a/pappyproxy/plugins/vim_repeater/repeater.py b/pappyproxy/plugins/vim_repeater/repeater.py index 97bc455..0609e21 100644 --- a/pappyproxy/plugins/vim_repeater/repeater.py +++ b/pappyproxy/plugins/vim_repeater/repeater.py @@ -119,6 +119,7 @@ def submit_current_buffer(): full_request = '\n'.join(curbuf) commdata = {'action': 'submit', 'full_message': base64.b64encode(full_request), + 'tags': ['repeater'], 'port': int(vim.eval("s:repport")), 'host': vim.eval("s:rephost")} if vim.eval("s:repisssl") == '1': diff --git a/pappyproxy/templates/intmacro.py b/pappyproxy/templates/intmacro.py index 861ad22..5f941ff 100644 --- a/pappyproxy/templates/intmacro.py +++ b/pappyproxy/templates/intmacro.py @@ -5,6 +5,7 @@ SHORT_NAME = '{{short_name}}' runargs = [] def init(args): + global runargs runargs = args def mangle_request(request): diff --git a/pappyproxy/tests/test_http.py b/pappyproxy/tests/test_http.py index 1d869b0..80bf3d6 100644 --- a/pappyproxy/tests/test_http.py +++ b/pappyproxy/tests/test_http.py @@ -846,6 +846,24 @@ def test_request_url_blankpath(): assert r.full_path == '/?foo=bar' assert r.url == 'https://www.google.com?foo=bar' +def test_request_modify_header2(): + r = http.Request(('POST /some/path HTTP/1.1\r\n' + 'Host: test.host.thing\r\n' + 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:43.0) Gecko/20100101 Firefox/43.0\r\n' + 'Content-Length: 282\r\n' + 'Connection: keep-alive\r\n' + '\r\n' + 'a|b|c|d')) + r2 = r.copy() + r2.headers['User-Agent'] = 'Moziller/6.9' + assert r2.full_message == ('POST /some/path HTTP/1.1\r\n' + 'Host: test.host.thing\r\n' + 'User-Agent: Moziller/6.9\r\n' + 'Content-Length: 7\r\n' + 'Connection: keep-alive\r\n' + '\r\n' + 'a|b|c|d') + #################### ## Response tests diff --git a/setup.py b/setup.py index 8d7c947..00be536 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import pkgutil from setuptools import setup, find_packages -VERSION = '0.2.4' +VERSION = '0.2.5' setup(name='pappyproxy', version=VERSION,