From 0b6a63ddbb80a0467823402445d19fe3e19d267b Mon Sep 17 00:00:00 2001 From: Rob Glew Date: Thu, 28 Jan 2016 14:53:34 -0600 Subject: [PATCH] Version 0.2.4 --- README.md | 121 +++++++++++++++++++++++++++- docs/source/conf.py | 4 +- pappyproxy/config.py | 13 +++ pappyproxy/console.py | 34 +++++++- pappyproxy/default_user_config.json | 1 + pappyproxy/pappy.py | 1 + pappyproxy/plugins/debug.py | 2 +- pappyproxy/plugins/decode.py | 49 ++++++++++- pappyproxy/plugins/misc.py | 16 ++-- pappyproxy/plugins/view.py | 59 ++++++++++---- pappyproxy/util.py | 5 ++ setup.py | 2 +- 12 files changed, 281 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index d9b2469..03dfaee 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,115 @@ The following tokens will also be replaced with values: See the default `config.json` for examples. +General Console Techniques +-------------------------- +There are a few tricks you can use in general when using Pappy's console. Most of these are provided by the [cmd](https://docs.python.org/2/library/cmd.html) and [cmd2](https://pythonhosted.org/cmd2/index.html). + +### Run a shell command + +You can run a shell command with `!`: + +``` +pappy> ls +ID Verb Host Path S-Code Req Len Rsp Len Time Mngl +5 GET vitaly.sexy /netscape.gif 304 Not Modified 0 0 0.08 -- +4 GET vitaly.sexy /esr1.jpg 304 Not Modified 0 0 0.07 -- +3 GET vitaly.sexy /construction.gif 304 Not Modified 0 0 0.07 -- +2 GET vitaly.sexy /vitaly2.jpg 0 N/A -- -- +1 GET vitaly.sexy / 304 Not Modified 0 0 0.07 -- +pappy> !ls +cmdhistory config.json data.db +pappy> +``` + +### Running Python Code + +You can use the `py` command to either run python code or to drop down to a Python shell. + +``` +pappy> py print ':D '*10 +:D :D :D :D :D :D :D :D :D :D +pappy> py +Python 2.7.6 (default, Jun 22 2015, 17:58:13) +[GCC 4.8.2] on linux2 +Type "help", "copyright", "credits" or "license" for more information. +(ProxyCmd) + + py : Executes a Python command. + py: Enters interactive Python mode. + End with ``Ctrl-D`` (Unix) / ``Ctrl-Z`` (Windows), ``quit()``, '`exit()``. + Non-python commands can be issued with ``cmd("your command")``. + Run python code from external files with ``run("filename.py")`` + +>>> from pappyproxy import config +>>> config.CONFIG_DICT +{u'data_file': u'./data.db', u'history_size': 1000, u'cert_dir': u'{DATADIR}/certs', u'proxy_listeners': [{u'interface': u'127.0.0.1', u'port': 8000}]} +>>> exit() +pappy> +``` + +### Redirect Output To File + +You can use `>` to direct output to a file. However, a number of commands use colored output. If you just redirect these to a file, there will be additional bytes which represent the ANSI color codes. To get around this, use the `nocolor` command to remove the color from the command output. + +``` +pappy> ls > ls.txt +pappy> !xxd -c 32 -g 4 ls.txt +0000000: 1b5b316d 1b5b346d 49442020 56657262 2020486f 73742020 20202020 20202050 .[1m.[4mID Verb Host P +0000020: 61746820 20202020 20202020 20202020 2020532d 436f6465 20202020 20202020 ath S-Code +0000040: 20202020 52657120 4c656e20 20527370 204c656e 20205469 6d652020 20204d6e Req Len Rsp Len Time Mn +0000060: 676c2020 1b5b306d 0a352020 201b5b33 366d4745 541b5b30 6d202020 1b5b3931 gl .[0m.5 .[36mGET.[0m .[91 +0000080: 6d766974 616c792e 73657879 1b5b306d 20201b5b 33366d1b 5b306d2f 1b5b3334 mvitaly.sexy.[0m .[36m.[0m/.[34 +00000a0: 6d6e6574 73636170 652e6769 661b5b30 6d202020 2020201b 5b33356d 33303420 mnetscape.gif.[0m .[35m304 +00000c0: 4e6f7420 4d6f6469 66696564 1b5b306d 20203020 20202020 20202030 20202020 Not Modified.[0m 0 0 +00000e0: 20202020 302e3038 20202020 2d2d2020 20200a34 2020201b 5b33366d 4745541b 0.08 -- .4 .[36mGET. +0000100: 5b306d20 20201b5b 39316d76 6974616c 792e7365 78791b5b 306d2020 1b5b3336 [0m .[91mvitaly.sexy.[0m .[36 +0000120: 6d1b5b30 6d2f1b5b 33346d65 7372312e 6a70671b 5b306d20 20202020 20202020 m.[0m/.[34mesr1.jpg.[0m +0000140: 201b5b33 356d3330 34204e6f 74204d6f 64696669 65641b5b 306d2020 30202020 .[35m304 Not Modified.[0m 0 +0000160: 20202020 20302020 20202020 2020302e 30372020 20202d2d 20202020 0a332020 0 0.07 -- .3 +0000180: 201b5b33 366d4745 541b5b30 6d202020 1b5b3931 6d766974 616c792e 73657879 .[36mGET.[0m .[91mvitaly.sexy +00001a0: 1b5b306d 20201b5b 33366d1b 5b306d2f 1b5b3334 6d636f6e 73747275 6374696f .[0m .[36m.[0m/.[34mconstructio +00001c0: 6e2e6769 661b5b30 6d20201b 5b33356d 33303420 4e6f7420 4d6f6469 66696564 n.gif.[0m .[35m304 Not Modified +00001e0: 1b5b306d 20203020 20202020 20202030 20202020 20202020 302e3037 20202020 .[0m 0 0 0.07 +0000200: 2d2d2020 20200a32 2020201b 5b33366d 4745541b 5b306d20 20201b5b 39316d76 -- .2 .[36mGET.[0m .[91mv +0000220: 6974616c 792e7365 78791b5b 306d2020 1b5b3336 6d1b5b30 6d2f1b5b 33346d76 italy.sexy.[0m .[36m.[0m/.[34mv +0000240: 6974616c 79322e6a 70671b5b 306d2020 20202020 201b5b33 366d3230 30204f4b italy2.jpg.[0m .[36m200 OK +0000260: 1b5b306d 20202020 20202020 20202020 30202020 20202020 20323033 34303033 .[0m 0 2034003 +0000280: 20203135 352e3131 20202d2d 20202020 0a312020 201b5b33 366d4745 541b5b30 155.11 -- .1 .[36mGET.[0 +00002a0: 6d202020 1b5b3931 6d766974 616c792e 73657879 1b5b306d 20201b5b 33366d1b m .[91mvitaly.sexy.[0m .[36m. +00002c0: 5b306d2f 1b5b3334 6d1b5b30 6d202020 20202020 20202020 20202020 2020201b [0m/.[34m.[0m . +00002e0: 5b33356d 33303420 4e6f7420 4d6f6469 66696564 1b5b306d 20203020 20202020 [35m304 Not Modified.[0m 0 +0000300: 20202030 20202020 20202020 302e3037 20202020 2d2d2020 20200a 0 0.07 -- . +pappy> nocolor ls > ls2.txt +pappy> !xxd -c 32 -g 4 ls2.txt +0000000: 49442020 56657262 2020486f 73742020 20202020 20202050 61746820 20202020 ID Verb Host Path +0000020: 20202020 20202020 2020532d 436f6465 20202020 20202020 20202020 52657120 S-Code Req +0000040: 4c656e20 20527370 204c656e 20205469 6d652020 20204d6e 676c2020 0a352020 Len Rsp Len Time Mngl .5 +0000060: 20474554 20202076 6974616c 792e7365 78792020 2f6e6574 73636170 652e6769 GET vitaly.sexy /netscape.gi +0000080: 66202020 20202033 3034204e 6f74204d 6f646966 69656420 20302020 20202020 f 304 Not Modified 0 +00000a0: 20203020 20202020 20202030 2e303820 2020202d 2d202020 200a3420 20204745 0 0.08 -- .4 GE +00000c0: 54202020 76697461 6c792e73 65787920 202f6573 72312e6a 70672020 20202020 T vitaly.sexy /esr1.jpg +00000e0: 20202020 33303420 4e6f7420 4d6f6469 66696564 20203020 20202020 20202030 304 Not Modified 0 0 +0000100: 20202020 20202020 302e3037 20202020 2d2d2020 20200a33 20202047 45542020 0.07 -- .3 GET +0000120: 20766974 616c792e 73657879 20202f63 6f6e7374 72756374 696f6e2e 67696620 vitaly.sexy /construction.gif +0000140: 20333034 204e6f74 204d6f64 69666965 64202030 20202020 20202020 30202020 304 Not Modified 0 0 +0000160: 20202020 20302e30 37202020 202d2d20 2020200a 32202020 47455420 20207669 0.07 -- .2 GET vi +0000180: 74616c79 2e736578 7920202f 76697461 6c79322e 6a706720 20202020 20203230 taly.sexy /vitaly2.jpg 20 +00001a0: 30204f4b 20202020 20202020 20202020 30202020 20202020 20323033 34303033 0 OK 0 2034003 +00001c0: 20203135 352e3131 20202d2d 20202020 0a312020 20474554 20202076 6974616c 155.11 -- .1 GET vital +00001e0: 792e7365 78792020 2f202020 20202020 20202020 20202020 20202033 3034204e y.sexy / 304 N +0000200: 6f74204d 6f646966 69656420 20302020 20202020 20203020 20202020 20202030 ot Modified 0 0 0 +0000220: 2e303720 2020202d 2d202020 200a0a .07 -- .. +pappy> +``` + +If you want to write the contents of a request or response to a file, don't use `nocolor` with `vfq` or `vfs`. Use just the `vbq` or `vbs` commands. + +| Command | Description | +|:--------|:------------| +| `nocolor` | Run a command and print its output without ASCII escape codes. Intended for use when redirecting output to a file. Should only be used with text and not with binary data. | + + Generating Pappy's CA Cert -------------------------- In order to intercept and modify requests to sites that use HTTPS, you have to generate and install CA certs to your browser. You can do this by running the `gencerts` command in Pappy. By default, certs are stored `~/.pappy/certs`. This is also the default location that Pappy will look for certificates (unless you specify otherwise in `config.json`.) In addition, you can give the `gencerts` command an argument to have it put the generated certs in a different directory. @@ -110,11 +219,13 @@ The following commands can be used to view requests and responses | Command | Aliases | Description | |:--------|:--------|:------------| | `ls [a|`]| list, ls |List requests that are in the current context (see Context section). Has information like the host, target path, and status code. With no arguments, it will print the 25 most recent requests in the current context. If you pass 'a' or 'all' as an argument, it will print all the requests in the current context. If you pass a number "n" as an argument, it will print the n most recent requests in the current context. | -| `sm` | sm, site_map | Print a tree showing the site map. It will display all requests in the current context that did not have a 404 response. | +| `sm` | sm, site_map | Print a tree showing the site map. It will display all requests in the current context that did not have a 404 response. This has to go through all of the requests in the current context so it may be slow. | | `viq ` | view_request_info, viq | View additional information about requests. Includes the target port, if SSL was used, applied tags, and other information. | | `vfq ` | view_full_request, vfq | [V]iew [F]ull Re[Q]uest, prints the full request including headers and data. | +| `vbq ` | view_request_bytes, vbq | [V]iew [B]ytes of Re[Q]uest, prints the full request including headers and data without coloring or additional newlines. Use this if you want to write a request to a file. | | `vhq ` | view_request_headers, vhq | [V]iew [H]eaders of a Re[Q]uest. Prints just the headers of a request. | | `vfs ` | view_full_response, vfs |[V]iew [F]ull Re[S]ponse, prints the full response associated with a request including headers and data. | +| `vbs ` | view_response_bytes, vbs | [V]iew [B]ytes of Re[S]ponse, prints the full response including headers and data without coloring or additional newlines. Use this if you want to write a response to a file. | | `vhs ` | view_response_headers, vhs | [V]iew [H]eaders of a Re[S]ponse. Prints just the headers of a response associated with a request. | The table shown by `ls` will have the following columns: @@ -326,6 +437,8 @@ The following commands can be used to encode/decode strings: |`base64_encode`|`base64_encode`, `b64e` | Base64 encode a string | |`asciihex_decode`|`asciihex_decode`, `ahd` | Decode an ASCII hex string | |`asciihex_encode`|`asciihex_encode`, `ahe` | Encode an ASCII hex string | +|`html_decode`|`html_decode`, `htmld` | Decode an html encoded string | +|`html_encode`|`html_encode`, `htmle` | Encode a string to html encode all of the characters | |`url_decode`|`url_decode`, `urld` | Url decode a string | |`url_encode`|`url_encode`, `urle` | Url encode a string | |`gzip_decode`|`gzip_decode`, `gzd` | Gzip decompress a string. Probably won't work too well since there's not a great way to get binary data passed in as an argument. I'm working on this. | @@ -334,6 +447,8 @@ The following commands can be used to encode/decode strings: |`base64_encode_raw`|`base64_encode_raw`, `b64er` | Same as `base64_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | |`asciihex_decode_raw`|`asciihex_decode_raw`, `ahdr` | Same as `asciihex_decode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | |`asciihex_encode_raw`|`asciihex_encode_raw`, `aher` | Same as `asciihex_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`html_decode_raw`|`html_decode_raw`, `htmldr` | Same as `html_decode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`html_encode_raw`|`html_encode_raw`, `htmler` | Same as `html_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | |`url_decode_raw`|`url_decode_raw`, `urldr` | Same as `url_decode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | |`url_encode_raw`|`url_encode_raw`, `urler` | Same as `url_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | |`gzip_decode_raw`|`gzip_decode_raw`, `gzdr` | Same as `gzip_decode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | @@ -829,6 +944,10 @@ Changelog --------- The boring part of the readme +* 0.2.4 + * Add command history saving between sessions + * Add html encoder/decoder + * All the bugs were fixed so I added some more for 0.2.5 * 0.2.3 * Decoder functions * Add `merge` command diff --git a/docs/source/conf.py b/docs/source/conf.py index d83fdf2..6ff9673 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -59,9 +59,9 @@ author = u'Rob Glew' # built documents. # # The short X.Y version. -version = u'0.2.3' +version = u'0.2.4' # The full version, including alpha/beta/rc tags. -release = version +release = u'0.2.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pappyproxy/config.py b/pappyproxy/config.py index 6be479c..e8a39de 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -52,6 +52,12 @@ The configuration settings for the proxy. :Default: ``['{DATA_DIR}/plugins', '{PAPPY_DIR}/plugins']`` +.. data: SAVE_HISTORY + + Whether command history should be saved to a file/loaded at startup. + + :Default: True + .. data: CONFIG_DICT The dictionary read from config.json. When writing plugins, use this to load @@ -85,6 +91,8 @@ LISTENERS = [(8000, '127.0.0.1')] SSL_CA_FILE = 'certificate.crt' SSL_PKEY_FILE = 'private.key' +HISTSIZE = 1000 + PLUGIN_DIRS = [os.path.join(DATA_DIR, 'plugins'), os.path.join(PAPPY_DIR, 'plugins')] CONFIG_DICT = {} @@ -108,6 +116,7 @@ def load_settings(proj_config): global DATA_DIR global SSL_CA_FILE global SSL_PKEY_FILE + global HISTSIZE # Substitution dictionary subs = {} @@ -134,6 +143,10 @@ def load_settings(proj_config): for l in proj_config["proxy_listeners"]: LISTENERS.append((l['port'], l['interface'])) + # History saving settings + if "history_size" in proj_config: + HISTSIZE = proj_config['history_size'] + def load_global_settings(global_config): from .http import Request global CACHE_SIZE diff --git a/pappyproxy/console.py b/pappyproxy/console.py index 2698b88..10efe2f 100644 --- a/pappyproxy/console.py +++ b/pappyproxy/console.py @@ -3,14 +3,19 @@ Contains helpers for interacting with the console. Includes definition for the class that is used to run the console. """ +import StringIO +import atexit import cmd2 +import os import re +import readline import string import sys import itertools from .util import PappyException from .colors import Styles, Colors, verb_color, scode_color, path_formatter, host_color +from . import config from twisted.internet import defer ################### @@ -262,7 +267,17 @@ class ProxyCmd(cmd2.Cmd): self._cmds = {} self._aliases = {} + + atexit.register(self.save_histfile) + readline.set_history_length(config.HISTSIZE) + if os.path.exists('cmdhistory'): + if config.HISTSIZE != 0: + readline.read_history_file('cmdhistory') + else: + os.remove('cmdhistory') + cmd2.Cmd.__init__(self, *args, **kwargs) + def __dir__(self): # Hack to get cmd2 to detect that we can run a command @@ -319,6 +334,12 @@ class ProxyCmd(cmd2.Cmd): if self._cmds[real_command][1]: return self._cmds[real_command][1] raise AttributeError(attr) + + def save_histfile(self): + # Write the command to the history file + if config.HISTSIZE != 0: + readline.set_history_length(config.HISTSIZE) + readline.write_history_file('cmdhistory') def get_names(self): # Hack to get cmd to recognize do_/etc functions as functions for things @@ -356,4 +377,15 @@ class ProxyCmd(cmd2.Cmd): """ for command, alias in alias_list: self.add_alias(command, alias) - + +# Taken from http://stackoverflow.com/questions/16571150/how-to-capture-stdout-output-from-a-python-function-call +# then modified +class Capturing(): + def __enter__(self): + self._stdout = sys.stdout + sys.stdout = self._stringio = StringIO.StringIO() + return self + + def __exit__(self, *args): + self.val = self._stringio.getvalue() + sys.stdout = self._stdout diff --git a/pappyproxy/default_user_config.json b/pappyproxy/default_user_config.json index 8993bb8..1960092 100644 --- a/pappyproxy/default_user_config.json +++ b/pappyproxy/default_user_config.json @@ -1,6 +1,7 @@ { "data_file": "./data.db", "cert_dir": "{DATADIR}/certs", + "history_size": 1000, "proxy_listeners": [ {"port": 8000, "interface": "127.0.0.1"} ] diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index da20eea..20557d7 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -72,6 +72,7 @@ def main(): conf_settings = config.get_default_config() conf_settings['debug_dir'] = None conf_settings['debug_to_file'] = False + conf_settings['history_size'] = 0 with tempfile.NamedTemporaryFile(delete=False) as tf: conf_settings['data_file'] = tf.name print 'Temporary datafile is %s' % tf.name diff --git a/pappyproxy/plugins/debug.py b/pappyproxy/plugins/debug.py index a6c4a71..5e805bd 100644 --- a/pappyproxy/plugins/debug.py +++ b/pappyproxy/plugins/debug.py @@ -145,7 +145,7 @@ def load_cmds(cmd): 'gccollect': (collect, None), 'graphobj': (graph_randobj, None), 'meminfo': (memory_info, None), - 'bigdata': (big_fucking_data_file, None), + 'genbigdata': (big_fucking_data_file, None), 'checkcache': (check_cache, None), 'loadblock': (loadblock, None), 'time': (time_cmd, None), diff --git a/pappyproxy/plugins/decode.py b/pappyproxy/plugins/decode.py index d690d0d..200a324 100644 --- a/pappyproxy/plugins/decode.py +++ b/pappyproxy/plugins/decode.py @@ -1,3 +1,4 @@ +import HTMLParser import StringIO import base64 import clipboard @@ -42,7 +43,13 @@ def gzip_decode_helper(s): dec_data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(s)) dec_data = dec_data.read() return dec_data - + +def html_encode_helper(s): + return ''.join(['&#x{0:x};'.format(ord(c)) for c in s]) + +def html_decode_helper(s): + return HTMLParser.HTMLParser().unescape(s) + def _code_helper(line, func, copy=True): args = shlex.split(line) if not args: @@ -110,6 +117,22 @@ def asciihex_encode(line): Results are copied to the clipboard. """ print_maybe_bin(_code_helper(line, asciihex_encode_helper)) + +def html_decode(line): + """ + Decode an html encoded string. + If no string is given, will decode the contents of the clipboard. + Results are copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, html_decode_helper)) + +def html_encode(line): + """ + Encode a string and escape html control characters. + If no string is given, will encode the contents of the clipboard. + Results are copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, html_encode_helper)) def gzip_decode(line): """ @@ -175,6 +198,22 @@ def asciihex_encode_raw(line): """ print _code_helper(line, asciihex_encode_helper, copy=False) +def html_decode_raw(line): + """ + Same as html_decode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, html_decode_helper, copy=False) + +def html_encode_raw(line): + """ + Same as html_encode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, html_encode_helper, copy=False) + def gzip_decode_raw(line): """ Same as gzip_decode but the output will never be printed as a hex dump and @@ -199,6 +238,8 @@ def load_cmds(cmd): 'asciihex_encode': (asciihex_encode, None), 'url_decode': (url_decode, None), 'url_encode': (url_encode, None), + 'html_decode': (html_decode, None), + 'html_encode': (html_encode, None), 'gzip_decode': (gzip_decode, None), 'gzip_encode': (gzip_encode, None), 'base64_decode_raw': (base64_decode_raw, None), @@ -207,6 +248,8 @@ def load_cmds(cmd): 'asciihex_encode_raw': (asciihex_encode_raw, None), 'url_decode_raw': (url_decode_raw, None), 'url_encode_raw': (url_encode_raw, None), + 'html_decode_raw': (html_decode_raw, None), + 'html_encode_raw': (html_encode_raw, None), 'gzip_decode_raw': (gzip_decode_raw, None), 'gzip_encode_raw': (gzip_encode_raw, None), }) @@ -217,6 +260,8 @@ def load_cmds(cmd): ('asciihex_encode', 'ahe'), ('url_decode', 'urld'), ('url_encode', 'urle'), + ('html_decode', 'htmld'), + ('html_encode', 'htmle'), ('gzip_decode', 'gzd'), ('gzip_encode', 'gze'), ('base64_decode_raw', 'b64dr'), @@ -225,6 +270,8 @@ def load_cmds(cmd): ('asciihex_encode_raw', 'aher'), ('url_decode_raw', 'urldr'), ('url_encode_raw', 'urler'), + ('html_decode_raw', 'htmldr'), + ('html_encode_raw', 'htmler'), ('gzip_decode_raw', 'gzdr'), ('gzip_encode_raw', 'gzer'), ]) diff --git a/pappyproxy/plugins/misc.py b/pappyproxy/plugins/misc.py index fb2b3af..ff48cc4 100644 --- a/pappyproxy/plugins/misc.py +++ b/pappyproxy/plugins/misc.py @@ -2,10 +2,10 @@ import crochet import pappyproxy import shlex -from pappyproxy.console import confirm, load_reqlist -from pappyproxy.util import PappyException -from pappyproxy.http import Request +from pappyproxy.console import confirm, load_reqlist, Capturing +from pappyproxy.util import PappyException, remove_color from pappyproxy.requestcache import RequestCache +from pappyproxy.pappy import cons from twisted.internet import defer from twisted.enterprise import adbapi @@ -114,7 +114,12 @@ def merge_datafile(line): print 'Added %d requests' % count finally: other_dbpool.close() - + +def run_without_color(line): + with Capturing() as output: + cons.onecmd(line.strip()) + print remove_color(output.val) + def load_cmds(cmd): cmd.set_cmds({ 'clrmem': (clrmem, None), @@ -122,7 +127,8 @@ def load_cmds(cmd): 'sv': (save, None), 'export': (export, None), 'log': (log, None), - 'merge': (merge_datafile, None) + 'merge': (merge_datafile, None), + 'nocolor': (run_without_color, None), }) cmd.add_aliases([ #('rpy', ''), diff --git a/pappyproxy/plugins/view.py b/pappyproxy/plugins/view.py index 1ec7cf9..eeb4e47 100644 --- a/pappyproxy/plugins/view.py +++ b/pappyproxy/plugins/view.py @@ -178,9 +178,7 @@ def list_reqs(line): def view_request_info(line): """ View information about request - Usage: view_request_info [u] - If 'u' is given as an additional argument, the unmangled version - of the request will be displayed. + Usage: view_request_info """ args = shlex.split(line) reqids = args[0] @@ -197,9 +195,7 @@ def view_request_info(line): def view_request_headers(line): """ View the headers of the request - Usage: view_request_headers [u] - If 'u' is given as an additional argument, the unmangled version - of the request will be displayed. + Usage: view_request_headers """ args = shlex.split(line) reqid = args[0] @@ -208,10 +204,10 @@ def view_request_headers(line): for req in reqs: if len(reqs) > 1: print 'Request %s:' % req.reqid - print '' view_full_message(req, True) if len(reqs) > 1: print '-'*30 + print '' @crochet.wait_for(timeout=None) @@ -219,9 +215,7 @@ def view_request_headers(line): def view_full_request(line): """ View the full data of the request - Usage: view_full_request [u] - If 'u' is given as an additional argument, the unmangled version - of the request will be displayed. + Usage: view_full_request """ args = shlex.split(line) reqid = args[0] @@ -230,18 +224,36 @@ def view_full_request(line): for req in reqs: if len(reqs) > 1: print 'Request %s:' % req.reqid - print '' view_full_message(req) if len(reqs) > 1: print '-'*30 + print '' + +@crochet.wait_for(timeout=None) +@defer.inlineCallbacks +def view_request_bytes(line): + """ + View the raw bytes of the request. Use this if you want to redirect output to a file. + Usage: view_request_bytes + """ + args = shlex.split(line) + reqid = args[0] + reqs = yield load_reqlist(reqid) + for req in reqs: + if len(reqs) > 1: + print 'Request %s:' % req.reqid + print req.full_message + if len(reqs) > 1: + print '-'*30 + print '' @crochet.wait_for(timeout=None) @defer.inlineCallbacks def view_response_headers(line): """ View the headers of the response - Usage: view_response_headers + Usage: view_response_headers """ reqs = yield load_reqlist(line) for req in reqs: @@ -269,7 +281,22 @@ def view_full_response(line): else: print "Request %s does not have a response" % req.reqid - +@crochet.wait_for(timeout=None) +@defer.inlineCallbacks +def view_response_bytes(line): + """ + View the full data of the response associated with a request + Usage: view_request_bytes + """ + reqs = yield load_reqlist(line) + for req in reqs: + if req.response: + if len(reqs) > 1: + print '-'*15 + (' %s ' % req.reqid) + '-'*15 + print req.response.full_message + else: + print "Request %s does not have a response" % req.reqid + @crochet.wait_for(timeout=None) @defer.inlineCallbacks def dump_response(line): @@ -317,8 +344,10 @@ def load_cmds(cmd): 'view_request_info': (view_request_info, None), 'view_request_headers': (view_request_headers, None), 'view_full_request': (view_full_request, None), + 'view_request_bytes': (view_request_bytes, None), 'view_response_headers': (view_response_headers, None), 'view_full_response': (view_full_response, None), + 'view_response_bytes': (view_response_bytes, None), 'site_map': (site_map, None), 'dump_response': (dump_response, None), }) @@ -327,8 +356,10 @@ def load_cmds(cmd): ('view_request_info', 'viq'), ('view_request_headers', 'vhq'), ('view_full_request', 'vfq'), + ('view_request_bytes', 'vbq'), ('view_response_headers', 'vhs'), - ('site_map', 'sm'), ('view_full_response', 'vfs'), + ('view_response_bytes', 'vbs'), + ('site_map', 'sm'), #('dump_response', 'dr'), ]) diff --git a/pappyproxy/util.py b/pappyproxy/util.py index 5c61da6..56020d4 100644 --- a/pappyproxy/util.py +++ b/pappyproxy/util.py @@ -1,3 +1,4 @@ +import re import string import time import datetime @@ -25,6 +26,10 @@ def printable_data(data): chars += '.' return ''.join(chars) +def remove_color(s): + ansi_escape = re.compile(r'\x1b[^m]*m') + return ansi_escape.sub('', s) + # Taken from http://stackoverflow.com/questions/4770297/python-convert-utc-datetime-string-to-local-datetime def utc2local(utc): epoch = time.mktime(utc.timetuple()) diff --git a/setup.py b/setup.py index 5ed3943..8d7c947 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import pkgutil from setuptools import setup, find_packages -VERSION = '0.2.3' +VERSION = '0.2.4' setup(name='pappyproxy', version=VERSION,