From f4274e1e82c644f0cb40fe796b9e9ee40d5c77cc Mon Sep 17 00:00:00 2001 From: Rob Glew Date: Thu, 18 Feb 2016 15:29:43 -0600 Subject: [PATCH] Version 0.2.7 --- README.md | 110 ++- docs/source/conf.py | 4 +- docs/source/overview.rst | 645 ++++++++++++++++- pappyproxy/Makefile | 3 + pappyproxy/colors.py | 2 + pappyproxy/comm.py | 5 +- pappyproxy/config.py | 41 +- pappyproxy/context.py | 114 ++- pappyproxy/default_user_config.json | 3 +- pappyproxy/http.py | 63 +- pappyproxy/macros.py | 82 ++- pappyproxy/pappy.py | 21 +- pappyproxy/plugin.py | 42 +- pappyproxy/plugins/debug.py | 16 +- pappyproxy/plugins/decode.py | 13 +- pappyproxy/plugins/macrocmds.py | 10 +- pappyproxy/plugins/manglecmds.py | 2 +- pappyproxy/plugins/misc.py | 5 +- pappyproxy/plugins/view.py | 147 +++- pappyproxy/plugins/vim_repeater/repeater.py | 2 +- pappyproxy/proxy.py | 386 ++++++----- pappyproxy/requestcache.py | 4 +- pappyproxy/tests/test_macros.py | 65 ++ pappyproxy/tests/test_proxy.py | 733 ++++++++++++++------ pappyproxy/tests/testutil.py | 81 ++- pappyproxy/util.py | 11 +- setup.py | 3 +- 27 files changed, 2128 insertions(+), 485 deletions(-) create mode 100644 pappyproxy/tests/test_macros.py diff --git a/README.md b/README.md index ad84add..c6b7fe9 100644 --- a/README.md +++ b/README.md @@ -219,16 +219,18 @@ The following commands can be used to view requests and responses | Command | Aliases | Description | |:--------|:--------|:------------| | `ls [a|`]| list, ls |List requests that are in the current context (see Context section). Has information like the host, target path, and status code. With no arguments, it will print the 25 most recent requests in the current context. If you pass 'a' or 'all' as an argument, it will print all the requests in the current context. If you pass a number "n" as an argument, it will print the n most recent requests in the current context. | -| `sm` | sm, site_map | Print a tree showing the site map. It will display all requests in the current context that did not have a 404 response. This has to go through all of the requests in the current context so it may be slow. | +| `sm` [p] | sm, site_map | Print a tree showing the site map. It will display all requests in the current context that did not have a 404 response. This has to go through all of the requests in the current context so it may be slow. If the `p` option is given, it will print the paths as paths rather than as a tree. | | `viq ` | view_request_info, viq | View additional information about requests. Includes the target port, if SSL was used, applied tags, and other information. | -| `vfq ` | view_full_request, vfq | [V]iew [F]ull Re[Q]uest, prints the full request including headers and data. | +| `vfq ` | view_full_request, vfq, kjq | [V]iew [F]ull Re[Q]uest, prints the full request including headers and data. | | `vbq ` | view_request_bytes, vbq | [V]iew [B]ytes of Re[Q]uest, prints the full request including headers and data without coloring or additional newlines. Use this if you want to write a request to a file. | -| `ppq [format]` | pretty_print_request, ppq | Pretty print a request. If a format is given, it will try and print the body of the request with that format. Otherwise it will make a guess based off of the Content-Type header. | +| `ppq ` | pretty_print_request, ppq | Pretty print a request with a specific format. See the table below for a list of formats. | | `vhq ` | view_request_headers, vhq | [V]iew [H]eaders of a Re[Q]uest. Prints just the headers of a request. | -| `vfs ` | view_full_response, vfs |[V]iew [F]ull Re[S]ponse, prints the full response associated with a request including headers and data. | +| `vfs ` | view_full_response, vfs, kjs |[V]iew [F]ull Re[S]ponse, prints the full response associated with a request including headers and data. | | `vhs ` | view_response_headers, vhs | [V]iew [H]eaders of a Re[S]ponse. Prints just the headers of a response associated with a request. | | `vbs ` | view_response_bytes, vbs | [V]iew [B]ytes of Re[S]ponse, prints the full response including headers and data without coloring or additional newlines. Use this if you want to write a response to a file. | -| `pps [format]` | pretty_print_response, pps | Pretty print a response. If a format is given, it will try and print the body of the response with that format. Otherwise it will make a guess based off of the Content-Type header. | +| `pps ` | pretty_print_response, pps | Pretty print a response with a specific format. See the table below for a list of formats. | +| `pprm ` | print_params, pprm | Print a summary of the parameters submitted with the request. It will include URL params, POST params, and/or cookies | +| `pri [ct] [key(s)] | param_info, pri | Print a summary of the parameters and values submitted by in-context requests. You can pass in keys to limit which values will be shown. If you also provide `ct` as the first argument, it will include any keys that are passed as arguments. | | `watch` | watch | Print requests and responses in real time as they pass through the proxy. | Available formats for `ppq` and `pps` commands: @@ -362,9 +364,13 @@ Matches both A and B but not C | host | host, domain, hs, dm | The target host (ie www.target.com) | String | | path | path, pt | The path of the url (ie /path/to/secrets.php) | String | | body | body, data, bd, dt | The body (data section) of either the request or the response | String | +| reqbody | qbody, qdata, qbd, qdt | The body (data section) of th request | String | +| rspbody | sbody, sdata, sbd, sdt | The body (data section) of th response | String | | verb | verb, vb | The HTTP verb of the request (ie GET, POST) | String | | param | param, pm | Either the get or post parameters | Key/Value | | header | header, hd | An HTTP header (ie User-Agent, Basic-Authorization) in the request or response | Key/Value | +| reqheader | reqheader, qhd | An HTTP header in the request | Key/Value | +| rspheader | rspheader, shd | An HTTP header in the response | Key/Value | | rawheaders | rawheaders, rh | The entire header section (as one string) of either the head or the response | String | | sentcookie | sentcookie, sck | A cookie sent in a request | Key/Value | | setcookie | setcookie, stck | A cookie set by a response | Key/Value | @@ -392,6 +398,20 @@ A few filters don't conform to the field, comparer, value format. You can still |:--|:--|:--| | before | before, bf, b4 | Filters out any request that is not before the given request. Filters out any request without a time. | | after | after, af | Filters out any request that is not before the given request. Filters out any request without a time. | +| inv | inf | Inverts a filter string. Anything that matches the filter string will not pass the filter. | + +Examples: + +``` +Only show requests before request 1234 + f b4 1234 + +Only show requests after request 1234 + f af 1234 + +Show requests without a csrf parameter + f inv param ct csrf +``` Scope ----- @@ -463,6 +483,7 @@ The following commands can be used to encode/decode strings: |`url_encode_raw`|`url_encode_raw`, `urler` | Same as `url_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | |`gzip_decode_raw`|`gzip_decode_raw`, `gzdr` | Same as `gzip_decode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | |`gzip_encode_raw`|`gzip_encode_raw`, `gzer` | Same as `gzip_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`unixtime_decode`| `unixtime_decode`, `uxtd` | Take in a unix timestamp and print a human readable timestamp | Interceptor ----------- @@ -923,6 +944,75 @@ Settings included in `~/.pappy/global_config.json`: |:--------|:------------| | cache_size | The number of requests from history that will be included in memory at any given time. Set to -1 to keep everything in memory. See the request cache section for more info. | +Using a SOCKS Server +-------------------- +Pappy allows you to use an upstream SOCKS server. You can do this by adding a `socks_proxy` value to config.json. You can use the following for anonymous access to the proxy: + +``` + "socks_proxy": {"host":"socks.proxy.host", "port":5555} +``` + +To use credentials you add a `username` and `password` value to the dictionary: + +``` + "socks_proxy": {"host":"socks.proxy.host", "port":5555, "username": "mario", "password":"ilovemushrooms"} +``` + +Anything that passes through any of the active listeners will use the proxy. + +Transparent Host Redirection +---------------------------- +Sometimes you get a frustrating thick client that doesn’t let you mess with proxy settings to get it to go through a proxy. However, if you can redirect where it sends its traffic to localhost, you can get Pappy to take that traffic and redirect it to go where it should. + +It takes root permissions to listen on low numbered ports. As a result, we’ll need to do some root stuff to listen on ports 80 and 443 and get the data to Pappy. There are two ways to get the traffic to Pappy. The first is to set up port forwarding as root to send traffic from localhost:80 to localhost:8080 and localhost:443 to localhost:8443 (since we can listen on 8080 and 8443 without root). Or you can YOLO, run Pappy as root and just have it listen on 80 and 443. + +According to Google you can use the following command to forward port 80 on localhost to 8080 on Linux: + +``` +iptables -t nat -A PREROUTING -i ppp0 -p tcp --dport 80 -j REDIRECT --to-ports 8080 +``` + +Then to route 443 to 8443: + +``` +iptables -t nat -A PREROUTING -i ppp0 -p tcp --dport 443 -j REDIRECT --to-ports 8443 +``` + +Of course, both of these need to be run as root. + +Then on mac it’s + +``` +echo " +rdr pass inet proto tcp from any to any port 80 -> 127.0.0.1 port 8080 +rdr pass inet proto tcp from any to any port 443 -> 127.0.0.1 port 8443 +" | sudo pfctl -ef - +Then to turn it off on mac it’s +sudo pfctl -F all -f /etc/pf.conf +``` + +Then modify the listener settings in the project’s config.json to be: + +``` +"proxy_listeners": [ + {"port": 8080, "interface": "127.0.0.1", "forward_host": "www.example.faketld"}, + {"port": 8443, "interface": "127.0.0.1", "forward_host_ssl": "www.example.faketld"}, + ] +``` + +This configuration will cause Pappy to open a port on 8080 that will accept connections normally and a port on 8443 which will accept SSL connections. The forward_host setting tells Pappy to redirect any requests sent to the port to the given host. It will also update the request’s host header. forward_host_ssl does the same thing, but it listens for SSL connections and forces the connection to use SSL. + +Or if you’re going to YOLO it do the same thing then listen on port 80/443 directly. I do not suggest you do this. + +``` +"proxy_listeners": [ + {"port": 80, "interface": "127.0.0.1", "forward_host": "www.example.faketld"}, + {"port": 443, "interface": "127.0.0.1", "forward_host_ssl": "www.example.faketld"}, + ] +``` + +Pappy will automatically use this host to make the connection and forward the request to the new server. + FAQ --- @@ -954,6 +1044,16 @@ Changelog --------- The boring part of the readme +* 0.2.7 + * boring unit tests + * should make future releases more stable I guess + * Support for upstream SOCKS servers + * `print_params` command + * `inv` filter + * `param_info` command + * Filters by request/response only headers/body + * Transparent host redirection + * Some easier to type aliases for common commands * 0.2.6 * Fix pip being dumb * `watch` command to watch requests/responses in real time diff --git a/docs/source/conf.py b/docs/source/conf.py index c2151cb..e768b25 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -59,9 +59,9 @@ author = u'Rob Glew' # built documents. # # The short X.Y version. -version = u'0.2.6' +version = u'0.2.7' # The full version, including alpha/beta/rc tags. -release = u'0.2.6' +release = u'0.2.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/overview.rst b/docs/source/overview.rst index f135ae9..3a8ffa7 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -1,6 +1,9 @@ The Pappy Proxy =============== +`Documentation `__ - +`Tutorial `__ + Introduction ------------ @@ -10,8 +13,8 @@ testing. Its features are often similar, or straight up rippoffs from `Burp Suite `__. However, Burp Suite is neither open source nor a command line tool, thus making a proxy like Pappy inevitable. The project is still in its early stages, so there are -bugs and only the bare minimum features, but it should be able to do -some cool stuff soon (I'm already using it for real work). +bugs and only the bare minimum features, but it can already do some cool +stuff. Contributing ------------ @@ -28,6 +31,15 @@ Right now the codebase is kind of rough and I have refactored it a few times already, but I would be more than happy to find a stable part of the codebase that you can contribute to. +Another option is to try writing a plugin. It might be a bit easier than +contributing code and plugins are extremely easy to integrate as a core +feature. So you can also contribute by writing a plugin and letting me +know about it. You can find out more by looking at `the official plugin +docs `__. + +You can find ideas for features to add on `the contributing page in the +docs `__. + How to Use It ============= @@ -49,9 +61,9 @@ Quickstart ---------- Pappy projects take up an entire directory. Any generated scripts, -exported responses, etc. will be placed in the current directory so it's -good to give your project a directory of its own. To start a project, do -something like: +exported responses, plugin data, etc. will be placed in the current +directory so it's good to give your project a directory of its own. To +start a project, do something like: :: @@ -164,6 +176,130 @@ The following tokens will also be replaced with values: See the default ``config.json`` for examples. +General Console Techniques +-------------------------- + +There are a few tricks you can use in general when using Pappy's +console. Most of these are provided by the +`cmd `__ and +`cmd2 `__. + +Run a shell command +~~~~~~~~~~~~~~~~~~~ + +You can run a shell command with ``!``: + +:: + + pappy> ls + ID Verb Host Path S-Code Req Len Rsp Len Time Mngl + 5 GET vitaly.sexy /netscape.gif 304 Not Modified 0 0 0.08 -- + 4 GET vitaly.sexy /esr1.jpg 304 Not Modified 0 0 0.07 -- + 3 GET vitaly.sexy /construction.gif 304 Not Modified 0 0 0.07 -- + 2 GET vitaly.sexy /vitaly2.jpg 0 N/A -- -- + 1 GET vitaly.sexy / 304 Not Modified 0 0 0.07 -- + pappy> !ls + cmdhistory config.json data.db + pappy> + +Running Python Code +~~~~~~~~~~~~~~~~~~~ + +You can use the ``py`` command to either run python code or to drop down +to a Python shell. + +:: + + pappy> py print ':D '*10 + :D :D :D :D :D :D :D :D :D :D + pappy> py + Python 2.7.6 (default, Jun 22 2015, 17:58:13) + [GCC 4.8.2] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + (ProxyCmd) + + py : Executes a Python command. + py: Enters interactive Python mode. + End with ``Ctrl-D`` (Unix) / ``Ctrl-Z`` (Windows), ``quit()``, '`exit()``. + Non-python commands can be issued with ``cmd("your command")``. + Run python code from external files with ``run("filename.py")`` + + >>> from pappyproxy import config + >>> config.CONFIG_DICT + {u'data_file': u'./data.db', u'history_size': 1000, u'cert_dir': u'{DATADIR}/certs', u'proxy_listeners': [{u'interface': u'127.0.0.1', u'port': 8000}]} + >>> exit() + pappy> + +Redirect Output To File +~~~~~~~~~~~~~~~~~~~~~~~ + +You can use ``>`` to direct output to a file. However, a number of +commands use colored output. If you just redirect these to a file, there +will be additional bytes which represent the ANSI color codes. To get +around this, use the ``nocolor`` command to remove the color from the +command output. + +:: + + pappy> ls > ls.txt + pappy> !xxd -c 32 -g 4 ls.txt + 0000000: 1b5b316d 1b5b346d 49442020 56657262 2020486f 73742020 20202020 20202050 .[1m.[4mID Verb Host P + 0000020: 61746820 20202020 20202020 20202020 2020532d 436f6465 20202020 20202020 ath S-Code + 0000040: 20202020 52657120 4c656e20 20527370 204c656e 20205469 6d652020 20204d6e Req Len Rsp Len Time Mn + 0000060: 676c2020 1b5b306d 0a352020 201b5b33 366d4745 541b5b30 6d202020 1b5b3931 gl .[0m.5 .[36mGET.[0m .[91 + 0000080: 6d766974 616c792e 73657879 1b5b306d 20201b5b 33366d1b 5b306d2f 1b5b3334 mvitaly.sexy.[0m .[36m.[0m/.[34 + 00000a0: 6d6e6574 73636170 652e6769 661b5b30 6d202020 2020201b 5b33356d 33303420 mnetscape.gif.[0m .[35m304 + 00000c0: 4e6f7420 4d6f6469 66696564 1b5b306d 20203020 20202020 20202030 20202020 Not Modified.[0m 0 0 + 00000e0: 20202020 302e3038 20202020 2d2d2020 20200a34 2020201b 5b33366d 4745541b 0.08 -- .4 .[36mGET. + 0000100: 5b306d20 20201b5b 39316d76 6974616c 792e7365 78791b5b 306d2020 1b5b3336 [0m .[91mvitaly.sexy.[0m .[36 + 0000120: 6d1b5b30 6d2f1b5b 33346d65 7372312e 6a70671b 5b306d20 20202020 20202020 m.[0m/.[34mesr1.jpg.[0m + 0000140: 201b5b33 356d3330 34204e6f 74204d6f 64696669 65641b5b 306d2020 30202020 .[35m304 Not Modified.[0m 0 + 0000160: 20202020 20302020 20202020 2020302e 30372020 20202d2d 20202020 0a332020 0 0.07 -- .3 + 0000180: 201b5b33 366d4745 541b5b30 6d202020 1b5b3931 6d766974 616c792e 73657879 .[36mGET.[0m .[91mvitaly.sexy + 00001a0: 1b5b306d 20201b5b 33366d1b 5b306d2f 1b5b3334 6d636f6e 73747275 6374696f .[0m .[36m.[0m/.[34mconstructio + 00001c0: 6e2e6769 661b5b30 6d20201b 5b33356d 33303420 4e6f7420 4d6f6469 66696564 n.gif.[0m .[35m304 Not Modified + 00001e0: 1b5b306d 20203020 20202020 20202030 20202020 20202020 302e3037 20202020 .[0m 0 0 0.07 + 0000200: 2d2d2020 20200a32 2020201b 5b33366d 4745541b 5b306d20 20201b5b 39316d76 -- .2 .[36mGET.[0m .[91mv + 0000220: 6974616c 792e7365 78791b5b 306d2020 1b5b3336 6d1b5b30 6d2f1b5b 33346d76 italy.sexy.[0m .[36m.[0m/.[34mv + 0000240: 6974616c 79322e6a 70671b5b 306d2020 20202020 201b5b33 366d3230 30204f4b italy2.jpg.[0m .[36m200 OK + 0000260: 1b5b306d 20202020 20202020 20202020 30202020 20202020 20323033 34303033 .[0m 0 2034003 + 0000280: 20203135 352e3131 20202d2d 20202020 0a312020 201b5b33 366d4745 541b5b30 155.11 -- .1 .[36mGET.[0 + 00002a0: 6d202020 1b5b3931 6d766974 616c792e 73657879 1b5b306d 20201b5b 33366d1b m .[91mvitaly.sexy.[0m .[36m. + 00002c0: 5b306d2f 1b5b3334 6d1b5b30 6d202020 20202020 20202020 20202020 2020201b [0m/.[34m.[0m . + 00002e0: 5b33356d 33303420 4e6f7420 4d6f6469 66696564 1b5b306d 20203020 20202020 [35m304 Not Modified.[0m 0 + 0000300: 20202030 20202020 20202020 302e3037 20202020 2d2d2020 20200a 0 0.07 -- . + pappy> nocolor ls > ls2.txt + pappy> !xxd -c 32 -g 4 ls2.txt + 0000000: 49442020 56657262 2020486f 73742020 20202020 20202050 61746820 20202020 ID Verb Host Path + 0000020: 20202020 20202020 2020532d 436f6465 20202020 20202020 20202020 52657120 S-Code Req + 0000040: 4c656e20 20527370 204c656e 20205469 6d652020 20204d6e 676c2020 0a352020 Len Rsp Len Time Mngl .5 + 0000060: 20474554 20202076 6974616c 792e7365 78792020 2f6e6574 73636170 652e6769 GET vitaly.sexy /netscape.gi + 0000080: 66202020 20202033 3034204e 6f74204d 6f646966 69656420 20302020 20202020 f 304 Not Modified 0 + 00000a0: 20203020 20202020 20202030 2e303820 2020202d 2d202020 200a3420 20204745 0 0.08 -- .4 GE + 00000c0: 54202020 76697461 6c792e73 65787920 202f6573 72312e6a 70672020 20202020 T vitaly.sexy /esr1.jpg + 00000e0: 20202020 33303420 4e6f7420 4d6f6469 66696564 20203020 20202020 20202030 304 Not Modified 0 0 + 0000100: 20202020 20202020 302e3037 20202020 2d2d2020 20200a33 20202047 45542020 0.07 -- .3 GET + 0000120: 20766974 616c792e 73657879 20202f63 6f6e7374 72756374 696f6e2e 67696620 vitaly.sexy /construction.gif + 0000140: 20333034 204e6f74 204d6f64 69666965 64202030 20202020 20202020 30202020 304 Not Modified 0 0 + 0000160: 20202020 20302e30 37202020 202d2d20 2020200a 32202020 47455420 20207669 0.07 -- .2 GET vi + 0000180: 74616c79 2e736578 7920202f 76697461 6c79322e 6a706720 20202020 20203230 taly.sexy /vitaly2.jpg 20 + 00001a0: 30204f4b 20202020 20202020 20202020 30202020 20202020 20323033 34303033 0 OK 0 2034003 + 00001c0: 20203135 352e3131 20202d2d 20202020 0a312020 20474554 20202076 6974616c 155.11 -- .1 GET vital + 00001e0: 792e7365 78792020 2f202020 20202020 20202020 20202020 20202033 3034204e y.sexy / 304 N + 0000200: 6f74204d 6f646966 69656420 20302020 20202020 20203020 20202020 20202030 ot Modified 0 0 0 + 0000220: 2e303720 2020202d 2d202020 200a0a .07 -- .. + pappy> + +If you want to write the contents of a request or response to a file, +don't use ``nocolor`` with ``vfq`` or ``vfs``. Use just the ``vbq`` or +``vbs`` commands. + ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Command | Description | ++===============+==============================================================================================================================================================================+ +| ``nocolor`` | Run a command and print its output without ASCII escape codes. Intended for use when redirecting output to a file. Should only be used with text and not with binary data. | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + Generating Pappy's CA Cert -------------------------- @@ -186,23 +322,45 @@ Browsing Recorded Requests/Responses The following commands can be used to view requests and responses -+--------------------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Command | Aliases | Description | -+====================+================================+====================================================================================================================================================================================================================================================================================================================================================================================================================================+ -| ``ls [a|``] | list, ls | List requests that are in the current context (see Context section). Has information like the host, target path, and status code. With no arguments, it will print the 25 most recent requests in the current context. If you pass 'a' or 'all' as an argument, it will print all the requests in the current context. If you pass a number "n" as an argument, it will print the n most recent requests in the current context. | -+--------------------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``sm`` | sm, site\_map | Print a tree showing the site map. It will display all requests in the current context that did not have a 404 response. | -+--------------------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``viq `` | view\_request\_info, viq | View additional information about requests. Includes the target port, if SSL was used, applied tags, and other information. | -+--------------------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``vfq `` | view\_full\_request, vfq | [V]iew [F]ull Re[Q]uest, prints the full request including headers and data. | -+--------------------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``vhq `` | view\_request\_headers, vhq | [V]iew [H]eaders of a Re[Q]uest. Prints just the headers of a request. | -+--------------------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``vfs `` | view\_full\_response, vfs | [V]iew [F]ull Re[S]ponse, prints the full response associated with a request including headers and data. | -+--------------------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``vhs `` | view\_response\_headers, vhs | [V]iew [H]eaders of a Re[S]ponse. Prints just the headers of a response associated with a request. | -+--------------------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Command | Aliases | Description | ++==============================================================================================================================================================================================================================================================================================================+==================================+====================================================================================================================================================================================================================================================================================================================================================================================================================================+ +| ``ls [a|``] | list, ls | List requests that are in the current context (see Context section). Has information like the host, target path, and status code. With no arguments, it will print the 25 most recent requests in the current context. If you pass 'a' or 'all' as an argument, it will print all the requests in the current context. If you pass a number "n" as an argument, it will print the n most recent requests in the current context. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``sm`` [p] | sm, site\_map | Print a tree showing the site map. It will display all requests in the current context that did not have a 404 response. This has to go through all of the requests in the current context so it may be slow. If the ``p`` option is given, it will print the paths as paths rather than as a tree. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``viq `` | view\_request\_info, viq | View additional information about requests. Includes the target port, if SSL was used, applied tags, and other information. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``vfq `` | view\_full\_request, vfq, kjq | [V]iew [F]ull Re[Q]uest, prints the full request including headers and data. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``vbq `` | view\_request\_bytes, vbq | [V]iew [B]ytes of Re[Q]uest, prints the full request including headers and data without coloring or additional newlines. Use this if you want to write a request to a file. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``ppq `` | pretty\_print\_request, ppq | Pretty print a request with a specific format. See the table below for a list of formats. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``vhq `` | view\_request\_headers, vhq | [V]iew [H]eaders of a Re[Q]uest. Prints just the headers of a request. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``vfs `` | view\_full\_response, vfs, kjs | [V]iew [F]ull Re[S]ponse, prints the full response associated with a request including headers and data. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``vhs `` | view\_response\_headers, vhs | [V]iew [H]eaders of a Re[S]ponse. Prints just the headers of a response associated with a request. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``vbs `` | view\_response\_bytes, vbs | [V]iew [B]ytes of Re[S]ponse, prints the full response including headers and data without coloring or additional newlines. Use this if you want to write a response to a file. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``pps `` | pretty\_print\_response, pps | Pretty print a response with a specific format. See the table below for a list of formats. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``pprm `` | print\_params, pprm | Print a summary of the parameters submitted with the request. It will include URL params, POST params, and/or cookies | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``pri [ct] [key(s)] | param_info, pri | Print a summary of the parameters and values submitted by in-context requests. You can pass in keys to limit which values will be shown. If you also provide``\ ct\ ``as the first argument, it will include any keys that are passed as arguments. | |``\ watch\` | watch | Print requests and responses in real time as they pass through the proxy. | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Available formats for ``ppq`` and ``pps`` commands: + ++------------+------------------------------------------------------------+ +| Format | Description | ++============+============================================================+ +| ``form`` | Print POST data submitted from a form (normal post data) | ++------------+------------------------------------------------------------+ +| ``json`` | Print as JSON | ++------------+------------------------------------------------------------+ The table shown by ``ls`` will have the following columns: @@ -399,12 +557,20 @@ List of fields +--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ | body | body, data, bd, dt | The body (data section) of either the request or the response | String | +--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ +| reqbody | qbody, qdata, qbd, qdt | The body (data section) of th request | String | ++--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ +| rspbody | sbody, sdata, sbd, sdt | The body (data section) of th response | String | ++--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ | verb | verb, vb | The HTTP verb of the request (ie GET, POST) | String | +--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ | param | param, pm | Either the get or post parameters | Key/Value | +--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ | header | header, hd | An HTTP header (ie User-Agent, Basic-Authorization) in the request or response | Key/Value | +--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ +| reqheader | reqheader, qhd | An HTTP header in the request | Key/Value | ++--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ +| rspheader | rspheader, shd | An HTTP header in the response | Key/Value | ++--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ | rawheaders | rawheaders, rh | The entire header section (as one string) of either the head or the response | String | +--------------+--------------------------------+----------------------------------------------------------------------------------+-------------+ | sentcookie | sentcookie, sck | A cookie sent in a request | Key/Value | @@ -456,6 +622,21 @@ can still negate these. +-----------+------------------+---------------------------------------------------------------------------------------------------------+ | after | after, af | Filters out any request that is not before the given request. Filters out any request without a time. | +-----------+------------------+---------------------------------------------------------------------------------------------------------+ +| inv | inf | Inverts a filter string. Anything that matches the filter string will not pass the filter. | ++-----------+------------------+---------------------------------------------------------------------------------------------------------+ + +Examples: + +:: + + Only show requests before request 1234 + f b4 1234 + + Only show requests after request 1234 + f af 1234 + + Show requests without a csrf parameter + f inv param ct csrf Scope ----- @@ -505,6 +686,81 @@ The ``fbi`` command also supports tab completion. | ``fbi `` | ``builtin_filter``, ``fbi`` | Apply a built-in filter to the current context | +--------------------+-------------------------------+--------------------------------------------------+ +Decoding Strings +---------------- + +These features try to fill a similar role to Burp's decoder. Each +command will automatically copy the results to the clipboard. In +addition, if no string is given, the commands will encode/decode +whatever is already in the clipboard. Here is an example of how to +base64 encode/decode a string. + +:: + + pappy> b64e "Hello World!" + SGVsbG8gV29ybGQh + pappy> b64d + Hello World! + pappy> + +And if the result contains non-printable characters, a hexdump will be +produced instead + +:: + + pappy> b64d ImALittleTeapot= + 0000 22 60 0b 8a db 65 79 37 9a a6 8b "`...ey7... + + pappy> + +The following commands can be used to encode/decode strings: + ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Command | Aliases | Description | ++===========================+=====================================+=====================================================================================================================================================================+ +| ``base64_decode`` | ``base64_decode``, ``b64d`` | Base64 decode a string | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``base64_encode`` | ``base64_encode``, ``b64e`` | Base64 encode a string | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``asciihex_decode`` | ``asciihex_decode``, ``ahd`` | Decode an ASCII hex string | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``asciihex_encode`` | ``asciihex_encode``, ``ahe`` | Encode an ASCII hex string | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``html_decode`` | ``html_decode``, ``htmld`` | Decode an html encoded string | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``html_encode`` | ``html_encode``, ``htmle`` | Encode a string to html encode all of the characters | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``url_decode`` | ``url_decode``, ``urld`` | Url decode a string | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``url_encode`` | ``url_encode``, ``urle`` | Url encode a string | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``gzip_decode`` | ``gzip_decode``, ``gzd`` | Gzip decompress a string. Probably won't work too well since there's not a great way to get binary data passed in as an argument. I'm working on this. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``gzip_encode`` | ``gzip_encode``, ``gze`` | Gzip compress a string. Result doesn't get copied to the clipboard. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``base64_decode_raw`` | ``base64_decode_raw``, ``b64dr`` | Same as ``base64_decode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``base64_encode_raw`` | ``base64_encode_raw``, ``b64er`` | Same as ``base64_encode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``asciihex_decode_raw`` | ``asciihex_decode_raw``, ``ahdr`` | Same as ``asciihex_decode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``asciihex_encode_raw`` | ``asciihex_encode_raw``, ``aher`` | Same as ``asciihex_encode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``html_decode_raw`` | ``html_decode_raw``, ``htmldr`` | Same as ``html_decode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``html_encode_raw`` | ``html_encode_raw``, ``htmler`` | Same as ``html_encode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``url_decode_raw`` | ``url_decode_raw``, ``urldr`` | Same as ``url_decode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``url_encode_raw`` | ``url_encode_raw``, ``urler`` | Same as ``url_encode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``gzip_decode_raw`` | ``gzip_decode_raw``, ``gzdr`` | Same as ``gzip_decode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``gzip_encode_raw`` | ``gzip_encode_raw``, ``gzer`` | Same as ``gzip_encode`` but will not print a hexdump if it contains non-printable characters. It is suggested you use ``>`` to redirect the output to a file. | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``unixtime_decode`` | ``unixtime_decode``, ``uxtd`` | Take in a unix timestamp and print a human readable timestamp | ++---------------------------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + Interceptor ----------- @@ -604,7 +860,7 @@ a simple macro would be: :: - --- macro_print.py + ### macro_print.py MACRO_NAME = 'Print Macro' @@ -717,15 +973,15 @@ Request Objects The main method of interacting with the proxy is through ``Request`` objects. You can submit a request with ``req.sumbit()`` and save it to the data file with ``req.save()``. The objects also have attributes -which can be used to modify the request in a high-level way. -Unfortunately, I haven't gotten around to writing full docs on the API -and it's still changing every once in a while so I apologize if I pull -the carpet out from underneath you. +which can be used to modify the request in a high-level way. You can see +the `full +documentation `__ +for more details on using these objects. Dict-like objects are represented with a custom class called a -``RepeatableDict``. I haven't gotten around to writing docs on it yet, -so just interact with it like a dict and don't be surprised if it's -missing some methods you would expect a dict to have. +``RepeatableDict``. Again, look at the docs for details. For the most +part, you can interact with it like a normal dictionary, but don't be +surprised if it's missing some methods you would expect. Here is a quick list of attributes that you can use with ``Request`` objects: @@ -1024,6 +1280,8 @@ error checking. +----------------------------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``export `` | ``export`` | Writes either the full request or response to a file in the current directory. | +----------------------------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``merge `` | ``merge`` | Add all the requests from another datafile to the current datafile | ++----------------------------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ Response streaming ~~~~~~~~~~~~~~~~~~ @@ -1033,9 +1291,241 @@ data to the browser as it gets it. However, if you're trying to mangle messages/responses, Pappy will need to download the entire message first. +Plugins +------- + +Note that this section is a very quick overview of plugins. For a full +description of how to write them, please see `the official +docs `__. + +It is also possible to write plugins which are reusable across projects. +Plugins are simply Python scripts located in ``~/.pappy/plugins``. +Plugins are able to create new console commands and maintain state +throughout a Pappy session. They can access the same API as macros, but +the plugin system is designed to allow you to create general purpose +commands as compared to macros which are meant to be project-specific +scripts. Still, it may not be a bad idea to try building a macro to do +something in a quick and dirty way before writing a plugin since plugins +are more complicated to write. + +A simple hello world plugin could be something like: + +:: + + ## hello.py + import shlex + + def hello_world(line): + if line: + args = shlex.split(line) + print 'Hello, %s!' % (', '.join(args)) + else: + print "Hello, world!" + + ############### + ## Plugin hooks + + def load_cmds(cmd): + cmd.set_cmds({ + 'hello': (hello_world, None), + }) + cmd.add_aliases([ + ('hello', 'hlo'), + ('hello', 'ho'), + ]) + +You can also create commands which support autocomplete: + +:: + + import shlex + + _AUTOCOMPLETE_NAMES = ['alice', 'allie', 'sarah', 'mallory', 'slagathor'] + + def hello_world(line): + if line: + args = shlex.split(line) + print 'Hello, %s!' % (', '.join(args)) + else: + print "Hello, world!" + + def complete_hello_world(text, line, begidx, endidx): + return [n for n in _AUTOCOMPLETE_NAMES if n.startswith(text)] + + ############### + ## Plugin hooks + + def load_cmds(cmd): + cmd.set_cmds({ + 'hello': (hello_world, complete_hello_world), + }) + cmd.add_aliases([ + ('hello', 'hlo'), + ]) + +Then when you run Pappy you can use the ``hello`` command: + +:: + + $ pappy -l + Temporary datafile is /tmp/tmpBOXyJ3 + Proxy is listening on port 8000 + pappy> ho + Hello, world! + pappy> ho foo bar baz + Hello, foo, bar, baz! + pappy> ho foo bar "baz lihtyur" + Hello, foo, bar, baz lihtyur! + pappy> + +Should I Write a Plugin or a Macro? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A lot of the time, you can get away with writing a macro. However, you +may consider writing a plugin if: + +- You find yourself copying one macro to multiple projects +- You want to write a general tool that can be applied to any website +- You need to maintain state during the Pappy session + +My guess is that if you need one quick thing for a project, you're +better off writing a macro first and seeing if you end up using it in +future projects. Then if you find yourself needing it a lot, write a +plugin for it. You may also consider keeping a ``mine.py`` plugin where +you can write out commands that you use regularly but may not be worth +creating a dedicated plugin for. + +Global Settings +--------------- + +There are some settings that apply to Pappy as a whole and are stored in +``~/.pappy/global_config.json``. These settings are generally for tuning +performance or modifying behavior on a system-wide level. No information +about projects is put in here since it is world readable. You can +technically add settings in here for plugins that you write, but if it's +at all possible, please keep settings in the normal project config. + +Settings included in ``~/.pappy/global_config.json``: + ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Setting | Description | ++===============+===============================================================================================================================================================================+ +| cache\_size | The number of requests from history that will be included in memory at any given time. Set to -1 to keep everything in memory. See the request cache section for more info. | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Using a SOCKS Server +-------------------- + +Pappy allows you to use an upstream SOCKS server. You can do this by +adding a ``socks_proxy`` value to config.json. You can use the following +for anonymous access to the proxy: + +:: + + "socks_proxy": {"host":"socks.proxy.host", "port":5555} + +To use credentials you add a ``username`` and ``password`` value to the +dictionary: + +:: + + "socks_proxy": {"host":"socks.proxy.host", "port":5555, "username": "mario", "password":"ilovemushrooms"} + +Anything that passes through any of the active listeners will use the +proxy. + +Transparent Host Redirection +---------------------------- + +Sometimes you get a frustrating thick client that doesn’t let you mess +with proxy settings to get it to go through a proxy. However, if you can +redirect where it sends its traffic to localhost, you can get Pappy to +take that traffic and redirect it to go where it should. + +It takes root permissions to listen on low numbered ports. As a result, +we’ll need to do some root stuff to listen on ports 80 and 443 and get +the data to Pappy. There are two ways to get the traffic to Pappy. The +first is to set up port forwarding as root to send traffic from +localhost:80 to localhost:8080 and localhost:443 to localhost:8443 +(since we can listen on 8080 and 8443 without root). Or you can YOLO, +run Pappy as root and just have it listen on 80 and 443. + +According to Google you can use the following command to forward port 80 +on localhost to 8080 on Linux: + +:: + + iptables -t nat -A PREROUTING -i ppp0 -p tcp --dport 80 -j REDIRECT --to-ports 8080 + +Then to route 443 to 8443: + +:: + + iptables -t nat -A PREROUTING -i ppp0 -p tcp --dport 443 -j REDIRECT --to-ports 8443 + +Of course, both of these need to be run as root. + +Then on mac it’s + +:: + + echo " + rdr pass inet proto tcp from any to any port 80 -> 127.0.0.1 port 8080 + rdr pass inet proto tcp from any to any port 443 -> 127.0.0.1 port 8443 + " | sudo pfctl -ef - + Then to turn it off on mac it’s + sudo pfctl -F all -f /etc/pf.conf + +Then modify the listener settings in the project’s config.json to be: + +:: + + "proxy_listeners": [ + {"port": 8080, "interface": "127.0.0.1", "forward_host": "www.example.faketld"}, + {"port": 8443, "interface": "127.0.0.1", "forward_host_ssl": "www.example.faketld"}, + ] + +This configuration will cause Pappy to open a port on 8080 that will +accept connections normally and a port on 8443 which will accept SSL +connections. The forward\_host setting tells Pappy to redirect any +requests sent to the port to the given host. It will also update the +request’s host header. forward\_host\_ssl does the same thing, but it +listens for SSL connections and forces the connection to use SSL. + +Or if you’re going to YOLO it do the same thing then listen on port +80/443 directly. I do not suggest you do this. + +:: + + "proxy_listeners": [ + {"port": 80, "interface": "127.0.0.1", "forward_host": "www.example.faketld"}, + {"port": 443, "interface": "127.0.0.1", "forward_host_ssl": "www.example.faketld"}, + ] + +Pappy will automatically use this host to make the connection and +forward the request to the new server. + FAQ --- +I still like Burp, but Pappy looks interesting, can I use both? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Yes! If you don't want to go completely over to Pappy yet, you can +configure Burp to use Pappy as an upstream proxy server. That way, +traffic will go through both Burp and Pappy and you can use whichever +you want to do your testing. + +How to have Burp forward traffic through Pappy: + +1. Open Burp +2. Go to ``Options -> Connections -> Upstream Proxy Servers`` +3. Click ``Add`` +4. Leave ``Destination Host`` blank, but put ``127.0.0.1`` in + ``Proxy Host`` and ``8000`` into ``Port`` (assuming you're using the + default listener) +5. Configure your browser to use Burp as a proxy + Why does my request have an id of ``--``?!?! ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1044,17 +1534,98 @@ saved to disk. In between the time when a request is decoded and when it's saved to disk, it will have an ID of ``--``. So just wait a little bit and it will get an ID you can use. +Boring, Technical Stuff +----------------------- + +I do some stuff to try and keep speed and memory usage to reasonable +levels. Unfortunately, things might seem slow in some areas. This is +where I try and explain why those exist. Honestly, you probably don't +care about this, but I'd rather have it written down and have nobody +read it than just leave people in the dark. + +Request Cache / Memory usage +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For performance reasons, Pappy by default will not store every request +in memory. The cache will store a certain number of the most recently +accessed requests in memory. This means that if you go through all of +history, it could be slow (for example running ``ls a`` or ``sm``). If +you have enough RAM to keep everything in memory, you can set the +request cache size to -1 to just keep everything in memory. However, +even if the cache size is unlimited, it still won't load a request into +memory untill you access it. So if you want to load everything in +memory, run ``ls a``. + +By default, Pappy will cache 2000 requests. This is kind of heavy, but +it's assumed you're doing testing on a reasonably specced laptop. +Personally, I live on the edge and use -1 until I run into memory +issues. + Changelog --------- The boring part of the readme -- 0.1.2 -- Refactor almost every part of proxy -- Basic framework for plugins -- Bugfixes probably -- Create changelog +- 0.2.7 + + - boring unit tests + - should make future releases more stable I guess + - Support for upstream SOCKS servers + - ``print_params`` command + - ``inv`` filter + - ``param_info`` command + - Filters by request/response only headers/body + - Transparent host redirection + - Some easier to type aliases for common commands + +- 0.2.6 + + - Fix pip being dumb + - ``watch`` command to watch requests/responses in real time + - Added ``pp[qs] form `` to print POST data + - Bugfixes + +- 0.2.5 + + - Requests sent with repeater now are given ``repeater`` tag + - Add ppq and pps commands + - Look at the pretty prompt + - Bugfixes + +- 0.2.4 + + - Add command history saving between sessions + - Add html encoder/decoder + - All the bugs were fixed so I added some more for 0.2.5 + +- 0.2.3 + + - Decoder functions + - Add ``merge`` command + - Bugfixes + +- 0.2.2 + + - COLORS + - Performance improvements + - Bugfixes (duh) + +- 0.2.1 + + - Improve memory usage + - Tweaked plugin API + +- 0.2.0 + + - Lots of refactoring + - Plugins + - Bugfixes probably + - Change prompt to make Pappy look more professional (but it will + always be pappy time in your heart, I promise) + - Create changelog + - Add response streaming if no intercepting macros are active + - 0.1.1 -- Start using sane versioning system -- No idea what I added + - Start using sane versioning system + - Did proxy things diff --git a/pappyproxy/Makefile b/pappyproxy/Makefile index 1545d98..a65c7ab 100644 --- a/pappyproxy/Makefile +++ b/pappyproxy/Makefile @@ -7,3 +7,6 @@ test: test-verbose: py.test -v -rw --twisted --cov-config .coveragerc --cov-report term-missing --cov=. tests/ + +test-macros: + py.test -v -rw --twisted tests/test_macros.py diff --git a/pappyproxy/colors.py b/pappyproxy/colors.py index b461f20..c89fe55 100644 --- a/pappyproxy/colors.py +++ b/pappyproxy/colors.py @@ -63,6 +63,8 @@ class Styles: KV_KEY = Colors.GREEN KV_VAL = Colors.ENDC + UNPRINTABLE_DATA = Colors.CYAN + def verb_color(verb): if verb and verb == 'GET': diff --git a/pappyproxy/comm.py b/pappyproxy/comm.py index 2c6b46d..60d1c60 100644 --- a/pappyproxy/comm.py +++ b/pappyproxy/comm.py @@ -1,3 +1,4 @@ +import sys import base64 import json @@ -20,6 +21,7 @@ def set_comm_port(port): comm_port = port class CommServer(LineReceiver): + MAX_LENGTH=sys.maxint def __init__(self): self.delimiter = '\n' @@ -32,6 +34,7 @@ class CommServer(LineReceiver): def lineReceived(self, line): from .http import Request, Response + line = line.strip() if line == '': return @@ -98,7 +101,7 @@ class CommServer(LineReceiver): @defer.inlineCallbacks def action_submit_request(self, data): message = base64.b64decode(data['full_message']) - req = yield Request.submit_new(data['host'], data['port'], data['is_ssl'], message) + req = yield Request.submit_new(data['host'].encode('utf-8'), data['port'], data['is_ssl'], message) if 'tags' in data: req.tags = set(data['tags']) yield req.async_deep_save() diff --git a/pappyproxy/config.py b/pappyproxy/config.py index e8a39de..986db3c 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -46,6 +46,20 @@ The configuration settings for the proxy. :Default: ``[(8000, '127.0.0.1')]`` +.. data: SOCKS_PROXY + + Details for a SOCKS proxy. It is a dict with the following key/values:: + + host: The SOCKS proxy host + port: The proxy port + username: Username (optional) + password: Password (optional) + + If null, no proxy will be used. + + :Default: ``null`` + + .. data: PLUGIN_DIRS List of directories that plugins are loaded from. Not modifiable. @@ -87,6 +101,7 @@ DEBUG_TO_FILE = False DEBUG_VERBOSITY = 0 LISTENERS = [(8000, '127.0.0.1')] +SOCKS_PROXY = None SSL_CA_FILE = 'certificate.crt' SSL_PKEY_FILE = 'private.key' @@ -112,6 +127,7 @@ def load_settings(proj_config): global DEBUG_TO_FILE global DEBUG_VERBOSITY global LISTENERS + global SOCKS_PROXY global PAPPY_DIR global DATA_DIR global SSL_CA_FILE @@ -141,7 +157,30 @@ def load_settings(proj_config): if "proxy_listeners" in proj_config: LISTENERS = [] for l in proj_config["proxy_listeners"]: - LISTENERS.append((l['port'], l['interface'])) + ll = {} + if 'forward_host_ssl' in l: + l['forward_host_ssl'] = l['forward_host_ssl'].encode('utf-8') + if 'forward_host' in l: + l['forward_host'] = l['forward_host'].encode('utf-8') + LISTENERS.append(l) + + # SOCKS proxy settings + if "socks_proxy" in proj_config: + SOCKS_PROXY = None + if proj_config['socks_proxy'] is not None: + conf = proj_config['socks_proxy'] + if 'host' in conf and 'port' in conf: + SOCKS_PROXY = {} + SOCKS_PROXY['host'] = conf['host'].encode('utf-8') + SOCKS_PROXY['port'] = conf['port'] + if 'username' in conf: + if 'password' in conf: + SOCKS_PROXY['username'] = conf['username'].encode('utf-8') + SOCKS_PROXY['password'] = conf['password'].encode('utf-8') + else: + print 'SOCKS proxy has a username but no password. Ignoring creds.' + else: + print 'Host is missing host/port.' # History saving settings if "history_size" in proj_config: diff --git a/pappyproxy/context.py b/pappyproxy/context.py index 87be087..7087cc8 100644 --- a/pappyproxy/context.py +++ b/pappyproxy/context.py @@ -122,15 +122,19 @@ class Filter(object): @staticmethod @defer.inlineCallbacks - def from_filter_string(filter_string): + def from_filter_string(filter_string=None, parsed_args=None): """ from_filter_string(filter_string) - Create a filter from a filter string. + Create a filter from a filter string. If passed a list of arguments, they + will be used instead of parsing the string. :rtype: Deferred that returns a :class:`pappyproxy.context.Filter` """ - args = shlex.split(filter_string) + if parsed_args is not None: + args = parsed_args + else: + args = shlex.split(filter_string) if len(args) == 0: raise PappyException('Field is required') field = args[0] @@ -145,12 +149,20 @@ class Filter(object): new_filter = gen_filter_by_path(field_args) elif field in ("body", "bd", "data", "dt"): new_filter = gen_filter_by_body(field_args) + elif field in ("reqbody", "qbd", "reqdata", "qdt"): + new_filter = gen_filter_by_req_body(field_args) + elif field in ("rspbody", "sbd", "qspdata", "sdt"): + new_filter = gen_filter_by_rsp_body(field_args) elif field in ("verb", "vb"): new_filter = gen_filter_by_verb(field_args) elif field in ("param", "pm"): new_filter = gen_filter_by_params(field_args) elif field in ("header", "hd"): new_filter = gen_filter_by_headers(field_args) + elif field in ("reqheader", "qhd"): + new_filter = gen_filter_by_request_headers(field_args) + elif field in ("rspheader", "shd"): + new_filter = gen_filter_by_response_headers(field_args) elif field in ("rawheaders", "rh"): new_filter = gen_filter_by_raw_headers(field_args) elif field in ("sentcookie", "sck"): @@ -169,6 +181,8 @@ class Filter(object): new_filter = yield gen_filter_by_before(field_args) elif field in ("after", "af"): new_filter = yield gen_filter_by_after(field_args) + elif field in ("inv",): + new_filter = yield gen_filter_by_inverse(field_args) else: raise FilterParseError("%s is not a valid field" % field) @@ -181,33 +195,53 @@ class Filter(object): defer.returnValue(new_filter) def cmp_is(a, b): + if a is None or b is None: + return False return str(a) == str(b) def cmp_contains(a, b): + if a is None or b is None: + return False return (b.lower() in a.lower()) def cmp_exists(a, b=None): + if a is None or b is None: + return False return (a is not None and a != []) def cmp_len_eq(a, b): + if a is None or b is None: + return False return (len(a) == int(b)) def cmp_len_gt(a, b): + if a is None or b is None: + return False return (len(a) > int(b)) def cmp_len_lt(a, b): + if a is None or b is None: + return False return (len(a) < int(b)) def cmp_eq(a, b): + if a is None or b is None: + return False return (int(a) == int(b)) def cmp_gt(a, b): + if a is None or b is None: + return False return (int(a) > int(b)) def cmp_lt(a, b): + if a is None or b is None: + return False return (int(a) < int(b)) def cmp_containsr(a, b): + if a is None or b is None: + return False try: if re.search(b, a): return True @@ -328,38 +362,50 @@ def compval_from_args_repdict(args): return retfunc def gen_filter_by_all(args): - compval_from_args(args) # try and throw an error + compval = compval_from_args(args) def f(req): - compval = compval_from_args(args) if args[0][0] == 'n': - return compval(req.full_message) and (not req.response or compval(req.response.full_message)) + return compval(req.full_message) and ((not req.response) or compval(req.response.full_message)) else: return compval(req.full_message) or (req.response and compval(req.response.full_message)) return f def gen_filter_by_host(args): - compval_from_args(args) # try and throw an error + compval = compval_from_args(args) def f(req): - compval = compval_from_args(args) return compval(req.host) return f def gen_filter_by_body(args): - compval_from_args(args) # try and throw an error + compval = compval_from_args(args) def f(req): - compval = compval_from_args(args) if args[0][0] == 'n': - return compval(req.body) and (not req.response or compval(req.response.body)) + return compval(req.body) and ((not req.response) or compval(req.response.body)) else: return compval(req.body) or (req.response and compval(req.response.body)) return f +def gen_filter_by_req_body(args): + compval = compval_from_args(args) + def f(req): + return compval(req.body) + return f + +def gen_filter_by_rsp_body(args): + compval = compval_from_args(args) + def f(req): + if args[0][0] == 'n': + return (not req.response) or compval(req.response.body) + else: + return req.response and compval(req.response.body) + return f + def gen_filter_by_raw_headers(args): - compval_from_args(args) # try and throw an error + compval = compval_from_args(args) def f(req): - compval = compval_from_args(args) if args[0][0] == 'n': - return compval(req.headers_section) and (not req.response or compval(req.response.headers_section)) + # compval already negates comparison + return compval(req.headers_section) and ((not req.response) or compval(req.response.headers_section)) else: return compval(req.headers_section) or (req.response and compval(req.response.headers_section)) return f @@ -374,30 +420,26 @@ def gen_filter_by_response_code(args): return f def gen_filter_by_path(args): - compval_from_args(args) + compval = compval_from_args(args) def f(req): - compval = compval_from_args(args) return compval(req.path) return f def gen_filter_by_responsetime(args): - compval_from_args(args) + compval = compval_from_args(args) def f(req): - compval = compval_from_args(args) return compval(req.rsptime) return f def gen_filter_by_verb(args): - compval_from_args(args) + compval = compval_from_args(args) def f(req): - compval = compval_from_args(args) return compval(req.verb) return f def gen_filter_by_tag(args): - compval_from_args(args) + compval = compval_from_args(args) def f(req): - compval = compval_from_args(args) for tag in req.tags: if compval(tag): return True @@ -418,7 +460,7 @@ def gen_filter_by_saved(args): def gen_filter_by_before(args): if len(args) != 1: raise PappyException('Invalid number of arguments') - r = yield http.Request.load_request(args[0]) + r = yield Request.load_request(args[0]) def f(req): if req.time_start is None: return False @@ -431,7 +473,7 @@ def gen_filter_by_before(args): def gen_filter_by_after(reqid, negate=False): if len(args) != 1: raise PappyException('Invalid number of arguments') - r = yield http.Request.load_request(args[0]) + r = yield Request.load_request(args[0]) def f(req): if req.time_start is None: return False @@ -444,11 +486,26 @@ def gen_filter_by_headers(args): comparer = compval_from_args_repdict(args) def f(req): if args[0][0] == 'n': - return comparer(req.headers) and (not req.response or comparer(req.response.headers)) + return comparer(req.headers) and ((not req.response) or comparer(req.response.headers)) else: return comparer(req.headers) or (req.response and comparer(req.response.headers)) return f +def gen_filter_by_request_headers(args): + comparer = compval_from_args_repdict(args) + def f(req): + return comparer(req.headers) + return f + +def gen_filter_by_response_headers(args): + comparer = compval_from_args_repdict(args) + def f(req): + if args[0][0] == 'n': + return (not req.response) or comparer(req.response.headers) + else: + return req.response and comparer(req.response.headers) + return f + def gen_filter_by_submitted_cookies(args): comparer = compval_from_args_repdict(args) def f(req): @@ -484,6 +541,13 @@ def gen_filter_by_params(args): return comparer(req.url_params) or comparer(req.post_params) return f +@defer.inlineCallbacks +def gen_filter_by_inverse(args): + filt = yield Filter.from_filter_string(parsed_args=args) + def f(req): + return not filt(req) + defer.returnValue(f) + @defer.inlineCallbacks def filter_reqs(reqids, filters): to_delete = set() diff --git a/pappyproxy/default_user_config.json b/pappyproxy/default_user_config.json index 1960092..2a3f306 100644 --- a/pappyproxy/default_user_config.json +++ b/pappyproxy/default_user_config.json @@ -4,5 +4,6 @@ "history_size": 1000, "proxy_listeners": [ {"port": 8000, "interface": "127.0.0.1"} - ] + ], + "socks_proxy": null } diff --git a/pappyproxy/http.py b/pappyproxy/http.py index 50e53d2..4679896 100644 --- a/pappyproxy/http.py +++ b/pappyproxy/http.py @@ -541,14 +541,21 @@ class HTTPMessage(object): # Initializes instance variables too self.clear() + self.metadata_unique_keys = tuple() if full_message is not None: self._from_full_message(full_message, update_content_length) def __eq__(self, other): - # TODO check meta if self.full_message != other.full_message: return False - if self.get_metadata() != other.get_metadata(): + m1 = self.get_metadata() + m2 = other.get_metadata() + for k in self.metadata_unique_keys: + if k in m1: + del m1[k] + if k in m2: + del m2[k] + if m1 != m2: return False return True @@ -556,7 +563,7 @@ class HTTPMessage(object): if not self.complete: raise PappyException("Cannot copy incomplete http messages") retmsg = self.__class__(self.full_message) - retmsg.set_metadata(self.get_metadata()) + retmsg.set_metadata(self.get_metadata(include_unique=False)) return retmsg def copy(self): @@ -840,8 +847,13 @@ class HTTPMessage(object): """ Called when the body of the message is complete """ - self.body = _decode_encoded(self._data_obj.body, - self._encoding_type) + try: + self.body = _decode_encoded(self._data_obj.body, + self._encoding_type) + except IOError as e: + # Screw handling it gracefully, this is the server's fault. + print 'Error decoding request, storing raw data in body instead' + self.body = self._data_obj.body def update_from_body(self): """ @@ -982,6 +994,9 @@ class Request(HTTPMessage): # instance vars HTTPMessage.__init__(self, full_request, update_content_length) + # metadata that is unique to a specific Request instance + self.metadata_unique_keys = ('reqid',) + # After message init so that other instance vars are initialized self._set_dict_callbacks() @@ -1267,7 +1282,7 @@ class Request(HTTPMessage): ########### ## Metadata - def get_metadata(self): + def get_metadata(self, include_unique=True): data = {} if self.port is not None: data['port'] = self.port @@ -1277,6 +1292,10 @@ class Request(HTTPMessage): if self.response: data['response_id'] = self.response.rspid data['tags'] = list(self.tags) + if not include_unique: + for k in self.metadata_unique_keys: + if k in data: + del data[k] return data def set_metadata(self, data): @@ -1344,7 +1363,7 @@ class Request(HTTPMessage): # Updates metadata that's based off of data HTTPMessage.update_from_body(self) if 'content-type' in self.headers: - if self.headers['content-type'] == 'application/x-www-form-urlencoded': + if 'application/x-www-form-urlencoded' in self.headers['content-type']: self.post_params = repeatable_parse_qs(self.body) self._set_dict_callbacks() @@ -1501,10 +1520,7 @@ class Request(HTTPMessage): else: use_cache = Request.cache if not self.reqid: - print 'adding' use_cache.add(self) - else: - print 'else adding' @defer.inlineCallbacks def async_save(self, cust_dbpool=None, cust_cache=None): @@ -2035,17 +2051,21 @@ class Request(HTTPMessage): :type full_request: string :rtype: Twisted deferred that calls back with a Request """ - from .proxy import ProxyClientFactory, get_next_connection_id, ClientTLSContext + from .proxy import ProxyClientFactory, get_next_connection_id, ClientTLSContext, get_endpoint + from .config import SOCKS_PROXY new_req = Request(full_request) new_req.is_ssl = is_ssl new_req.port = port - factory = ProxyClientFactory(new_req, save_all=False) + new_req._host = host + + factory = ProxyClientFactory(new_req, save_all=False, stream_response=False, return_transport=None) + factory.intercepting_macros = {} factory.connection_id = get_next_connection_id() - if is_ssl: - reactor.connectSSL(host, port, factory, ClientTLSContext()) - else: - reactor.connectTCP(host, port, factory) + yield factory.prepare_request() + endpoint = get_endpoint(host, port, is_ssl, + socks_config=SOCKS_PROXY) + yield endpoint.connect(factory) new_req = yield factory.data_defer defer.returnValue(new_req) @@ -2099,11 +2119,14 @@ class Response(HTTPMessage): def __init__(self, full_response=None, update_content_length=True): # Resets instance variables self.clear() - + # Called after instance vars since some callbacks depend on # instance vars HTTPMessage.__init__(self, full_response, update_content_length) + # metadata that is unique to a specific Response instance + self.metadata_unique_keys = ('rspid',) + # After message init so that other instance vars are initialized self._set_dict_callbacks() @@ -2190,9 +2213,13 @@ class Response(HTTPMessage): ########### ## Metadata - def get_metadata(self): + def get_metadata(self, include_unique=True): data = {} data['rspid'] = self.rspid + if not include_unique: + for k in self.metadata_unique_keys: + if k in data: + del data[k] return data def set_metadata(self, data): diff --git a/pappyproxy/macros.py b/pappyproxy/macros.py index 4144d7f..5c3e0d6 100644 --- a/pappyproxy/macros.py +++ b/pappyproxy/macros.py @@ -73,10 +73,8 @@ class InterceptMacro(object): self.intercept_requests = False self.intercept_responses = False - self.do_req = False - self.do_rsp = False - self.do_async_req = False - self.do_async_rsp = False + self.async_req = False + self.async_rsp = False def __repr__(self): return "" % self.name @@ -301,3 +299,79 @@ def gen_imacro(short_name='', long_name=''): template = env.get_template('intmacro.py.template') return template.render(**subs) +@defer.inlineCallbacks +def mangle_request(request, intmacros): + """ + Mangle a request with a list of intercepting macros. + Returns a tuple that contains the resulting request (with its unmangled + value set if needed) and a bool that states whether the request was modified + Returns (None, True) if the request was dropped. + + :rtype: (Request, Bool) + """ + # Mangle requests with list of intercepting macros + if not intmacros: + defer.returnValue((request, False)) + + cur_req = request.copy() + for k, macro in intmacros.iteritems(): + if macro.intercept_requests: + if macro.async_req: + cur_req = yield macro.async_mangle_request(cur_req.copy()) + else: + cur_req = macro.mangle_request(cur_req.copy()) + + if cur_req is None: + defer.returnValue((None, True)) + + mangled = False + if not cur_req == request or \ + not cur_req.host == request.host or \ + not cur_req.port == request.port: + # copy unique data to new request and clear it off old one + cur_req.unmangled = request + cur_req.unmangled.is_unmangled_version = True + if request.response: + cur_req.response = request.response + request.response = None + mangled = True + else: + # return the original request + cur_req = request + defer.returnValue((cur_req, mangled)) + +@defer.inlineCallbacks +def mangle_response(request, intmacros): + """ + Mangle a request's response with a list of intercepting macros. + Returns a bool stating whether the request's response was modified. + Unmangled values will be updated as needed. + + :rtype: Bool + """ + if not intmacros: + defer.returnValue(False) + + old_rsp = request.response + # We copy so that changes to request.response doesn't mangle the original response + request.response = request.response.copy() + for k, macro in intmacros.iteritems(): + if macro.intercept_responses: + if macro.async_rsp: + request.response = yield macro.async_mangle_response(request) + else: + request.response = macro.mangle_response(request) + + if request.response is None: + defer.returnValue(True) + + mangled = False + if not old_rsp == request.response: + request.response.rspid = old_rsp + old_rsp.rspid = None + request.response.unmangled = old_rsp + request.response.unmangled.is_unmangled_version = True + mangled = True + else: + request.response = old_rsp + defer.returnValue(mangled) diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index ec9878d..3915e75 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -26,7 +26,7 @@ from twisted.internet.protocol import ServerFactory from twisted.internet.threads import deferToThread crochet.no_setup() -server_factory = None +server_factories = [] main_context = context.Context() all_contexts = [main_context] plugin_loader = None @@ -69,7 +69,7 @@ def custom_int_handler(signum, frame): @defer.inlineCallbacks def main(): - global server_factory + global server_factories global plugin_loader global cons settings = parse_args() @@ -116,17 +116,24 @@ def main(): if config.DEBUG_DIR and os.path.exists(config.DEBUG_DIR): shutil.rmtree(config.DEBUG_DIR) print 'Removing old debugging output' - server_factory = proxy.ProxyServerFactory(save_all=True) listen_strs = [] ports = [] for listener in config.LISTENERS: + server_factory = proxy.ProxyServerFactory(save_all=True) try: - port = reactor.listenTCP(listener[0], server_factory, interface=listener[1]) - listener_str = 'port %d' % listener[0] - if listener[1] not in ('127.0.0.1', 'localhost'): - listener_str += ' (bound to %s)' % listener[1] + if 'forward_host_ssl' in listener and listener['forward_host_ssl']: + server_factory.force_ssl = True + server_factory.forward_host = listener['forward_host_ssl'] + elif 'forward_host' in listener and listener['forward_host']: + server_factory.force_ssl = False + server_factory.forward_host = listener['forward_host'] + port = reactor.listenTCP(listener['port'], server_factory, interface=listener['interface']) + listener_str = 'port %d' % listener['port'] + if listener['interface'] not in ('127.0.0.1', 'localhost'): + listener_str += ' (bound to %s)' % listener['interface'] listen_strs.append(listener_str) ports.append(port) + server_factories.append(server_factory) except CannotListenError as e: print repr(e) if listen_strs: diff --git a/pappyproxy/plugin.py b/pappyproxy/plugin.py index 4787859..860ccda 100644 --- a/pappyproxy/plugin.py +++ b/pappyproxy/plugin.py @@ -13,6 +13,7 @@ import stat from .proxy import add_intercepting_macro as proxy_add_intercepting_macro from .proxy import remove_intercepting_macro as proxy_remove_intercepting_macro +from .colors import Colors from .util import PappyException from twisted.internet import defer @@ -93,7 +94,8 @@ def add_intercepting_macro(name, macro): only use this if you may need to modify messages before they are passed along. """ - proxy_add_intercepting_macro(name, macro, pappyproxy.pappy.server_factory.intercepting_macros) + for factory in pappyproxy.pappy.server_factories: + proxy_add_intercepting_macro(name, macro, factory.intercepting_macros) def remove_intercepting_macro(name): """ @@ -102,14 +104,18 @@ def remove_intercepting_macro(name): :func:`pappyproxy.plugin.add_intercepting_macro` to identify which macro you would like to stop. """ - proxy_remove_intercepting_macro(name, pappyproxy.pappy.server_factory.intercepting_macros) + for factory in pappyproxy.pappy.server_factories: + proxy_remove_intercepting_macro(name, factory.intercepting_macros) def active_intercepting_macros(): """ Returns a list of the active intercepting macro objects. Modifying this list will not affect which macros are active. """ - return [v for k, v in pappyproxy.pappy.server_factory.intercepting_macros.iteritems() ] + ret = [] + for factory in pappyproxy.pappy.server_factories: + ret += [v for k, v in factory.intercepting_macros.iteritems() ] + return ret def in_memory_reqs(): """ @@ -158,3 +164,33 @@ def run_cmd(cmd): existing APIs to do what you want before using this. """ pappyproxy.pappy.cons.onecmd(cmd) + +def require_modules(*largs): + """ + A wrapper to make sure that plugin dependencies are installed. For example, + if a command requires the ``psutil`` and ``objgraph`` package, you should + format your command like:: + + @require_modules('psutil', 'objgraph') + def my_command(line): + import objgraph + import psutil + # ... rest of command ... + + If you try to run the command without being able to import all of the required + modules, the command will print an error and not run the command. + """ + def wr(func): + def wr2(*args, **kwargs): + missing = [] + for l in largs: + try: + imp.find_module(l) + except ImportError: + missing.append(l) + if missing: + print 'Command requires %s module(s)' % (', '.join([Colors.RED+m+Colors.ENDC for m in missing])) + else: + return func(*args, **kwargs) + return wr2 + return wr diff --git a/pappyproxy/plugins/debug.py b/pappyproxy/plugins/debug.py index 5e805bd..66afd98 100644 --- a/pappyproxy/plugins/debug.py +++ b/pappyproxy/plugins/debug.py @@ -11,6 +11,7 @@ from pappyproxy.util import PappyException from pappyproxy.requestcache import RequestCache from pappyproxy.console import print_requests from pappyproxy.pappy import heapstats, cons +from pappyproxy.plugin import require_modules from twisted.internet import defer def cache_info(line): @@ -23,19 +24,16 @@ def cache_info(line): rs = sorted(rl, key=lambda r: Request.cache._last_used[r.reqid], reverse=True) print_requests(rs) +@require_modules('psutil') def memory_info(line): - try: - import psutil - except ImportError: - raise PappyException('This command requires the psutil package') + import psutil proc = psutil.Process(os.getpid()) mem = proc.memory_info().rss megabyte = (float(mem)/1024)/1024 print 'Memory usage: {0:.2f} Mb ({1} bytes)'.format(megabyte, mem) +@require_modules('guppy') def heap_info(line): - if heapstats is None: - raise PappyException('Command requires the guppy library') size = heapstats.heap().size print 'Heap usage: {0:.2f} Mb'.format(size/(1024.0*1024.0)) print heapstats.heap() @@ -54,11 +52,9 @@ def limit_info(line): print 'Soft limit is now:', soft print 'Hard limit is now:', hard +@require_modules('objgraph') def graph_randobj(line): - try: - import objgraph - except ImportError: - raise PappyException('This command requires the objgraph library') + import objgraph args = shlex.split(line) if len(args) > 1: fname = args[1] diff --git a/pappyproxy/plugins/decode.py b/pappyproxy/plugins/decode.py index c348d53..9072d61 100644 --- a/pappyproxy/plugins/decode.py +++ b/pappyproxy/plugins/decode.py @@ -2,12 +2,13 @@ import HTMLParser import StringIO import base64 import clipboard +import datetime import gzip import shlex import string import urllib -from pappyproxy.util import PappyException, hexdump +from pappyproxy.util import PappyException, hexdump, printable_data def print_maybe_bin(s): binary = False @@ -231,6 +232,14 @@ def gzip_encode_raw(line): to a file. """ print _code_helper(line, gzip_encode_helper, copy=False) + +def unix_time_decode_helper(line): + unix_time = int(line.strip()) + dtime = datetime.datetime.fromtimestamp(unix_time) + return dtime.strftime('%Y-%m-%d %H:%M:%S') + +def unix_time_decode(line): + print _code_helper(line, unix_time_decode_helper) def load_cmds(cmd): cmd.set_cmds({ @@ -254,6 +263,7 @@ def load_cmds(cmd): 'html_encode_raw': (html_encode_raw, None), 'gzip_decode_raw': (gzip_decode_raw, None), 'gzip_encode_raw': (gzip_encode_raw, None), + 'unixtime_decode': (unix_time_decode, None), }) cmd.add_aliases([ ('base64_decode', 'b64d'), @@ -276,4 +286,5 @@ def load_cmds(cmd): ('html_encode_raw', 'htmler'), ('gzip_decode_raw', 'gzdr'), ('gzip_encode_raw', 'gzer'), + ('unixtime_decode', 'uxtd'), ]) diff --git a/pappyproxy/plugins/macrocmds.py b/pappyproxy/plugins/macrocmds.py index c2843d5..abbb910 100644 --- a/pappyproxy/plugins/macrocmds.py +++ b/pappyproxy/plugins/macrocmds.py @@ -96,9 +96,13 @@ def run_int_macro(line): if args[0] not in int_macro_dict: raise PappyException('%s not a loaded intercepting macro' % line) macro = int_macro_dict[args[0]] - macro.init(args[1:]) - add_intercepting_macro(macro.name, macro) - print '"%s" started' % macro.name + try: + macro.init(args[1:]) + add_intercepting_macro(macro.name, macro) + print '"%s" started' % macro.name + except Exception as e: + print 'Error initializing macro:' + raise e def stop_int_macro(line): """ diff --git a/pappyproxy/plugins/manglecmds.py b/pappyproxy/plugins/manglecmds.py index 67f1e49..bed2c01 100644 --- a/pappyproxy/plugins/manglecmds.py +++ b/pappyproxy/plugins/manglecmds.py @@ -58,6 +58,7 @@ class MangleInterceptMacro(InterceptMacro): defer.returnValue(None) mangled_req = Request(text, update_content_length=True) + mangled_req._host = request.host mangled_req.port = request.port mangled_req.is_ssl = request.is_ssl @@ -126,7 +127,6 @@ def check_reqid(reqid): def start_editor(reqid): script_loc = os.path.join(config.PAPPY_DIR, "plugins", "vim_repeater", "repeater.vim") - #print "RepeaterSetup %d %d"%(reqid, comm_port) subprocess.call(["vim", "-S", script_loc, "-c", "RepeaterSetup %s %d"%(reqid, comm.comm_port)]) #################### diff --git a/pappyproxy/plugins/misc.py b/pappyproxy/plugins/misc.py index 7b67d0a..1b20f36 100644 --- a/pappyproxy/plugins/misc.py +++ b/pappyproxy/plugins/misc.py @@ -66,7 +66,10 @@ def clrmem(line): """ to_delete = list(pappyproxy.http.Request.cache.inmem_reqs) for r in to_delete: - yield r.deep_delete() + try: + yield r.deep_delete() + except PappyException as e: + print str(e) def gencerts(line): """ diff --git a/pappyproxy/plugins/view.py b/pappyproxy/plugins/view.py index ea083a3..2a3f8f3 100644 --- a/pappyproxy/plugins/view.py +++ b/pappyproxy/plugins/view.py @@ -74,9 +74,10 @@ def print_request_extended(request): print_pairs = [] print_pairs.append(('Made on', time_made_str)) print_pairs.append(('ID', request.reqid)) - print_pairs.append(('Verb', verb)) + print_pairs.append(('URL', request.url_color)) print_pairs.append(('Host', host)) print_pairs.append(('Path', path_formatter(request.full_path))) + print_pairs.append(('Verb', verb)) print_pairs.append(('Status Code', response_code)) print_pairs.append(('Request Length', reqlen)) print_pairs.append(('Response Length', rsplen)) @@ -97,6 +98,14 @@ def print_tree(tree): # Prints a tree. Takes in a sorted list of path tuples _print_tree_helper(tree, 0, []) +def guess_pretty_print_fmt(msg): + if 'content-type' in msg.headers: + if 'json' in msg.headers['content-type']: + return 'json' + elif 'www-form' in msg.headers['content-type']: + return 'form' + return 'text' + def pretty_print_body(fmt, body): try: if fmt.lower() == 'json': @@ -111,6 +120,8 @@ def pretty_print_body(fmt, body): s += Colors.ENDC s += urllib.unquote(v) print s + elif fmt.lower() == 'text': + print body else: raise PappyException('"%s" is not a valid format' % fmt) except PappyException as e: @@ -165,8 +176,59 @@ def _print_tree_helper(tree, depth, print_bars): curkey = '/' print _get_tree_prefix(depth, print_bars, True) + curkey _print_tree_helper(subtree, depth+1, print_bars + [False]) - +def print_params(req, params=None): + if not req.url_params.all_pairs() and not req.body: + print 'Request %s has no url or data parameters' % req.reqid + print '' + if req.url_params.all_pairs(): + print Styles.TABLE_HEADER + "Url Params" + Colors.ENDC + for k, v in req.url_params.all_pairs(): + if params is None or (params and k in params): + print Styles.KV_KEY+str(k)+': '+Styles.KV_VAL+str(v) + print '' + if req.body: + print Styles.TABLE_HEADER + "Body/POST Params" + Colors.ENDC + pretty_print_body(guess_pretty_print_fmt(req), req.body) + print '' + if req.cookies.all_pairs(): + print Styles.TABLE_HEADER + "Cookies" + Colors.ENDC + for k, v in req.cookies.all_pairs(): + if params is None or (params and k in params): + print Styles.KV_KEY+str(k)+': '+Styles.KV_VAL+str(v) + print '' + # multiform request when we support it + +def add_param(found_params, kind, k, v, reqid): + if not k in found_params: + found_params[k] = {} + if kind in found_params[k]: + found_params[k][kind].append((reqid, v)) + else: + found_params[k][kind] = [(reqid, v)] + +def print_param_info(param_info): + for k, d in param_info.iteritems(): + print Styles.TABLE_HEADER + k + Colors.ENDC + for param_type, valpairs in d.iteritems(): + print param_type + value_ids = {} + for reqid, val in valpairs: + ids = value_ids.get(val, []) + ids.append(reqid) + value_ids[val] = ids + for val, ids in value_ids.iteritems(): + if len(ids) <= 15: + idstr = ', '.join(ids) + else: + idstr = ', '.join(ids[:15]) + '...' + if val == '': + printstr = (Colors.RED + 'BLANK' + Colors.ENDC + 'x%d (%s)') % (len(ids), idstr) + else: + printstr = (Colors.GREEN + '%s' + Colors.ENDC + 'x%d (%s)') % (val, len(ids), idstr) + print printstr + print '' + #################### ## Command functions @@ -359,6 +421,70 @@ def pretty_print_response(line): else: print 'No response associated with request %s' % req.reqid +@crochet.wait_for(timeout=None) +@defer.inlineCallbacks +def print_params_cmd(line): + """ + View the headers of the request + Usage: view_request_headers + """ + args = shlex.split(line) + reqid = args[0] + if len(args) > 1: + keys = args[1:] + else: + keys = None + + reqs = yield load_reqlist(reqid) + for req in reqs: + if len(reqs) > 1: + print 'Request %s:' % req.reqid + print_params(req, keys) + if len(reqs) > 1: + print '-'*30 + +@crochet.wait_for(timeout=None) +@defer.inlineCallbacks +def get_param_info(line): + args = shlex.split(line) + if args and args[0] == 'ct': + contains = True + args = args[1:] + else: + contains = False + + if args: + params = tuple(args) + else: + params = None + + def check_key(k, params, contains): + if contains: + for p in params: + if p.lower() in k.lower(): + return True + else: + if params is None or k in params: + return True + return False + + found_params = {} + + ids = yield main_context_ids() + for i in ids: + req = yield Request.load_request(i) + for k, v in req.url_params.all_pairs(): + if check_key(k, params, contains): + add_param(found_params, 'Url Parameter', k, v, req.reqid) + for k, v in req.post_params.all_pairs(): + if check_key(k, params, contains): + add_param(found_params, 'POST Parameter', k, v, req.reqid) + for k, v in req.cookies.all_pairs(): + if check_key(k, params, contains): + add_param(found_params, 'Cookie', k, v, req.reqid) + print_param_info(found_params) + + @crochet.wait_for(timeout=None) @defer.inlineCallbacks def dump_response(line): @@ -387,6 +513,11 @@ def site_map(line): Print the site map. Only includes requests in the current context. Usage: site_map """ + args = shlex.split(line) + if len(args) > 0 and args[0] == 'p': + paths = True + else: + paths = False ids = yield main_context_ids() paths_set = set() for reqid in ids: @@ -394,7 +525,11 @@ def site_map(line): if req.response and req.response.response_code != 404: paths_set.add(req.path_tuple) tree = sorted(list(paths_set)) - print_tree(tree) + if paths: + for p in tree: + print ('/'.join(list(p))) + else: + print_tree(tree) ############### @@ -412,6 +547,8 @@ def load_cmds(cmd): 'view_full_response': (view_full_response, None), 'view_response_bytes': (view_response_bytes, None), 'pretty_print_response': (pretty_print_response, None), + 'print_params': (print_params_cmd, None), + 'param_info': (get_param_info, None), 'site_map': (site_map, None), 'dump_response': (dump_response, None), }) @@ -420,12 +557,16 @@ def load_cmds(cmd): ('view_request_info', 'viq'), ('view_request_headers', 'vhq'), ('view_full_request', 'vfq'), + ('view_full_request', 'kjq'), ('view_request_bytes', 'vbq'), ('pretty_print_request', 'ppq'), ('view_response_headers', 'vhs'), ('view_full_response', 'vfs'), + ('view_full_response', 'kjs'), ('view_response_bytes', 'vbs'), ('pretty_print_response', 'pps'), + ('print_params', 'pprm'), + ('param_info', 'pri'), ('site_map', 'sm'), #('dump_response', 'dr'), ]) diff --git a/pappyproxy/plugins/vim_repeater/repeater.py b/pappyproxy/plugins/vim_repeater/repeater.py index d6115bd..0609e21 100644 --- a/pappyproxy/plugins/vim_repeater/repeater.py +++ b/pappyproxy/plugins/vim_repeater/repeater.py @@ -119,7 +119,7 @@ def submit_current_buffer(): full_request = '\n'.join(curbuf) commdata = {'action': 'submit', 'full_message': base64.b64encode(full_request), - 'tags': {'repeater'}, + 'tags': ['repeater'], 'port': int(vim.eval("s:repport")), 'host': vim.eval("s:rephost")} if vim.eval("s:repisssl") == '1': diff --git a/pappyproxy/proxy.py b/pappyproxy/proxy.py index 93bbaf9..107168d 100644 --- a/pappyproxy/proxy.py +++ b/pappyproxy/proxy.py @@ -1,3 +1,4 @@ +import collections import copy import datetime import os @@ -8,6 +9,7 @@ from OpenSSL import crypto from pappyproxy import config from pappyproxy import context from pappyproxy import http +from pappyproxy import macros from pappyproxy.util import PappyException, printable_data from twisted.internet import defer from twisted.internet import reactor, ssl @@ -56,6 +58,34 @@ def log_request(request, id=None, symbol='*', verbosity_level=3): r_split = request.split('\r\n') for l in r_split: log(l, id, symbol, verbosity_level) + +def get_endpoint(target_host, target_port, target_ssl, socks_config=None): + + # Imports go here to allow mocking for tests + from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint + from txsocksx.client import SOCKS5ClientEndpoint + from txsocksx.tls import TLSWrapClientEndpoint + from twisted.internet.interfaces import IOpenSSLClientConnectionCreator + + if socks_config is not None: + sock_host = socks_config['host'] + sock_port = int(socks_config['port']) + methods = {'anonymous': ()} + if 'username' in socks_config and 'password' in socks_config: + methods['login'] = (socks_config['username'], socks_config['password']) + tcp_endpoint = TCP4ClientEndpoint(reactor, sock_host, sock_port) + socks_endpoint = SOCKS5ClientEndpoint(target_host, target_port, tcp_endpoint, methods=methods) + if target_ssl: + endpoint = TLSWrapClientEndpoint(ClientTLSContext(), socks_endpoint) + else: + endpoint = socks_endpoint + else: + if target_ssl: + endpoint = SSL4ClientEndpoint(reactor, target_host, target_port, + ClientTLSContext()) + else: + endpoint = TCP4ClientEndpoint(reactor, target_host, target_port) + return endpoint class ClientTLSContext(ssl.ClientContextFactory): isClient = 1 @@ -71,6 +101,7 @@ class ProxyClient(LineReceiver): self._sent = False self.request = request self.data_defer = defer.Deferred() + self.completed = False self._response_obj = http.Response() @@ -83,24 +114,12 @@ class ProxyClient(LineReceiver): line = '' self._response_obj.add_line(line) self.log(line, symbol='r<', verbosity_level=3) - if self.factory.stream_response: - self.log('Returning line back through stream') - self.factory.return_transport.write(line+'\r\n') - else: - self.log('Not streaming, not returning') - self.log(self.factory.stream_response) if self._response_obj.headers_complete: - if self._response_obj.complete: - self.handle_response_end() - return - self.log("Headers end, length given, waiting for data", verbosity_level=3) self.setRawMode() def rawDataReceived(self, *args, **kwargs): data = args[0] self.log('Returning data back through stream') - if self.factory.stream_response: - self.factory.return_transport.write(data) if not self._response_obj.complete: if data: if config.DEBUG_TO_FILE or config.DEBUG_VERBOSITY > 0: @@ -110,71 +129,21 @@ class ProxyClient(LineReceiver): self.log(l, symbol=' 0: log_request(printable_data(request.response.full_response), id=self.connection_id, symbol=' 0): log_request(printable_data(request.response.full_response), @@ -267,8 +261,10 @@ class ProxyClientFactory(ClientFactory): class ProxyServerFactory(ServerFactory): def __init__(self, save_all=False): - self.intercepting_macros = {} + self.intercepting_macros = collections.OrderedDict() self.save_all = save_all + self.force_ssl = False + self.forward_host = None def buildProtocol(self, addr): prot = ProxyServer() @@ -288,101 +284,167 @@ class ProxyServer(LineReceiver): self._connect_response = False self._forward = True self._connect_uri = None + self._connect_host = None + self._connect_ssl = None + self._connect_port = None + self._client_factory = None def lineReceived(self, *args, **kwargs): line = args[0] self.log(line, symbol='>', verbosity_level=3) self._request_obj.add_line(line) - if self._request_obj.verb.upper() == 'CONNECT': - self._connect_response = True - self._forward = False - self._connect_uri = self._request_obj.url - if self._request_obj.headers_complete: self.setRawMode() - - if self._request_obj.complete: - self.setLineMode() - try: - self.full_request_received() - except PappyException as e: - print str(e) def rawDataReceived(self, *args, **kwargs): data = args[0] self._request_obj.add_data(data) self.log(data, symbol='d>', verbosity_level=3) + def dataReceived(self, *args, **kwargs): + # receives the data then checks if the request is complete. + # if it is, it calls full_Request_received + LineReceiver.dataReceived(self, *args, **kwargs) + if self._request_obj.complete: try: self.full_request_received() except PappyException as e: print str(e) - def full_request_received(self, *args, **kwargs): + def _start_tls(self, cert_host=None): + # Generate a cert for the hostname and start tls + if cert_host is None: + host = self._request_obj.host + else: + host = cert_host + if not host in cached_certs: + log("Generating cert for '%s'" % host, + verbosity_level=3) + (pkey, cert) = generate_cert(host, + config.CERT_DIR) + cached_certs[host] = (pkey, cert) + else: + log("Using cached cert for %s" % host, verbosity_level=3) + (pkey, cert) = cached_certs[host] + ctx = ServerTLSContext( + private_key=pkey, + certificate=cert, + ) + self.transport.startTLS(ctx, self.factory) + + def _connect_okay(self): + self.log('Responding to browser CONNECT request', verbosity_level=3) + okay_str = 'HTTP/1.1 200 Connection established\r\n\r\n' + self.transport.write(okay_str) + + def full_request_received(self): global cached_certs self.log('End of request', verbosity_level=3) - if self._connect_response: - self.log('Responding to browser CONNECT request', verbosity_level=3) - okay_str = 'HTTP/1.1 200 Connection established\r\n\r\n' - self.transport.write(okay_str) - - # Generate a cert for the hostname - if not self._request_obj.host in cached_certs: - log("Generating cert for '%s'" % self._request_obj.host, - verbosity_level=3) - (pkey, cert) = generate_cert(self._request_obj.host, - config.CERT_DIR) - cached_certs[self._request_obj.host] = (pkey, cert) - else: - log("Using cached cert for %s" % self._request_obj.host, verbosity_level=3) - (pkey, cert) = cached_certs[self._request_obj.host] - ctx = ServerTLSContext( - private_key=pkey, - certificate=cert, - ) - self.transport.startTLS(ctx, self.factory) - - if self._forward: - self.log("Forwarding to %s on %d" % (self._request_obj.host, self._request_obj.port)) - if not self.factory.intercepting_macros: - stream = True - else: - # We only want to call send_response_back if we're not streaming - stream = False - self.log('Creating client factory, stream=%s' % stream) - factory = ProxyClientFactory(self._request_obj, - save_all=self.factory.save_all, - stream_response=stream, - return_transport=self.transport) - factory.intercepting_macros = self.factory.intercepting_macros - factory.connection_id = self.connection_id - if not stream: - factory.data_defer.addCallback(self.send_response_back) - if self._request_obj.is_ssl: - self.log("Accessing over SSL...", verbosity_level=3) - reactor.connectSSL(self._request_obj.host, self._request_obj.port, factory, ClientTLSContext()) - else: - self.log("Accessing over TCP...", verbosity_level=3) - reactor.connectTCP(self._request_obj.host, self._request_obj.port, factory) - - # Reset per-request variables + forward = True + if self._request_obj.verb.upper() == 'CONNECT': + self._connect_okay() + self._start_tls() + self._connect_uri = self._request_obj.url + self._connect_host = self._request_obj.host + self._connect_ssl = True # do we just assume connect means ssl? + self._connect_port = self._request_obj.port + self.log('uri=%s, ssl=%s, connect_port=%s' % (self._connect_uri, self._connect_ssl, self._connect_port), verbosity_level=3) + forward = False + + # if self._request_obj.host == 'pappy': + # self._create_pappy_response() + # forward = False + + # if _request_obj.host is a listener, forward = False + + if forward: + self._generate_and_submit_client() + self._reset() + + def _reset(self): + # Reset per-request variables and have the request default to using + # some parameters from the connect request self.log("Resetting per-request data", verbosity_level=3) self._connect_response = False - self._forward = True self._request_obj = http.Request() if self._connect_uri: self._request_obj.url = self._connect_uri + if self._connect_host: + self._request_obj._host = self._connect_host + if self._connect_ssl: + self._request_obj.is_ssl = self._connect_ssl + if self._connect_port: + self._request_obj.port = self._connect_port self.setLineMode() + def _generate_and_submit_client(self): + """ + Sets up self._client_factory with self._request_obj then calls back to + submit the request + """ + + self.log("Forwarding to %s on %d" % (self._request_obj.host, self._request_obj.port)) + if self.factory.intercepting_macros: + stream = False + else: + stream = True + self.log('Creating client factory, stream=%s' % stream) + self._client_factory = ProxyClientFactory(self._request_obj, + save_all=self.factory.save_all, + stream_response=stream, + return_transport=self.transport) + self._client_factory.intercepting_macros = self.factory.intercepting_macros + self._client_factory.connection_id = self.connection_id + if not stream: + self._client_factory.data_defer.addCallback(self.send_response_back) + d = self._client_factory.prepare_request() + d.addCallback(self._make_remote_connection) + return d + + @defer.inlineCallbacks + def _make_remote_connection(self, req): + """ + Creates an endpoint to the target server using the given configuration + options then connects to the endpoint using self._client_factory + """ + self._request_obj = req + + # If we have a socks proxy, wrap the endpoint in it + if context.in_scope(self._request_obj): + # Modify the request connection settings to match settings in the factory + if self.factory.force_ssl: + self._request_obj.is_ssl = True + if self.factory.forward_host: + self._request_obj.host = self.factory.forward_host + + # Get connection from the request + endpoint = get_endpoint(self._request_obj.host, + self._request_obj.port, + self._request_obj.is_ssl, + socks_config=config.SOCKS_PROXY) + else: + endpoint = get_endpoint(self._request_obj.host, + self._request_obj.port, + self._request_obj.is_ssl) + + # Connect via the endpoint + self.log("Accessing using endpoint") + yield endpoint.connect(self._client_factory) + self.log("Connected") + def send_response_back(self, response): if response is not None: self.transport.write(response.response.full_response) self.log("Response sent back, losing connection") self.transport.loseConnection() + + def connectionMade(self): + if self.factory.force_ssl: + self._start_tls(self.factory.forward_host) def connectionLost(self, reason): self.log('Connection lost with browser: %s' % reason.getErrorMessage()) @@ -425,7 +487,7 @@ def load_certs_from_dir(cert_dir): with open(cert_dir+'/'+config.SSL_CA_FILE, 'rt') as f: ca_raw = f.read() except IOError: - raise PappyException("Could not load CA cert!") + raise PappyException("Could not load CA cert! Generate certs using the `gencerts` command then add the .crt file to your browser.") try: with open(cert_dir+'/'+config.SSL_PKEY_FILE, 'rt') as f: diff --git a/pappyproxy/requestcache.py b/pappyproxy/requestcache.py index 7fa9417..b0a68d4 100644 --- a/pappyproxy/requestcache.py +++ b/pappyproxy/requestcache.py @@ -186,8 +186,8 @@ class RequestCache(object): break @defer.inlineCallbacks - def load_by_tag(tag): - reqs = yield load_requests_by_tag(tag, cust_cache=self, cust_dbpool=self.dbpool) + def load_by_tag(self, tag): + reqs = yield pappyproxy.http.Request.load_requests_by_tag(tag, cust_cache=self, cust_dbpool=self.dbpool) for req in reqs: self.add(req) defer.returnValue(reqs) diff --git a/pappyproxy/tests/test_macros.py b/pappyproxy/tests/test_macros.py new file mode 100644 index 0000000..911ac97 --- /dev/null +++ b/pappyproxy/tests/test_macros.py @@ -0,0 +1,65 @@ +import pytest +import string +import mock + +from collections import OrderedDict +from testutil import mock_deferred, func_deleted, TLSStringTransport, freeze, mock_int_macro, no_tcp +from pappyproxy.http import Request, Response +from pappyproxy import macros + +class CloudToButtMacro(macros.InterceptMacro): + + def __init__(self): + macros.InterceptMacro.__init__(self) + self.intercept_requests = True + self.intercept_responses = True + + def mangle_request(self, request): + return Request(string.replace(request.full_message, 'cloud', 'butt')) + + def mangle_response(self, response): + return Response(string.replace(response.full_message, 'cloud', 'butt')) + +@pytest.fixture +def httprequest(): + return Request(('POST /test-request HTTP/1.1\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'AAAA')) + +@pytest.inlineCallbacks +def test_mangle_request_simple(httprequest): + orig_req = httprequest.copy() # in case it gets mangled + (new_req, mangled) = yield macros.mangle_request(orig_req, {}) + assert new_req == orig_req + assert httprequest == orig_req + assert not mangled + +@pytest.inlineCallbacks +def test_mangle_request_single(httprequest): + orig_req = httprequest.copy() # in case it gets mangled + macro = mock_int_macro(modified_req=('GET /modified HTTP/1.1\r\n\r\n')) + expected_req = Request('GET /modified HTTP/1.1\r\n\r\n') + (new_req, mangled) = yield macros.mangle_request(orig_req, {'testmacro': macro}) + assert new_req == expected_req + assert httprequest == orig_req + assert httprequest.unmangled is None + assert new_req.unmangled == orig_req + assert mangled + +@pytest.inlineCallbacks +def test_mangle_request_multiple(httprequest): + orig_req = httprequest.copy() # in case it gets mangled + macro = mock_int_macro(modified_req=('GET /cloud HTTP/1.1\r\n\r\n')) + macro2 = CloudToButtMacro() + intmacros = OrderedDict() + intmacros['testmacro'] = macro + intmacros['testmacro2'] = macro2 + (new_req, mangled) = yield macros.mangle_request(orig_req, intmacros) + + expected_req = Request('GET /butt HTTP/1.1\r\n\r\n') + assert new_req == expected_req + assert httprequest == orig_req + assert httprequest.unmangled is None + assert new_req.unmangled == orig_req + assert mangled diff --git a/pappyproxy/tests/test_proxy.py b/pappyproxy/tests/test_proxy.py index d29573d..f57efea 100644 --- a/pappyproxy/tests/test_proxy.py +++ b/pappyproxy/tests/test_proxy.py @@ -1,82 +1,56 @@ -import os import pytest import mock -import twisted.internet -import twisted.test +import random +import datetime +import pappyproxy from pappyproxy import http -from pappyproxy import macros -from pappyproxy import config -from pappyproxy.proxy import ProxyClient, ProxyClientFactory, ProxyServerFactory -from testutil import mock_deferred, func_deleted, func_ignored_deferred, func_ignored, no_tcp -from twisted.internet.protocol import ServerFactory -from twisted.test.iosim import FakeTransport -from twisted.internet import defer, reactor +from pappyproxy.proxy import ProxyClientFactory, ProxyServerFactory +from testutil import mock_deferred, func_deleted, TLSStringTransport, freeze, mock_int_macro, no_tcp -#################### -## Fixtures - -MANGLED_REQ = 'GET /mangled HTTP/1.1\r\n\r\n' -MANGLED_RSP = 'HTTP/1.1 500 MANGLED\r\nContent-Length: 0\r\n\r\n' - -@pytest.fixture -def unconnected_proxyserver(mocker): - mocker.patch("twisted.test.iosim.FakeTransport.startTLS") +@pytest.fixture(autouse=True) +def proxy_patches(mocker): + #mocker.patch("twisted.test.iosim.FakeTransport.startTLS") mocker.patch("pappyproxy.proxy.load_certs_from_dir", new=mock_generate_cert) - factory = ProxyServerFactory() - protocol = factory.buildProtocol(('127.0.0.1', 0)) - protocol.makeConnection(FakeTransport(protocol, True)) - return protocol @pytest.fixture -def proxyserver(mocker): - mocker.patch("twisted.test.iosim.FakeTransport.startTLS") - mocker.patch("pappyproxy.proxy.load_certs_from_dir", new=mock_generate_cert) +def server_factory(): + return gen_server_factory() + +def socks_config(mocker, config): + mocker.patch('pappyproxy.config.SOCKS_PROXY', new=config) + +def gen_server_factory(int_macros={}): factory = ProxyServerFactory() - protocol = factory.buildProtocol(('127.0.0.1', 0)) - protocol.makeConnection(FakeTransport(protocol, True)) - protocol.lineReceived('CONNECT https://www.AAAA.BBBB:443 HTTP/1.1') - protocol.lineReceived('') - protocol.transport.getOutBuffer() + factory.save_all = True + factory.intercepting_macros = int_macros + return factory + +def gen_server_protocol(int_macros={}): + server_factory = gen_server_factory(int_macros=int_macros) + protocol = server_factory.buildProtocol(('127.0.0.1', 0)) + tr = TLSStringTransport() + protocol.makeConnection(tr) return protocol - -@pytest.fixture -def proxy_connection(): - @defer.inlineCallbacks - def gen_connection(send_data, new_req=False, new_rsp=False, - drop_req=False, drop_rsp=False): - factory = ProxyClientFactory(http.Request(send_data)) - - macro = gen_mangle_macro(new_req, new_rsp, drop_req, drop_rsp) - factory.intercepting_macros['pappy_mangle'] = macro - - protocol = factory.buildProtocol(None) - tr = FakeTransport(protocol, True) - protocol.makeConnection(tr) - sent = yield protocol.data_defer - print sent - defer.returnValue((protocol, sent, factory.data_defer)) - return gen_connection -@pytest.fixture -def in_scope_true(mocker): - new_in_scope = mock.MagicMock() - new_in_scope.return_value = True - mocker.patch("pappyproxy.context.in_scope", new=new_in_scope) - return new_in_scope +def gen_client_protocol(req, stream_response=False): + return_transport = TLSStringTransport() + factory = ProxyClientFactory(req, + save_all=True, + stream_response=stream_response, + return_transport=return_transport) + protocol = factory.buildProtocol(('127.0.0.1', 0), _do_callback=False) + tr = TLSStringTransport() + protocol.makeConnection(tr) + return protocol @pytest.fixture -def in_scope_false(mocker): - new_in_scope = mock.MagicMock() - new_in_scope.return_value = False - mocker.patch("pappyproxy.context.in_scope", new=new_in_scope) - return new_in_scope +def server_protocol(): + return gen_server_protocol() -## Autorun fixtures - -@pytest.fixture(autouse=True) -def ignore_save(mocker): - mocker.patch("pappyproxy.http.Request.async_deep_save", func_ignored_deferred) +def mock_req_async_save(req): + req.reqid = str(random.randint(1,1000000)) + return mock_deferred() #################### ## Mock functions @@ -134,151 +108,522 @@ def mock_generate_cert(cert_dir): '-----END CERTIFICATE-----') return (ca_key, private_key) -def gen_mangle_macro(modified_req=None, modified_rsp=None, - drop_req=False, drop_rsp=False): - macro = mock.MagicMock() - if modified_req or drop_req: - macro.async_req = True - macro.intercept_requests = True - if drop_req: - newreq = None - else: - newreq = http.Request(modified_req) - macro.async_mangle_request.return_value = mock_deferred(newreq) - else: - macro.intercept_requests = False - - if modified_rsp or drop_rsp: - macro.async_rsp = True - macro.intercept_responses = True - if drop_rsp: - newrsp = None - else: - newrsp = http.Response(modified_rsp) - macro.async_mangle_response.return_value = mock_deferred(newrsp) - else: - macro.intercept_responses = False - return macro - -def notouch_mangle_req(request): - d = mock_deferred(request) - return d - -def notouch_mangle_rsp(request): - d = mock_deferred(request.response) - return d - -def req_mangler_change(request): - req = http.Request('GET /mangled HTTP/1.1\r\n\r\n') - d = mock_deferred(req) - return d - -def rsp_mangler_change(request): - rsp = http.Response('HTTP/1.1 500 MANGLED\r\n\r\n') - d = mock_deferred(rsp) - return d - -def req_mangler_drop(request): - return mock_deferred(None) - -def rsp_mangler_drop(request): - return mock_deferred(None) +######## +## Tests -#################### -## Unit test tests +def test_no_tcp(): + from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint + from txsocksx.client import SOCKS5ClientEndpoint + from txsocksx.tls import TLSWrapClientEndpoint + with pytest.raises(NotImplementedError): + SSL4ClientEndpoint('aasdfasdf.sdfwerqwer') + with pytest.raises(NotImplementedError): + TCP4ClientEndpoint('aasdfasdf.sdfwerqwer') + with pytest.raises(NotImplementedError): + SOCKS5ClientEndpoint('aasdfasdf.sdfwerqwer') + with pytest.raises(NotImplementedError): + TLSWrapClientEndpoint('asdf.2341') + +################ +### Proxy Server + +def test_proxy_server_connect(mocker, server_protocol): + mstarttls = mocker.patch('pappyproxy.tests.testutil.TLSStringTransport.startTLS') + server_protocol.dataReceived('CONNECT https://www.AAAA.BBBB:443 HTTP/1.1\r\n\r\n') + assert server_protocol.transport.value() == 'HTTP/1.1 200 Connection established\r\n\r\n' + assert mstarttls.called + +def test_proxy_server_forward_basic(mocker, server_protocol): + mforward = mocker.patch('pappyproxy.proxy.ProxyServer._generate_and_submit_client') + mreset = mocker.patch('pappyproxy.proxy.ProxyServer._reset') + + req_contents = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'ABCD') + server_protocol.dataReceived(req_contents) + + assert mforward.called + assert mreset.called + assert server_protocol._request_obj.full_message == req_contents + +def test_proxy_server_connect_uri(mocker, server_protocol): + mforward = mocker.patch('pappyproxy.proxy.ProxyServer._generate_and_submit_client') + server_protocol.dataReceived('CONNECT https://www.AAAA.BBBB:443 HTTP/1.1\r\n\r\n') + server_protocol.dataReceived('GET /fooo HTTP/1.1\r\nTest-Header: foo\r\n\r\n') + assert server_protocol._connect_uri == 'https://www.AAAA.BBBB' + assert server_protocol._request_obj.url == 'https://www.AAAA.BBBB' + assert server_protocol._request_obj.port == 443 + +## ProxyServer._generate_and_submit_client + +def test_proxy_server_create_client_factory(mocker, server_protocol): + mfactory = mock.MagicMock() + mfactory_class = mocker.patch('pappyproxy.proxy.ProxyClientFactory') + mfactory_class.return_value = mfactory + + mocker.patch('pappyproxy.proxy.ProxyServer._make_remote_connection') + + mfactory.prepare_request.return_value = mock_deferred(None) + full_req = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'ABCD') + server_protocol.connection_id = 100 + + server_protocol.dataReceived(full_req) + # Make sure we created a ClientFactory with the right arguments + f_args, f_kwargs = mfactory_class.call_args + assert len(f_args) == 1 + + # Make sure the request got to the client class + req = f_args[0] + assert req.full_message == full_req -def test_proxy_server_fixture(unconnected_proxyserver): - unconnected_proxyserver.transport.write('hello') - assert unconnected_proxyserver.transport.getOutBuffer() == 'hello' + # Make sure the correct settings got to the proxy + assert f_kwargs['stream_response'] == True + assert f_kwargs['save_all'] == True + + # Make sure we initialized the client factory + assert mfactory.prepare_request.called + assert mfactory.connection_id == 100 + assert server_protocol._make_remote_connection.called # should be immediately called because mock deferred + +def test_proxy_server_no_streaming_with_int_macros(mocker): + mfactory = mock.MagicMock() + mfactory_class = mocker.patch('pappyproxy.proxy.ProxyClientFactory') + mfactory_class.return_value = mfactory + + mocker.patch('pappyproxy.proxy.ProxyServer._make_remote_connection') + + mfactory.prepare_request.return_value = mock_deferred(None) + full_req = ('POST /fooo HTTP/1.1\r\n' + 'Test-Header: foo\r\n' + 'Content-Length: 4\r\n' + '\r\n' + 'ABCD') + + int_macros = [{'mockmacro': mock_int_macro(modified_req='GET / HTTP/1.1\r\n\r\n')}] + server_protocol = gen_server_protocol(int_macros=int_macros) + server_protocol.dataReceived(full_req) + f_args, f_kwargs = mfactory_class.call_args + assert f_kwargs['stream_response'] == False +## ProxyServer._make_remote_connection + @pytest.inlineCallbacks -def test_mock_deferreds(): - d = mock_deferred('Hello!') - r = yield d - assert r == 'Hello!' +def test_proxy_server_make_tcp_connection(mocker, server_protocol): + mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") + mtcpe_class.return_value = mtcpe = mock.MagicMock() + mtcpe.connect.return_value = mock_deferred() -def test_deleted(): - with pytest.raises(NotImplementedError): - reactor.connectTCP("www.google.com", "80", ServerFactory) - with pytest.raises(NotImplementedError): - reactor.connectSSL("www.google.com", "80", ServerFactory) + server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly + + req = http.Request("GET / HTTP/1.1\r\n\r\n") + req.host = 'Foo.Bar.Brazzers' + req.port = 80085 + server_protocol._request_obj = req + + yield server_protocol._make_remote_connection(req) + targs, tkwargs = mtcpe_class.call_args + assert targs[1] == 'Foo.Bar.Brazzers' + assert targs[2] == 80085 + assert tkwargs == {} + mtcpe.connect.assert_called_once_with(server_protocol._client_factory) + +@pytest.inlineCallbacks +def test_proxy_server_make_ssl_connection(mocker, server_protocol): + mssle_class = mocker.patch("twisted.internet.endpoints.SSL4ClientEndpoint") + mssle_class.return_value = mssle = mock.MagicMock() + mssle.connect.return_value = mock_deferred() + + server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly + + req = http.Request("GET / HTTP/1.1\r\n\r\n", is_ssl=True) + req.host = 'Foo.Bar.Brazzers' + req.port = 80085 + server_protocol._request_obj = req + + yield server_protocol._make_remote_connection(req) + targs, tkwargs = mssle_class.call_args + assert targs[1] == 'Foo.Bar.Brazzers' + assert targs[2] == 80085 + assert tkwargs == {} + mssle.connect.assert_called_once_with(server_protocol._client_factory) + +@pytest.inlineCallbacks +def test_proxy_server_make_tcp_connection_socks(mocker): + socks_config(mocker, {'host': '12345', 'port': 5555}) + + tls_wrap_class = mocker.patch("txsocksx.tls.TLSWrapClientEndpoint") -#################### -## Proxy Server Tests - -def test_proxy_server_connect(unconnected_proxyserver, mocker, in_scope_true): - mocker.patch("twisted.internet.reactor.connectSSL") - unconnected_proxyserver.lineReceived('CONNECT https://www.dddddd.fff:433 HTTP/1.1') - unconnected_proxyserver.lineReceived('') - assert unconnected_proxyserver.transport.getOutBuffer() == 'HTTP/1.1 200 Connection established\r\n\r\n' - assert unconnected_proxyserver._request_obj.is_ssl + mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") + mtcpe_class.return_value = mtcpe = mock.MagicMock() + + socks_class = mocker.patch("txsocksx.client.SOCKS5ClientEndpoint") + socks_class.return_value = sockse = mock.MagicMock() + + server_protocol = gen_server_protocol() + server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly + + req = http.Request("GET / HTTP/1.1\r\n\r\n") + req.host = 'Foo.Bar.Brazzers' + req.port = 80085 + server_protocol._request_obj = req + + yield server_protocol._make_remote_connection(req) + sargs, skwargs = socks_class.call_args + targs, tkwargs = mtcpe_class.call_args + assert targs[1] == '12345' + assert targs[2] == 5555 + assert sargs[0] == 'Foo.Bar.Brazzers' + assert sargs[1] == 80085 + assert sargs[2] == mtcpe + assert skwargs == {'methods': {'anonymous': ()}} + assert not tls_wrap_class.called + sockse.connect.assert_called_once_with(server_protocol._client_factory) + +@pytest.inlineCallbacks +def test_proxy_server_make_ssl_connection_socks(mocker): + socks_config(mocker, {'host': '12345', 'port': 5555}) + + tls_wrap_class = mocker.patch("txsocksx.tls.TLSWrapClientEndpoint") + tls_wrape = tls_wrap_class.return_value = mock.MagicMock() + + mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") + mtcpe_class.return_value = mtcpe = mock.MagicMock() + + socks_class = mocker.patch("txsocksx.client.SOCKS5ClientEndpoint") + socks_class.return_value = sockse = mock.MagicMock() + + server_protocol = gen_server_protocol() + server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly + + req = http.Request("GET / HTTP/1.1\r\n\r\n") + req.host = 'Foo.Bar.Brazzers' + req.port = 80085 + req.is_ssl = True + server_protocol._request_obj = req + + yield server_protocol._make_remote_connection(req) + sargs, skwargs = socks_class.call_args + targs, tkwargs = mtcpe_class.call_args + assert targs[1] == '12345' + assert targs[2] == 5555 + assert sargs[0] == 'Foo.Bar.Brazzers' + assert sargs[1] == 80085 + assert sargs[2] == mtcpe + assert skwargs == {'methods': {'anonymous': ()}} + assert not sockse.called + tls_wrape.connect.assert_called_once_with(server_protocol._client_factory) + +@pytest.inlineCallbacks +def test_proxy_server_make_ssl_connection_socks_username_only(mocker): + socks_config(mocker, {'host': '12345', 'port': 5555, 'username': 'foo'}) + + tls_wrap_class = mocker.patch("txsocksx.tls.TLSWrapClientEndpoint") + tls_wrape = tls_wrap_class.return_value = mock.MagicMock() -def test_proxy_server_basic(proxyserver, mocker, in_scope_true): - mocker.patch("twisted.internet.reactor.connectSSL") - mocker.patch('pappyproxy.proxy.ProxyServer.setRawMode') - proxyserver.lineReceived('GET / HTTP/1.1') - proxyserver.lineReceived('') - - assert proxyserver.setRawMode.called - args, kwargs = twisted.internet.reactor.connectSSL.call_args - assert args[0] == 'www.AAAA.BBBB' - assert args[1] == 443 + mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") + mtcpe_class.return_value = mtcpe = mock.MagicMock() + + socks_class = mocker.patch("txsocksx.client.SOCKS5ClientEndpoint") + socks_class.return_value = sockse = mock.MagicMock() + + server_protocol = gen_server_protocol() + server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly + + req = http.Request("GET / HTTP/1.1\r\n\r\n") + req.host = 'Foo.Bar.Brazzers' + req.port = 80085 + req.is_ssl = True + server_protocol._request_obj = req + + yield server_protocol._make_remote_connection(req) + sargs, skwargs = socks_class.call_args + targs, tkwargs = mtcpe_class.call_args + assert targs[1] == '12345' + assert targs[2] == 5555 + assert sargs[0] == 'Foo.Bar.Brazzers' + assert sargs[1] == 80085 + assert sargs[2] == mtcpe + assert skwargs == {'methods': {'anonymous': ()}} + assert not sockse.called + tls_wrape.connect.assert_called_once_with(server_protocol._client_factory) + +@pytest.inlineCallbacks +def test_proxy_server_make_ssl_connection_socks_username_password(mocker): + socks_config(mocker, {'host': '12345', 'port': 5555, 'username': 'foo', 'password': 'password'}) + + tls_wrap_class = mocker.patch("txsocksx.tls.TLSWrapClientEndpoint") + tls_wrape = tls_wrap_class.return_value = mock.MagicMock() + mtcpe_class = mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint") + mtcpe_class.return_value = mtcpe = mock.MagicMock() + + socks_class = mocker.patch("txsocksx.client.SOCKS5ClientEndpoint") + socks_class.return_value = sockse = mock.MagicMock() + + server_protocol = gen_server_protocol() + server_protocol._client_factory = mock.MagicMock() # We already tested that this gets set up correctly + + req = http.Request("GET / HTTP/1.1\r\n\r\n") + req.host = 'Foo.Bar.Brazzers' + req.port = 80085 + req.is_ssl = True + server_protocol._request_obj = req + + yield server_protocol._make_remote_connection(req) + sargs, skwargs = socks_class.call_args + targs, tkwargs = mtcpe_class.call_args + assert targs[1] == '12345' + assert targs[2] == 5555 + assert sargs[0] == 'Foo.Bar.Brazzers' + assert sargs[1] == 80085 + assert sargs[2] == mtcpe + assert skwargs == {'methods': {'login': ('foo','password'), 'anonymous': ()}} + assert not sockse.called + tls_wrape.connect.assert_called_once_with(server_protocol._client_factory) + + +######################## +### Proxy Client Factory + @pytest.inlineCallbacks -def test_proxy_client_nomangle(mocker, proxy_connection, in_scope_true): - # Make the connection - (prot, sent, retreq_deferred) = \ - yield proxy_connection('GET / HTTP/1.1\r\n\r\n', None, None) - assert sent.full_request == 'GET / HTTP/1.1\r\n\r\n' - prot.lineReceived('HTTP/1.1 200 OK') - prot.lineReceived('Content-Length: 0') - prot.lineReceived('') - ret_req = yield retreq_deferred - response = ret_req.response.full_response - assert response == 'HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n' +def test_proxy_client_factory_prepare_reqs_simple(mocker, freeze): + import datetime + freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) + + req = http.Request('GET / HTTP/1.1\r\n\r\n') + + rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + rsave.return_value = mock_deferred() + mocker.patch('pappyproxy.context.in_scope').return_value = True + mocker.patch('pappyproxy.macros.mangle_request').return_value = mock_deferred((req, False)) + + cf = ProxyClientFactory(req, + save_all=False, + stream_response=False, + return_transport=None) + yield cf.prepare_request() + assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) + assert req.reqid is None + assert not rsave.called + assert len(rsave.mock_calls) == 0 @pytest.inlineCallbacks -def test_proxy_client_mangle_req(mocker, proxy_connection, in_scope_true): - # Make the connection - (prot, sent, retreq_deferred) = \ - yield proxy_connection('GET / HTTP/1.1\r\n\r\n', MANGLED_REQ, None) - assert sent.full_request == 'GET /mangled HTTP/1.1\r\n\r\n' +def test_proxy_client_factory_prepare_reqs_360_noscope(mocker, freeze): + import datetime + freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) + + req = http.Request('GET / HTTP/1.1\r\n\r\n') + + rsave = mocker.patch('pappyproxy.http.Request.async_deep_save') + rsave.return_value = mock_deferred() + mocker.patch('pappyproxy.context.in_scope').return_value = False + mocker.patch('pappyproxy.macros.mangle_request', new=func_deleted) + + cf = ProxyClientFactory(req, + save_all=True, + stream_response=False, + return_transport=None) + yield cf.prepare_request() + assert req.time_start == None + assert req.reqid is None + assert not rsave.called + assert len(rsave.mock_calls) == 0 @pytest.inlineCallbacks -def test_proxy_client_mangle_rsp(mocker, proxy_connection, in_scope_true): - # Make the connection - (prot, sent, retreq_deferred) = \ - yield proxy_connection('GET / HTTP/1.1\r\n\r\n', None, MANGLED_RSP) - prot.lineReceived('HTTP/1.1 200 OK') - prot.lineReceived('Content-Length: 0') - prot.lineReceived('') - req = yield retreq_deferred - response = req.response.full_response - assert response == 'HTTP/1.1 500 MANGLED\r\nContent-Length: 0\r\n\r\n' +def test_proxy_client_factory_prepare_reqs_save(mocker, freeze): + freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) + + req = http.Request('GET / HTTP/1.1\r\n\r\n') + + rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + mocker.patch('pappyproxy.context.in_scope').return_value = True + mocker.patch('pappyproxy.macros.mangle_request').return_value = mock_deferred((req, False)) + + cf = ProxyClientFactory(req, + save_all=True, + stream_response=False, + return_transport=None) + yield cf.prepare_request() + assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) + assert req.reqid is not None + assert rsave.called + assert len(rsave.mock_calls) == 1 @pytest.inlineCallbacks -def test_proxy_drop_req(mocker, proxy_connection, in_scope_true): - (prot, sent, retreq_deferred) = \ - yield proxy_connection('GET / HTTP/1.1\r\n\r\n', None, None, True, False) - assert sent is None +def test_proxy_client_factory_prepare_reqs_360_noscope_save(mocker, freeze): + freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) + + req = http.Request('GET / HTTP/1.1\r\n\r\n') + mangreq = http.Request('BOOO / HTTP/1.1\r\n\r\n') + + rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + mocker.patch('pappyproxy.context.in_scope').return_value = False + mocker.patch('pappyproxy.macros.mangle_request', side_effect=func_deleted) + + cf = ProxyClientFactory(req, + save_all=True, + stream_response=False, + return_transport=None) + yield cf.prepare_request() + assert req.time_start == None + assert req.reqid is None + assert not rsave.called + assert len(rsave.mock_calls) == 0 @pytest.inlineCallbacks -def test_proxy_drop_rsp(mocker, proxy_connection, in_scope_true): - (prot, sent, retreq_deferred) = \ - yield proxy_connection('GET / HTTP/1.1\r\n\r\n', None, None, False, True) - prot.lineReceived('HTTP/1.1 200 OK') - prot.lineReceived('Content-Length: 0') - prot.lineReceived('') - retreq = yield retreq_deferred - assert retreq.response is None +def test_proxy_client_factory_prepare_mangle_req(mocker, freeze): + + freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) + + req = http.Request('GET / HTTP/1.1\r\n\r\n') + mangreq = http.Request('BOOO / HTTP/1.1\r\n\r\n') + + def inc_day_mangle(x, y): + freeze.delta(days=1) + return mock_deferred((mangreq, True)) + + rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + mocker.patch('pappyproxy.context.in_scope').return_value = True + mocker.patch('pappyproxy.macros.mangle_request', side_effect=inc_day_mangle) + + cf = ProxyClientFactory(req, + save_all=True, + stream_response=False, + return_transport=None) + yield cf.prepare_request() + + assert cf.request == mangreq + assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) + assert cf.request.time_start == datetime.datetime(2015, 1, 2, 3, 30, 15, 50) + assert cf.request.reqid is not None + assert len(rsave.mock_calls) == 2 + +@pytest.inlineCallbacks +def test_proxy_client_factory_prepare_mangle_req_drop(mocker, freeze): + + freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) + + def inc_day_mangle(x, y): + freeze.delta(days=1) + return mock_deferred((None, True)) + + req = http.Request('GET / HTTP/1.1\r\n\r\n') + + rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + mocker.patch('pappyproxy.context.in_scope').return_value = True + mocker.patch('pappyproxy.macros.mangle_request', side_effect=inc_day_mangle) + + cf = ProxyClientFactory(req, + save_all=True, + stream_response=False, + return_transport=None) + yield cf.prepare_request() + + assert cf.request is None + assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) + assert len(rsave.mock_calls) == 1 + +@pytest.inlineCallbacks +def test_proxy_client_factory_prepare_mangle_req(mocker, freeze): + + freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) + + req = http.Request('GET / HTTP/1.1\r\n\r\n') + mangreq = http.Request('BOOO / HTTP/1.1\r\n\r\n') + + def inc_day_mangle(x, y): + freeze.delta(days=1) + return mock_deferred((mangreq, True)) + + rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + mocker.patch('pappyproxy.context.in_scope').return_value = True + mocker.patch('pappyproxy.macros.mangle_request', side_effect=inc_day_mangle) + + cf = ProxyClientFactory(req, + save_all=True, + stream_response=False, + return_transport=None) + yield cf.prepare_request() + + assert cf.request == mangreq + assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) + assert cf.request.time_start == datetime.datetime(2015, 1, 2, 3, 30, 15, 50) + assert cf.request.reqid is not None + assert len(rsave.mock_calls) == 2 + +### return_request_pair + +# @pytest.inlineCallbacks +# def test_proxy_client_factory_prepare_mangle_rsp(mocker, freeze): + +# freeze.freeze(datetime.datetime(2015, 1, 1, 3, 30, 15, 50)) +# rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) +# mocker.patch('pappyproxy.context.in_scope').return_value = True + +# req = http.Request('GET / HTTP/1.1\r\n\r\n') +# req.reqid = 1 +# rsp = http.Response('HTTP/1.1 200 OK\r\n\r\n') +# req.response = rsp + +# mocker.patch('pappyproxy.macros.mangle_response').return_value = (req, False) + +# cf = ProxyClientFactory(req, +# save_all=False, +# stream_response=False, +# return_transport=None) +# result = yield cf.return_request_pair(req) +# assert result == req +# assert req.time_start == datetime.datetime(2015, 1, 1, 3, 30, 15, 50) +# assert len(rsave.mock_calls) == 0 + + +### ProxyClient tests + +@pytest.inlineCallbacks +def test_proxy_client_simple(mocker): + rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + req = http.Request('GET / HTTP/1.1\r\n\r\n') + client = gen_client_protocol(req, stream_response=False) + assert client.transport.value() == 'GET / HTTP/1.1\r\n\r\n' + client.transport.clear() + rsp = 'HTTP/1.1 200 OKILE DOKELY\r\n\r\n' + client.dataReceived(rsp) + retpair = yield client.data_defer + assert retpair.response.full_message == rsp + @pytest.inlineCallbacks -def test_proxy_client_360_noscope(mocker, proxy_connection, in_scope_false): - # Make the connection - (prot, sent, retreq_deferred) = yield proxy_connection('GET / HTTP/1.1\r\n\r\n') - assert sent.full_request == 'GET / HTTP/1.1\r\n\r\n' - prot.lineReceived('HTTP/1.1 200 OK') - prot.lineReceived('Content-Length: 0') - prot.lineReceived('') - req = yield retreq_deferred - assert req.response.full_response == 'HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n' +def test_proxy_client_stream(mocker): + rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + req = http.Request('GET / HTTP/1.1\r\n\r\n') + client = gen_client_protocol(req, stream_response=True) + client.transport.clear() + client.dataReceived('HTTP/1.1 404 GET FUCKE') + assert client.factory.return_transport.value() == 'HTTP/1.1 404 GET FUCKE' + client.factory.return_transport.clear() + client.dataReceived('D ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD') + assert client.factory.return_transport.value() == 'D ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD' + retpair = yield client.data_defer + assert retpair.response.full_message == 'HTTP/1.1 404 GET FUCKED ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD' + + +@pytest.inlineCallbacks +def test_proxy_client_nostream(mocker): + rsave = mocker.patch.object(pappyproxy.http.Request, 'async_deep_save', autospec=True, side_effect=mock_req_async_save) + req = http.Request('GET / HTTP/1.1\r\n\r\n') + client = gen_client_protocol(req, stream_response=False) + client.transport.clear() + client.dataReceived('HTTP/1.1 404 GET FUCKE') + assert client.factory.return_transport.value() == '' + client.factory.return_transport.clear() + client.dataReceived('D ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD') + assert client.factory.return_transport.value() == '' + retpair = yield client.data_defer + assert retpair.response.full_message == 'HTTP/1.1 404 GET FUCKED ASSHOLE\r\nContent-Length: 4\r\n\r\nABCD' + diff --git a/pappyproxy/tests/testutil.py b/pappyproxy/tests/testutil.py index bffd342..7a9fa35 100644 --- a/pappyproxy/tests/testutil.py +++ b/pappyproxy/tests/testutil.py @@ -3,12 +3,19 @@ import mock import pytest import StringIO from twisted.internet import defer +from twisted.test.proto_helpers import StringTransport +from pappyproxy import http next_mock_id = 0 class ClassDeleted(): pass +class TLSStringTransport(StringTransport): + + def startTLS(self, context, factory): + pass + def func_deleted(*args, **kwargs): raise NotImplementedError() @@ -18,7 +25,7 @@ def func_ignored(*args, **kwargs): def func_ignored_deferred(*args, **kwargs): return mock_deferred(None) -def mock_deferred(value): +def mock_deferred(value=None): # Generates a function that can be used to make a deferred that can be used # to mock out deferred-returning responses def g(data): @@ -33,6 +40,10 @@ def no_tcp(mocker): # Don't make tcp connections mocker.patch("twisted.internet.reactor.connectTCP", new=func_deleted) mocker.patch("twisted.internet.reactor.connectSSL", new=func_deleted) + mocker.patch("twisted.internet.endpoints.SSL4ClientEndpoint", new=func_deleted) + mocker.patch("twisted.internet.endpoints.TCP4ClientEndpoint", new=func_deleted) + mocker.patch("txsocksx.client.SOCKS5ClientEndpoint", new=func_deleted) + mocker.patch("txsocksx.tls.TLSWrapClientEndpoint", new=func_deleted) @pytest.fixture def ignore_tcp(mocker): @@ -73,3 +84,71 @@ def mock_deep_save(mocker, fake_saving): def print_fuck(*args, **kwargs): print 'fuck' +@pytest.fixture +def freeze(monkeypatch): + """ Now() manager patches datetime return a fixed, settable, value + (freezes time) + stolen from http://stackoverflow.com/a/28073449 + """ + import datetime + original = datetime.datetime + + class FreezeMeta(type): + def __instancecheck__(self, instance): + if type(instance) == original or type(instance) == Freeze: + return True + + class Freeze(datetime.datetime): + __metaclass__ = FreezeMeta + + @classmethod + def freeze(cls, val, utcval=None): + cls.utcfrozen = utcval + cls.frozen = val + + @classmethod + def now(cls): + return cls.frozen + + @classmethod + def utcnow(cls): + # added since requests use utcnow + return cls.utcfrozen or cls.frozen + + @classmethod + def delta(cls, timedelta=None, **kwargs): + """ Moves time fwd/bwd by the delta""" + from datetime import timedelta as td + if not timedelta: + timedelta = td(**kwargs) + cls.frozen += timedelta + + monkeypatch.setattr(datetime, 'datetime', Freeze) + Freeze.freeze(original.now()) + return Freeze + +def mock_int_macro(modified_req=None, modified_rsp=None, + drop_req=False, drop_rsp=False): + macro = mock.MagicMock() + if modified_req or drop_req: + macro.async_req = True + macro.intercept_requests = True + if drop_req: + newreq = None + else: + newreq = http.Request(modified_req) + macro.async_mangle_request.return_value = mock_deferred(newreq) + else: + macro.intercept_requests = False + + if modified_rsp or drop_rsp: + macro.async_rsp = True + macro.intercept_responses = True + if drop_rsp: + newrsp = None + else: + newrsp = http.Response(modified_rsp) + macro.async_mangle_response.return_value = mock_deferred(newrsp) + else: + macro.intercept_responses = False + return macro diff --git a/pappyproxy/util.py b/pappyproxy/util.py index 5638bc3..b1ae177 100644 --- a/pappyproxy/util.py +++ b/pappyproxy/util.py @@ -3,6 +3,8 @@ import string import time import datetime +from .colors import Colors, Styles + class PappyException(Exception): """ The exception class for Pappy. If a plugin command raises one of these, the @@ -19,10 +21,17 @@ def printable_data(data): :rtype: String """ chars = [] + colored = False for c in data: if c in string.printable: + if colored: + chars.append(Colors.ENDC) + colored = False chars.append(c) else: + if not colored: + chars.append(Styles.UNPRINTABLE_DATA) + colored = True chars.append('.') return ''.join(chars) @@ -43,6 +52,6 @@ def hexdump(src, length=16): for c in xrange(0, len(src), length): chars = src[c:c+length] hex = ' '.join(["%02x" % ord(x) for x in chars]) - printable = ''.join(["%s" % ((ord(x) <= 127 and FILTER[ord(x)]) or '.') for x in chars]) + printable = ''.join(["%s" % ((ord(x) <= 127 and FILTER[ord(x)]) or Styles.UNPRINTABLE_DATA+'.'+Colors.ENDC) for x in chars]) lines.append("%04x %-*s %s\n" % (c, length*3, hex, printable)) return ''.join(lines) diff --git a/setup.py b/setup.py index 6498da5..3d9b583 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import pkgutil from setuptools import setup, find_packages -VERSION = '0.2.6' +VERSION = '0.2.7' setup(name='pappyproxy', version=VERSION, @@ -33,6 +33,7 @@ setup(name='pappyproxy', 'pytest>=2.8.3', 'service_identity>=14.0.0', 'twisted>=15.4.0', + 'txsocksx>=1.15.0.2' ], classifiers=[ 'Intended Audience :: Developers',