diff --git a/README.md b/README.md index e2a8aac..d9b2469 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,48 @@ Pappy also includes some built in filters that you can apply. These are things t |:--------|:--------|:------------| | `fbi ` | `builtin_filter`, `fbi` | Apply a built-in filter to the current context | +Decoding Strings +---------------- +These features try to fill a similar role to Burp's decoder. Each command will automatically copy the results to the clipboard. In addition, if no string is given, the commands will encode/decode whatever is already in the clipboard. Here is an example of how to base64 encode/decode a string. + +``` +pappy> b64e "Hello World!" +SGVsbG8gV29ybGQh +pappy> b64d +Hello World! +pappy> +``` + +And if the result contains non-printable characters, a hexdump will be produced instead + +``` +pappy> b64d ImALittleTeapot= +0000 22 60 0b 8a db 65 79 37 9a a6 8b "`...ey7... + +pappy> +``` + +The following commands can be used to encode/decode strings: + +| Command | Aliases | Description | +|:--------|:--------|:------------| +|`base64_decode`|`base64_decode`, `b64d` | Base64 decode a string | +|`base64_encode`|`base64_encode`, `b64e` | Base64 encode a string | +|`asciihex_decode`|`asciihex_decode`, `ahd` | Decode an ASCII hex string | +|`asciihex_encode`|`asciihex_encode`, `ahe` | Encode an ASCII hex string | +|`url_decode`|`url_decode`, `urld` | Url decode a string | +|`url_encode`|`url_encode`, `urle` | Url encode a string | +|`gzip_decode`|`gzip_decode`, `gzd` | Gzip decompress a string. Probably won't work too well since there's not a great way to get binary data passed in as an argument. I'm working on this. | +|`gzip_encode`|`gzip_encode`, `gze` | Gzip compress a string. Result doesn't get copied to the clipboard. | +|`base64_decode_raw`|`base64_decode_raw`, `b64dr` | Same as `base64_decode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`base64_encode_raw`|`base64_encode_raw`, `b64er` | Same as `base64_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`asciihex_decode_raw`|`asciihex_decode_raw`, `ahdr` | Same as `asciihex_decode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`asciihex_encode_raw`|`asciihex_encode_raw`, `aher` | Same as `asciihex_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`url_decode_raw`|`url_decode_raw`, `urldr` | Same as `url_decode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`url_encode_raw`|`url_encode_raw`, `urler` | Same as `url_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`gzip_decode_raw`|`gzip_decode_raw`, `gzdr` | Same as `gzip_decode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | +|`gzip_encode_raw`|`gzip_encode_raw`, `gzer` | Same as `gzip_encode` but will not print a hexdump if it contains non-printable characters. It is suggested you use `>` to redirect the output to a file. | + Interceptor ----------- This feature is like Burp's proxy with "Intercept Mode" turned on, except it's not turned on unless you explicitly turn it on. When the proxy gets a request while in intercept mode, it lets you edit it before forwarding it to the server. In addition, it can stop responses from the server and let you edit them before they get forwarded to the browser. When you run the command, you can pass `req` and/or `rsp` as arguments to say whether you would like to intercept requests and/or responses. Only in-scope requests/responses will be intercepted (see Scope section). @@ -654,6 +696,7 @@ This is a list of other random stuff you can do that isn't categorized under any |:--------|:--------|:------------| | `dump_response [filename]` | `dump_response` | Dumps the data from the response to the given filename (useful for images, .swf, etc). If no filename is given, it uses the name given in the path. | | `export ` | `export` | Writes either the full request or response to a file in the current directory. | +| `merge ` | `merge` | Add all the requests from another datafile to the current datafile | ### Response streaming @@ -786,6 +829,10 @@ Changelog --------- The boring part of the readme +* 0.2.3 + * Decoder functions + * Add `merge` command + * Bugfixes * 0.2.2 * COLORS * Performance improvements diff --git a/docs/source/conf.py b/docs/source/conf.py index 402221f..d83fdf2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -59,9 +59,9 @@ author = u'Rob Glew' # built documents. # # The short X.Y version. -version = u'0.2.2' +version = u'0.2.3' # The full version, including alpha/beta/rc tags. -release = u'0.2.2' +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index b9919e6..3240fd1 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -33,8 +33,6 @@ Anyways, here's some ideas for things you could implement: Find a library to perform some kind of check for weak ciphers, etc on a host and print out any issues that are found. * Add a SQLMap button Make it easy to pass a request to SQLMap to check for SQLi. Make sure you can configure which fields you do/don't want tested and by default just give either "yes it looks like SQLi" or "no it doesn't look like SQLi" -* Decoder functionality - Add some commands for encoding/decoding text. If you go after this, let me know because I'm probably going to be pretty picky about how this is implemented. You'll have to do better than just a ``base64_decode `` command. * Additional macro templates Write some commands for generating additional types of macros. For example let people generate an intercepting macro that does search/replace or modifies a header. Save as much typing as possible for common actions. * Show requests/responses real-time as they go through the proxy diff --git a/pappyproxy/console.py b/pappyproxy/console.py index 16e5c86..2698b88 100644 --- a/pappyproxy/console.py +++ b/pappyproxy/console.py @@ -150,7 +150,7 @@ def print_requests(requests): rows = [] for req in requests: rows.append(get_req_data_row(req)) - print_table(cols, rows) + print_request_rows(rows) def print_request_rows(request_rows): """ diff --git a/pappyproxy/http.py b/pappyproxy/http.py index 6ba488d..09ea1a5 100644 --- a/pappyproxy/http.py +++ b/pappyproxy/http.py @@ -558,9 +558,6 @@ class HTTPMessage(object): retmsg.set_metadata(self.get_metadata()) return retmsg - def __deepcopy__(self): - return self.__copy__() - def copy(self): """ Returns a copy of the request @@ -569,6 +566,12 @@ class HTTPMessage(object): """ return self.__copy__() + def deepcopy(self): + """ + Returns a deep copy of the message. Implemented by child. + """ + return self.__deepcopy__() + def clear(self): """ Resets all internal data and clears the message @@ -973,6 +976,20 @@ class Request(HTTPMessage): if host: self._host = host + def __copy__(self): + if not self.complete: + raise PappyException("Cannot copy incomplete http messages") + retreq = self.__class__(self.full_message) + retreq.set_metadata(self.get_metadata()) + retreq.time_start = self.time_start + retreq.time_end = self.time_end + retreq.reqid = None + if self.response: + retreq.response = self.response.copy() + if self.unmangled: + retreq.unmangled = self.unmangled.copy() + return retreq + @property def rsptime(self): """ @@ -1425,7 +1442,7 @@ class Request(HTTPMessage): ## Data store functions @defer.inlineCallbacks - def async_save(self): + def async_save(self, cust_dbpool=None, cust_cache=None): """ async_save() Save/update the request in the data file. Returns a twisted deferred which @@ -1436,7 +1453,15 @@ class Request(HTTPMessage): from .context import Context from .pappy import main_context - assert(dbpool) + global dbpool + if cust_dbpool: + use_dbpool = cust_dbpool + use_cache = cust_cache + else: + use_dbpool = dbpool + use_cache = Request.cache + + assert(use_dbpool) if not self.reqid: self.reqid = '--' try: @@ -1444,15 +1469,16 @@ class Request(HTTPMessage): _ = int(self.reqid) # If we have reqid, we're updating - yield dbpool.runInteraction(self._update) + yield use_dbpool.runInteraction(self._update) assert(self.reqid is not None) - yield dbpool.runInteraction(self._update_tags) + yield use_dbpool.runInteraction(self._update_tags) except (ValueError, TypeError): # Either no id or in-memory - yield dbpool.runInteraction(self._insert) + yield use_dbpool.runInteraction(self._insert) assert(self.reqid is not None) - yield dbpool.runInteraction(self._update_tags) - Request.cache.add(self) + yield use_dbpool.runInteraction(self._update_tags) + if use_cache: + use_cache.add(self) main_context.cache_reset() @crochet.wait_for(timeout=180.0) @@ -1544,10 +1570,10 @@ class Request(HTTPMessage): queryargs.append(self.unmangled.reqid) if self.time_start: setnames.append('start_datetime=?') - queryargs.append(time.mktime(self.time_start.timetuple())) + queryargs.append((self.time_start-datetime.datetime(1970,1,1)).total_seconds()) if self.time_end: setnames.append('end_datetime=?') - queryargs.append(time.mktime(self.time_end.timetuple())) + queryargs.append((self.time_end-datetime.datetime(1970,1,1)).total_seconds()) setnames.append('is_ssl=?') if self.is_ssl: @@ -1593,10 +1619,10 @@ class Request(HTTPMessage): colvals.append(self.unmangled.reqid) if self.time_start: colnames.append('start_datetime') - colvals.append(time.mktime(self.time_start.timetuple())) + colvals.append((self.time_start-datetime.datetime(1970,1,1)).total_seconds()) if self.time_end: colnames.append('end_datetime') - colvals.append(time.mktime(self.time_end.timetuple())) + colvals.append((self.time_end-datetime.datetime(1970,1,1)).total_seconds()) colnames.append('submitted') if self.submitted: colvals.append('1') @@ -1632,31 +1658,41 @@ class Request(HTTPMessage): assert self.reqid is not None @defer.inlineCallbacks - def delete(self): + def delete(self, cust_dbpool=None, cust_cache=None): from .context import Context, reset_context_caches + global dbpool + if cust_dbpool: + use_dbpool = cust_dbpool + use_cache = cust_cache + else: + use_dbpool = dbpool + use_cache = Request.cache + if self.reqid is None: raise PappyException("Cannot delete request with id=None") - self.cache.evict(self.reqid) - RequestCache.ordered_ids.remove(self.reqid) - RequestCache.all_ids.remove(self.reqid) - if self.reqid in RequestCache.req_times: - del RequestCache.req_times[self.reqid] - if self.reqid in RequestCache.inmem_reqs: - RequestCache.inmem_reqs.remove(self.reqid) - if self.reqid in RequestCache.unmangled_ids: - RequestCache.unmangled_ids.remove(self.reqid) + + if use_cache: + use_cache.evict(self.reqid) + Request.cache.ordered_ids.remove(self.reqid) + Request.cache.all_ids.remove(self.reqid) + if self.reqid in Request.cache.req_times: + del Request.cache.req_times[self.reqid] + if self.reqid in Request.cache.inmem_reqs: + Request.cache.inmem_reqs.remove(self.reqid) + if self.reqid in Request.cache.unmangled_ids: + Request.cache.unmangled_ids.remove(self.reqid) reset_context_caches() if self.reqid[0] != 'm': - yield dbpool.runQuery( + yield use_dbpool.runQuery( """ DELETE FROM requests WHERE id=?; """, (self.reqid,) ) - yield dbpool.runQuery( + yield use_dbpool.runQuery( """ DELETE FROM tagged WHERE reqid=?; """, @@ -1693,21 +1729,33 @@ class Request(HTTPMessage): @staticmethod @defer.inlineCallbacks - def _from_sql_row(row): + def _from_sql_row(row, cust_dbpool=None, cust_cache=None): from .http import Request + global dbpool + if cust_dbpool: + use_dbpool = cust_dbpool + use_cache = cust_cache + else: + use_dbpool = dbpool + use_cache = Request.cache + req = Request(row[0]) if row[1]: - rsp = yield Response.load_response(str(row[1])) + rsp = yield Response.load_response(str(row[1]), + cust_dbpool=cust_dbpool, + cust_cache=cust_cache) req.response = rsp if row[3]: - unmangled_req = yield Request.load_request(str(row[3])) + unmangled_req = yield Request.load_request(str(row[3]), + cust_dbpool=cust_dbpool, + cust_cache=cust_cache) req.unmangled = unmangled_req req.unmangled.is_unmangled_version = True if row[4]: - req.time_start = datetime.datetime.fromtimestamp(row[4]) + req.time_start = datetime.datetime.utcfromtimestamp(row[4]) if row[5]: - req.time_end = datetime.datetime.fromtimestamp(row[5]) + req.time_end = datetime.datetime.utcfromtimestamp(row[5]) if row[6] is not None: req.port = int(row[6]) if row[7] == 1: @@ -1719,7 +1767,7 @@ class Request(HTTPMessage): req.reqid = str(row[2]) # tags - rows = yield dbpool.runQuery( + rows = yield use_dbpool.runQuery( """ SELECT tg.tag FROM tagged tgd, tags tg @@ -1734,7 +1782,7 @@ class Request(HTTPMessage): @staticmethod @defer.inlineCallbacks - def load_requests_by_time(first, num): + def load_requests_by_time(first, num, cust_dbpool=None, cust_cache=None): """ load_requests_by_time() Load all the requests in the data file and return them in a list. @@ -1745,23 +1793,42 @@ class Request(HTTPMessage): from .requestcache import RequestCache from .http import Request - starttime = RequestCache.req_times[first] - rows = yield dbpool.runQuery( - """ - SELECT %s - FROM requests - WHERE start_datetime<=? ORDER BY start_datetime desc LIMIT ?; - """ % Request._gen_sql_row(), (starttime, num) - ) + global dbpool + if cust_dbpool: + use_dbpool = cust_dbpool + use_cache = cust_cache + else: + use_dbpool = dbpool + use_cache = Request.cache + + if use_cache: + starttime = use_cache.req_times[first] + rows = yield use_dbpool.runQuery( + """ + SELECT %s + FROM requests + WHERE start_datetime<=? ORDER BY start_datetime desc LIMIT ?; + """ % Request._gen_sql_row(), (starttime, num) + ) + else: + rows = yield use_dbpool.runQuery( + """ + SELECT %s + FROM requests r1, requests r2 + WHERE r2.id=? AND + r1.start_datetime<=r2.start_datetime + ORDER BY start_datetime desc LIMIT ?; + """ % Request._gen_sql_row('r1'), (first, num) + ) reqs = [] for row in rows: - req = yield Request._from_sql_row(row) + req = yield Request._from_sql_row(row, cust_dbpool=cust_dbpool, cust_cache=cust_cache) reqs.append(req) defer.returnValue(reqs) @staticmethod @defer.inlineCallbacks - def load_requests_by_tag(tag): + def load_requests_by_tag(tag, cust_dbpool=None, cust_cache=None): """ load_requests_by_tag(tag) Load all the requests in the data file with a given tag and return them in a list. @@ -1770,8 +1837,17 @@ class Request(HTTPMessage): :rtype: twisted.internet.defer.Deferred """ from .http import Request + + global dbpool + if cust_dbpool: + use_dbpool = cust_dbpool + use_cache = cust_cache + else: + use_dbpool = dbpool + use_cache = Request.cache + # tags - rows = yield dbpool.runQuery( + rows = yield use_dbpool.runQuery( """ SELECT tgd.reqid FROM tagged tgd, tags tg @@ -1781,13 +1857,15 @@ class Request(HTTPMessage): ) reqs = [] for row in rows: - req = Request.load_request(row[0]) + req = Request.load_request(row[0], + cust_dbpool=cust_dbpool, + cust_cache=cust_cache) reqs.append(req) defer.returnValue(reqs) @staticmethod @defer.inlineCallbacks - def load_request(to_load, allow_special=True, use_cache=True): + def load_request(to_load, allow_special=True, use_cache=True, cust_dbpool=None, cust_cache=None): """ load_request(to_load) Load a request with the given request id and return it. @@ -1802,7 +1880,15 @@ class Request(HTTPMessage): """ from .context import Context - if not dbpool: + global dbpool + if cust_dbpool: + use_dbpool = cust_dbpool + cache_to_use = cust_cache + else: + use_dbpool = dbpool + cache_to_use = Request.cache + + if not use_dbpool: raise PappyException('No database connection to load from') if to_load == '--': @@ -1841,14 +1927,14 @@ class Request(HTTPMessage): return r # Get it through the cache - if use_cache: + if use_cache and cache_to_use: # If it's not cached, load_request will be called again and be told # not to use the cache. - r = yield Request.cache.get(loadid) + r = yield cache_to_use.get(loadid) defer.returnValue(retreq(r)) # Load it from the data file - rows = yield dbpool.runQuery( + rows = yield use_dbpool.runQuery( """ SELECT %s FROM requests @@ -1858,9 +1944,10 @@ class Request(HTTPMessage): ) if len(rows) != 1: raise PappyException("Request with id %s does not exist" % loadid) - req = yield Request._from_sql_row(rows[0]) + req = yield Request._from_sql_row(rows[0], cust_dbpool=cust_dbpool, cust_cache=cust_cache) assert req.reqid == loadid - Request.cache.add(req) + if cache_to_use: + cache_to_use.add(req) defer.returnValue(retreq(req)) ###################### @@ -1953,6 +2040,16 @@ class Response(HTTPMessage): # After message init so that other instance vars are initialized self._set_dict_callbacks() + def __copy__(self): + if not self.complete: + raise PappyException("Cannot copy incomplete http messages") + retrsp = self.__class__(self.full_message) + retrsp.set_metadata(self.get_metadata()) + retrsp.rspid = None + if self.unmangled: + retrsp.unmangled = self.unmangled.copy() + return retrsp + @property def raw_headers(self): """ @@ -2188,7 +2285,7 @@ class Response(HTTPMessage): ## Database interaction @defer.inlineCallbacks - def async_save(self): + def async_save(self, cust_dbpool=None, cust_cache=None): """ async_save() Save/update the just request in the data file. Returns a twisted deferred which @@ -2197,15 +2294,22 @@ class Response(HTTPMessage): :rtype: twisted.internet.defer.Deferred """ - assert(dbpool) + global dbpool + if cust_dbpool: + use_dbpool = cust_dbpool + use_cache = cust_cache + else: + use_dbpool = dbpool + use_cache = Request.cache + assert(use_dbpool) try: # Check for intyness _ = int(self.rspid) # If we have rspid, we're updating - yield dbpool.runInteraction(self._update) + yield use_dbpool.runInteraction(self._update) except (ValueError, TypeError): - yield dbpool.runInteraction(self._insert) + yield use_dbpool.runInteraction(self._insert) assert(self.rspid is not None) # Right now responses without requests are unviewable @@ -2246,7 +2350,7 @@ class Response(HTTPMessage): """ % (','.join(colnames), ','.join(['?']*len(colvals))), tuple(colvals) ) - self.rspid = txn.lastrowid + self.rspid = str(txn.lastrowid) assert(self.rspid is not None) @defer.inlineCallbacks @@ -2262,14 +2366,22 @@ class Response(HTTPMessage): @staticmethod @defer.inlineCallbacks - def load_response(respid): + def load_response(respid, cust_dbpool=None, cust_cache=None): """ Load a response from its response id. Returns a deferred. I don't suggest you use this. :rtype: twisted.internet.defer.Deferred """ - assert(dbpool) - rows = yield dbpool.runQuery( + global dbpool + if cust_dbpool: + use_dbpool = cust_dbpool + use_cache = cust_cache + else: + use_dbpool = dbpool + use_cache = Request.cache + + assert(use_dbpool) + rows = yield use_dbpool.runQuery( """ SELECT full_response, id, unmangled_id FROM responses @@ -2283,7 +2395,9 @@ class Response(HTTPMessage): resp = Response(full_response) resp.rspid = str(rows[0][1]) if rows[0][2]: - unmangled_response = yield Response.load_response(int(rows[0][2])) + unmangled_response = yield Response.load_response(int(rows[0][2]), + cust_dbpool=cust_dbpool, + cust_cache=cust_cache) resp.unmangled = unmangled_response defer.returnValue(resp) diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index 118c5b3..da20eea 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -67,7 +67,6 @@ def main(): global plugin_loader global cons settings = parse_args() - load_start = datetime.datetime.now() if settings['lite']: conf_settings = config.get_default_config() @@ -100,7 +99,7 @@ def main(): print 'Exiting...' reactor.stop() http.init(dbpool) - yield requestcache.RequestCache.load_ids() + yield http.Request.cache.load_ids() context.reset_context_caches() # Run the proxy @@ -136,13 +135,6 @@ def main(): yield context.load_scope(http.dbpool) context.reset_to_scope(main_context) - # Apologize for slow start times - load_end = datetime.datetime.now() - load_time = (load_end - load_start) - if load_time.total_seconds() > 20: - print 'Startup was slow (%s)! Sorry!' % load_time - print 'Database has {0} requests (~{1:.2f}ms per request)'.format(len(main_context.active_requests), ((load_time.total_seconds()/len(main_context.active_requests))*1000)) - sys.argv = [sys.argv[0]] # cmd2 tries to parse args cons = ProxyCmd() plugin_loader = plugin.PluginLoader(cons) diff --git a/pappyproxy/plugins/decode.py b/pappyproxy/plugins/decode.py new file mode 100644 index 0000000..d690d0d --- /dev/null +++ b/pappyproxy/plugins/decode.py @@ -0,0 +1,230 @@ +import StringIO +import base64 +import clipboard +import gzip +import shlex +import string +import urllib + +from pappyproxy.util import PappyException, hexdump + +def print_maybe_bin(s): + binary = False + for c in s: + if c not in string.printable: + binary = True + break + if binary: + print hexdump(s) + else: + print s + +def asciihex_encode_helper(s): + return ''.join('{0:x}'.format(ord(c)) for c in s) + +def asciihex_decode_helper(s): + ret = [] + try: + for a, b in zip(s[0::2], s[1::2]): + c = a+b + ret.append(chr(int(c, 16))) + return ''.join(ret) + except Exception as e: + raise PappyException(e) + +def gzip_encode_helper(s): + out = StringIO.StringIO() + with gzip.GzipFile(fileobj=out, mode="w") as f: + f.write(s) + return out.getvalue() + +def gzip_decode_helper(s): + dec_data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(s)) + dec_data = dec_data.read() + return dec_data + +def _code_helper(line, func, copy=True): + args = shlex.split(line) + if not args: + s = clipboard.paste() + s = func(s) + if copy: + try: + clipboard.copy(s) + except: + print 'Result cannot be copied to the clipboard. Result not copied.' + return s + else: + s = func(args[0].strip()) + if copy: + try: + clipboard.copy(s) + except: + print 'Result cannot be copied to the clipboard. Result not copied.' + return s + +def base64_decode(line): + """ + Base64 decode a string. + If no string is given, will decode the contents of the clipboard. + Results are copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, base64.b64decode)) + +def base64_encode(line): + """ + Base64 encode a string. + If no string is given, will encode the contents of the clipboard. + Results are copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, base64.b64encode)) + +def url_decode(line): + """ + URL decode a string. + If no string is given, will decode the contents of the clipboard. + Results are copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, urllib.unquote)) + +def url_encode(line): + """ + URL encode special characters in a string. + If no string is given, will encode the contents of the clipboard. + Results are copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, urllib.quote_plus)) + +def asciihex_decode(line): + """ + Decode an ascii hex string. + If no string is given, will decode the contents of the clipboard. + Results are copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, asciihex_decode_helper)) + +def asciihex_encode(line): + """ + Convert all the characters in a line to hex and combine them. + If no string is given, will encode the contents of the clipboard. + Results are copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, asciihex_encode_helper)) + +def gzip_decode(line): + """ + Un-gzip a string. + If no string is given, will decompress the contents of the clipboard. + Results are copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, gzip_decode_helper)) + +def gzip_encode(line): + """ + Gzip a string. + If no string is given, will decompress the contents of the clipboard. + Results are NOT copied to the clipboard. + """ + print_maybe_bin(_code_helper(line, gzip_encode_helper, copy=False)) + +def base64_decode_raw(line): + """ + Same as base64_decode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, base64.b64decode, copy=False) + +def base64_encode_raw(line): + """ + Same as base64_encode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, base64.b64encode, copy=False) + +def url_decode_raw(line): + """ + Same as url_decode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, urllib.unquote, copy=False) + +def url_encode_raw(line): + """ + Same as url_encode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, urllib.quote_plus, copy=False) + +def asciihex_decode_raw(line): + """ + Same as asciihex_decode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, asciihex_decode_helper, copy=False) + +def asciihex_encode_raw(line): + """ + Same as asciihex_encode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, asciihex_encode_helper, copy=False) + +def gzip_decode_raw(line): + """ + Same as gzip_decode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, gzip_decode_helper, copy=False) + +def gzip_encode_raw(line): + """ + Same as gzip_encode but the output will never be printed as a hex dump and + results will not be copied. It is suggested you redirect the output + to a file. + """ + print _code_helper(line, gzip_encode_helper, copy=False) + +def load_cmds(cmd): + cmd.set_cmds({ + 'base64_decode': (base64_decode, None), + 'base64_encode': (base64_encode, None), + 'asciihex_decode': (asciihex_decode, None), + 'asciihex_encode': (asciihex_encode, None), + 'url_decode': (url_decode, None), + 'url_encode': (url_encode, None), + 'gzip_decode': (gzip_decode, None), + 'gzip_encode': (gzip_encode, None), + 'base64_decode_raw': (base64_decode_raw, None), + 'base64_encode_raw': (base64_encode_raw, None), + 'asciihex_decode_raw': (asciihex_decode_raw, None), + 'asciihex_encode_raw': (asciihex_encode_raw, None), + 'url_decode_raw': (url_decode_raw, None), + 'url_encode_raw': (url_encode_raw, None), + 'gzip_decode_raw': (gzip_decode_raw, None), + 'gzip_encode_raw': (gzip_encode_raw, None), + }) + cmd.add_aliases([ + ('base64_decode', 'b64d'), + ('base64_encode', 'b64e'), + ('asciihex_decode', 'ahd'), + ('asciihex_encode', 'ahe'), + ('url_decode', 'urld'), + ('url_encode', 'urle'), + ('gzip_decode', 'gzd'), + ('gzip_encode', 'gze'), + ('base64_decode_raw', 'b64dr'), + ('base64_encode_raw', 'b64er'), + ('asciihex_decode_raw', 'ahdr'), + ('asciihex_encode_raw', 'aher'), + ('url_decode_raw', 'urldr'), + ('url_encode_raw', 'urler'), + ('gzip_decode_raw', 'gzdr'), + ('gzip_encode_raw', 'gzer'), + ]) diff --git a/pappyproxy/plugins/filter.py b/pappyproxy/plugins/filter.py index f6f765d..e3d9baf 100644 --- a/pappyproxy/plugins/filter.py +++ b/pappyproxy/plugins/filter.py @@ -143,7 +143,6 @@ def filter_prune(line): CANNOT BE UNDONE!! Be careful! Usage: filter_prune """ - from pappyproxy.requestcache import RequestCache # Delete filtered items from datafile print '' print 'Currently active filters:' @@ -152,8 +151,8 @@ def filter_prune(line): # We copy so that we're not removing items from a set we're iterating over act_reqs = yield pappyproxy.pappy.main_context.get_reqs() - inact_reqs = RequestCache.all_ids.difference(set(act_reqs)) - inact_reqs = inact_reqs.difference(set(RequestCache.unmangled_ids)) + inact_reqs = Request.cache.all_ids.difference(set(act_reqs)) + inact_reqs = inact_reqs.difference(set(Request.cache.unmangled_ids)) message = 'This will delete %d/%d requests. You can NOT undo this!! Continue?' % (len(inact_reqs), (len(inact_reqs) + len(act_reqs))) if not confirm(message, 'n'): defer.returnValue(None) diff --git a/pappyproxy/plugins/misc.py b/pappyproxy/plugins/misc.py index d6ba58d..fb2b3af 100644 --- a/pappyproxy/plugins/misc.py +++ b/pappyproxy/plugins/misc.py @@ -5,7 +5,9 @@ import shlex from pappyproxy.console import confirm, load_reqlist from pappyproxy.util import PappyException from pappyproxy.http import Request +from pappyproxy.requestcache import RequestCache from twisted.internet import defer +from twisted.enterprise import adbapi @crochet.wait_for(timeout=None) @defer.inlineCallbacks @@ -14,7 +16,7 @@ def clrmem(line): Delete all in-memory only requests Usage: clrmem """ - to_delete = list(pappyproxy.requestcache.RequestCache.inmem_reqs) + to_delete = list(pappyproxy.http.Request.cache.inmem_reqs) for r in to_delete: yield r.deep_delete() @@ -85,6 +87,34 @@ def export(line): except PappyException as e: print 'Unable to export %s: %s' % (req.reqid, e) +@crochet.wait_for(timeout=None) +@defer.inlineCallbacks +def merge_datafile(line): + """ + Add all the requests/responses from another data file to the current one + """ + + def set_text_factory(conn): + conn.text_factory = str + + line = line.strip() + other_dbpool = adbapi.ConnectionPool("sqlite3", line, + check_same_thread=False, + cp_openfun=set_text_factory, + cp_max=1) + try: + count = 0 + other_cache = RequestCache(cust_dbpool=other_dbpool) + yield other_cache.load_ids() + for req_d in other_cache.req_it(): + count += 1 + req = yield req_d + r = req.copy() + yield r.async_deep_save() + print 'Added %d requests' % count + finally: + other_dbpool.close() + def load_cmds(cmd): cmd.set_cmds({ 'clrmem': (clrmem, None), @@ -92,6 +122,7 @@ def load_cmds(cmd): 'sv': (save, None), 'export': (export, None), 'log': (log, None), + 'merge': (merge_datafile, None) }) cmd.add_aliases([ #('rpy', ''), diff --git a/pappyproxy/plugins/view.py b/pappyproxy/plugins/view.py index fb3c295..1ec7cf9 100644 --- a/pappyproxy/plugins/view.py +++ b/pappyproxy/plugins/view.py @@ -4,7 +4,7 @@ import pappyproxy import shlex from pappyproxy.console import load_reqlist, print_table, print_request_rows, get_req_data_row -from pappyproxy.util import PappyException +from pappyproxy.util import PappyException, utc2local from pappyproxy.http import Request from twisted.internet import defer from pappyproxy.plugin import main_context_ids @@ -57,7 +57,8 @@ def print_request_extended(request): is_ssl = 'NO' if request.time_start: - time_made_str = request.time_start.strftime('%a, %b %d, %Y, %I:%M:%S %p') + dtobj = utc2local(request.time_start) + time_made_str = dtobj.strftime('%a, %b %d, %Y, %I:%M:%S %p') else: time_made_str = '--' diff --git a/pappyproxy/proxy.py b/pappyproxy/proxy.py index f680cfa..95be86f 100644 --- a/pappyproxy/proxy.py +++ b/pappyproxy/proxy.py @@ -128,7 +128,7 @@ class ProxyClient(LineReceiver): if self.factory.save_all: # It isn't the actual time, but this should work in case # we do an 'ls' before it gets a real time saved - self.request.time_start = datetime.datetime.now() + self.request.time_start = datetime.datetime.utcnow() if self.factory.stream_response and not to_mangle: self.request.async_deep_save() else: @@ -157,13 +157,13 @@ class ProxyClient(LineReceiver): if sendreq != self.request: sendreq.unmangled = self.request if self.factory.save_all: - sendreq.time_start = datetime.datetime.now() + sendreq.time_start = datetime.datetime.utcnow() yield sendreq.async_deep_save() else: self.log("Request out of scope, passing along unmangled") if not self._sent: - self.factory.start_time = datetime.datetime.now() + self.factory.start_time = datetime.datetime.utcnow() self.transport.write(sendreq.full_request) self.request = sendreq self.request.submitted = True @@ -190,7 +190,7 @@ class ProxyClientFactory(ClientFactory): self.request = request self.connection_id = -1 self.data_defer = defer.Deferred() - self.start_time = datetime.datetime.now() + self.start_time = datetime.datetime.utcnow() self.end_time = None self.save_all = save_all self.stream_response = stream_response @@ -213,7 +213,7 @@ class ProxyClientFactory(ClientFactory): @defer.inlineCallbacks def return_request_pair(self, request): - self.end_time = datetime.datetime.now() + self.end_time = datetime.datetime.utcnow() log_request(printable_data(request.response.full_response), id=self.connection_id, symbol='= 100: RequestCache._preload_limit = int(cache_size * 0.30) @@ -33,6 +25,14 @@ class RequestCache(object): self._min_time = None self.hits = 0 self.misses = 0 + self.dbpool = cust_dbpool + self._next_in_mem_id = 1 + self._preload_limit = 10 + self.all_ids = set() + self.unmangled_ids = set() + self.ordered_ids = SortedCollection(key=lambda x: -self.req_times[x]) + self.inmem_reqs = set() + self.req_times = {} @property def hit_ratio(self): @@ -40,37 +40,37 @@ class RequestCache(object): return 0 return float(self.hits)/float(self.hits + self.misses) - @staticmethod - def get_memid(): - i = 'm%d' % RequestCache._next_in_mem_id - RequestCache._next_in_mem_id += 1 + def get_memid(self): + i = 'm%d' % self._next_in_mem_id + self._next_in_mem_id += 1 return i - @staticmethod @defer.inlineCallbacks - def load_ids(): - rows = yield pappyproxy.http.dbpool.runQuery( + def load_ids(self): + if not self.dbpool: + self.dbpool = pappyproxy.http.dbpool + rows = yield self.dbpool.runQuery( """ SELECT id, start_datetime FROM requests; """ ) for row in rows: if row[1]: - RequestCache.req_times[str(row[0])] = row[1] + self.req_times[str(row[0])] = row[1] else: - RequestCache.req_times[str(row[0])] = 0 - if str(row[0]) not in RequestCache.all_ids: - RequestCache.ordered_ids.insert(str(row[0])) - RequestCache.all_ids.add(str(row[0])) + self.req_times[str(row[0])] = 0 + if str(row[0]) not in self.all_ids: + self.ordered_ids.insert(str(row[0])) + self.all_ids.add(str(row[0])) - rows = yield pappyproxy.http.dbpool.runQuery( + rows = yield self.dbpool.runQuery( """ SELECT unmangled_id FROM requests WHERE unmangled_id is NOT NULL; """ ) for row in rows: - RequestCache.unmangled_ids.add(str(row[0])) + self.unmangled_ids.add(str(row[0])) def resize(self, size): if size >= self._cache_size or size == -1: @@ -107,7 +107,7 @@ class RequestCache(object): Add a request to the cache """ if not req.reqid: - req.reqid = RequestCache.get_memid() + req.reqid = self.get_memid() if req.reqid[0] == 'm': self.inmem_reqs.add(req) if req.is_unmangled_version: @@ -116,10 +116,10 @@ class RequestCache(object): self.unmangled_ids.add(req.unmangled.reqid) self._cached_reqs[req.reqid] = req self._update_last_used(req.reqid) - RequestCache.req_times[req.reqid] = req.sort_time - if req.reqid not in RequestCache.all_ids: - RequestCache.ordered_ids.insert(req.reqid) - RequestCache.all_ids.add(req.reqid) + self.req_times[req.reqid] = req.sort_time + if req.reqid not in self.all_ids: + self.ordered_ids.insert(req.reqid) + self.all_ids.add(req.reqid) if len(self._cached_reqs) > self._cache_size and self._cache_size != -1: self._evict_single() @@ -142,7 +142,7 @@ class RequestCache(object): """ Load a number of requests after an id into the cache """ - reqs = yield pappyproxy.http.Request.load_requests_by_time(first, num) + reqs = yield pappyproxy.http.Request.load_requests_by_time(first, num, cust_dbpool=self.dbpool, cust_cache=self) for r in reqs: self.add(r) # Bulk loading is faster, so let's just say that loading 10 requests is @@ -162,22 +162,22 @@ class RequestCache(object): req = yield self.get(reqid) defer.returnValue(req) - over = list(RequestCache.ordered_ids) + over = list(self.ordered_ids) for reqid in over: if ids is not None and reqid not in ids: continue - if not include_unmangled and reqid in RequestCache.unmangled_ids: + if not include_unmangled and reqid in self.unmangled_ids: continue do_load = True - if reqid in RequestCache.all_ids: - if count % RequestCache._preload_limit == 0: + if reqid in self.all_ids: + if count % self._preload_limit == 0: do_load = True if do_load and not self.check(reqid): do_load = False - if (num - count) < RequestCache._preload_limit and num != -1: + if (num - count) < self._preload_limit and num != -1: loadnum = num - count else: - loadnum = RequestCache._preload_limit + loadnum = self._preload_limit yield def_wrapper(reqid, load=True, num=loadnum) else: yield def_wrapper(reqid) @@ -187,7 +187,7 @@ class RequestCache(object): @defer.inlineCallbacks def load_by_tag(tag): - reqs = yield load_requests_by_tag(tag) + reqs = yield load_requests_by_tag(tag, cust_cache=self, cust_dbpool=self.dbpool) for req in reqs: self.add(req) defer.returnValue(reqs) diff --git a/pappyproxy/tests/test_requestcache.py b/pappyproxy/tests/test_requestcache.py index 376e37d..ac6625f 100644 --- a/pappyproxy/tests/test_requestcache.py +++ b/pappyproxy/tests/test_requestcache.py @@ -106,7 +106,7 @@ def test_cache_inmem_evict(): assert cache.check(reqs[3].reqid) # Testing the implementation - assert reqs[0] in RequestCache.inmem_reqs - assert reqs[1] in RequestCache.inmem_reqs - assert reqs[2] in RequestCache.inmem_reqs - assert reqs[3] in RequestCache.inmem_reqs + assert reqs[0] in cache.inmem_reqs + assert reqs[1] in cache.inmem_reqs + assert reqs[2] in cache.inmem_reqs + assert reqs[3] in cache.inmem_reqs diff --git a/pappyproxy/util.py b/pappyproxy/util.py index ecdcec1..5c61da6 100644 --- a/pappyproxy/util.py +++ b/pappyproxy/util.py @@ -1,4 +1,6 @@ import string +import time +import datetime class PappyException(Exception): """ @@ -22,3 +24,20 @@ def printable_data(data): else: chars += '.' return ''.join(chars) + +# Taken from http://stackoverflow.com/questions/4770297/python-convert-utc-datetime-string-to-local-datetime +def utc2local(utc): + epoch = time.mktime(utc.timetuple()) + offset = datetime.datetime.fromtimestamp(epoch) - datetime.datetime.utcfromtimestamp(epoch) + return utc + offset + +# Taken from https://gist.github.com/sbz/1080258 +def hexdump(src, length=16): + FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.' for x in range(256)]) + lines = [] + for c in xrange(0, len(src), length): + chars = src[c:c+length] + hex = ' '.join(["%02x" % ord(x) for x in chars]) + printable = ''.join(["%s" % ((ord(x) <= 127 and FILTER[ord(x)]) or '.') for x in chars]) + lines.append("%04x %-*s %s\n" % (c, length*3, hex, printable)) + return ''.join(lines) diff --git a/setup.py b/setup.py index c642db7..5ed3943 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import pkgutil from setuptools import setup, find_packages -VERSION = '0.2.2' +VERSION = '0.2.3' setup(name='pappyproxy', version=VERSION, @@ -22,6 +22,7 @@ setup(name='pappyproxy', download_url='https://github.com/roglew/pappy-proxy/archive/%s.tar.gz'%VERSION, install_requires=[ 'beautifulsoup4>=4.4.1', + 'clipboard>=0.0.4', 'cmd2>=0.6.8', 'crochet>=1.4.0', 'Jinja2>=2.8',