jab2870/PappyProxy
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Version 0.2.3

Browse source
This commit is contained in:
Rob Glew 2016-01-27 16:51:40 -06:00
parent 28b7b7e8ff
commit d805eabeec
15 changed files with 559 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

2
pappyproxy/console.py
View file

@ -150,7 +150,7 @@ def print_requests(requests):
rows = []
for req in requests:
rows.append(get_req_data_row(req))
print_table(cols, rows)
print_request_rows(rows)
def print_request_rows(request_rows):
"""

236
pappyproxy/http.py
View file

@ -558,9 +558,6 @@ class HTTPMessage(object):
retmsg.set_metadata(self.get_metadata())
return retmsg
def __deepcopy__(self):
return self.__copy__()
def copy(self):
"""
Returns a copy of the request
@ -569,6 +566,12 @@ class HTTPMessage(object):
"""
return self.__copy__()
def deepcopy(self):
"""
Returns a deep copy of the message. Implemented by child.
"""
return self.__deepcopy__()
def clear(self):
"""
Resets all internal data and clears the message
@ -973,6 +976,20 @@ class Request(HTTPMessage):
if host:
self._host = host
def __copy__(self):
if not self.complete:
raise PappyException("Cannot copy incomplete http messages")
retreq = self.__class__(self.full_message)
retreq.set_metadata(self.get_metadata())
retreq.time_start = self.time_start
retreq.time_end = self.time_end
retreq.reqid = None
if self.response:
retreq.response = self.response.copy()
if self.unmangled:
retreq.unmangled = self.unmangled.copy()
return retreq
@property
def rsptime(self):
"""
@ -1425,7 +1442,7 @@ class Request(HTTPMessage):
## Data store functions
@defer.inlineCallbacks
def async_save(self):
def async_save(self, cust_dbpool=None, cust_cache=None):
"""
async_save()
Save/update the request in the data file. Returns a twisted deferred which
@ -1436,7 +1453,15 @@ class Request(HTTPMessage):
from .context import Context
from .pappy import main_context
assert(dbpool)
global dbpool
if cust_dbpool:
use_dbpool = cust_dbpool
use_cache = cust_cache
else:
use_dbpool = dbpool
use_cache = Request.cache
assert(use_dbpool)
if not self.reqid:
self.reqid = '--'
try:
@ -1444,15 +1469,16 @@ class Request(HTTPMessage):
_ = int(self.reqid)
# If we have reqid, we're updating
yield dbpool.runInteraction(self._update)
yield use_dbpool.runInteraction(self._update)
assert(self.reqid is not None)
yield dbpool.runInteraction(self._update_tags)
yield use_dbpool.runInteraction(self._update_tags)
except (ValueError, TypeError):
# Either no id or in-memory
yield dbpool.runInteraction(self._insert)
yield use_dbpool.runInteraction(self._insert)
assert(self.reqid is not None)
yield dbpool.runInteraction(self._update_tags)
Request.cache.add(self)
yield use_dbpool.runInteraction(self._update_tags)
if use_cache:
use_cache.add(self)
main_context.cache_reset()
@crochet.wait_for(timeout=180.0)
@ -1544,10 +1570,10 @@ class Request(HTTPMessage):
queryargs.append(self.unmangled.reqid)
if self.time_start:
setnames.append('start_datetime=?')
queryargs.append(time.mktime(self.time_start.timetuple()))
queryargs.append((self.time_start-datetime.datetime(1970,1,1)).total_seconds())
if self.time_end:
setnames.append('end_datetime=?')
queryargs.append(time.mktime(self.time_end.timetuple()))
queryargs.append((self.time_end-datetime.datetime(1970,1,1)).total_seconds())
setnames.append('is_ssl=?')
if self.is_ssl:
@ -1593,10 +1619,10 @@ class Request(HTTPMessage):
colvals.append(self.unmangled.reqid)
if self.time_start:
colnames.append('start_datetime')
colvals.append(time.mktime(self.time_start.timetuple()))
colvals.append((self.time_start-datetime.datetime(1970,1,1)).total_seconds())
if self.time_end:
colnames.append('end_datetime')
colvals.append(time.mktime(self.time_end.timetuple()))
colvals.append((self.time_end-datetime.datetime(1970,1,1)).total_seconds())
colnames.append('submitted')
if self.submitted:
colvals.append('1')
@ -1632,31 +1658,41 @@ class Request(HTTPMessage):
assert self.reqid is not None
@defer.inlineCallbacks
def delete(self):
def delete(self, cust_dbpool=None, cust_cache=None):
from .context import Context, reset_context_caches
global dbpool
if cust_dbpool:
use_dbpool = cust_dbpool
use_cache = cust_cache
else:
use_dbpool = dbpool
use_cache = Request.cache
if self.reqid is None:
raise PappyException("Cannot delete request with id=None")
self.cache.evict(self.reqid)
RequestCache.ordered_ids.remove(self.reqid)
RequestCache.all_ids.remove(self.reqid)
if self.reqid in RequestCache.req_times:
del RequestCache.req_times[self.reqid]
if self.reqid in RequestCache.inmem_reqs:
RequestCache.inmem_reqs.remove(self.reqid)
if self.reqid in RequestCache.unmangled_ids:
RequestCache.unmangled_ids.remove(self.reqid)
if use_cache:
use_cache.evict(self.reqid)
Request.cache.ordered_ids.remove(self.reqid)
Request.cache.all_ids.remove(self.reqid)
if self.reqid in Request.cache.req_times:
del Request.cache.req_times[self.reqid]
if self.reqid in Request.cache.inmem_reqs:
Request.cache.inmem_reqs.remove(self.reqid)
if self.reqid in Request.cache.unmangled_ids:
Request.cache.unmangled_ids.remove(self.reqid)
reset_context_caches()
if self.reqid[0] != 'm':
yield dbpool.runQuery(
yield use_dbpool.runQuery(
"""
DELETE FROM requests WHERE id=?;
""",
(self.reqid,)
)
yield dbpool.runQuery(
yield use_dbpool.runQuery(
"""
DELETE FROM tagged WHERE reqid=?;
""",
@ -1693,21 +1729,33 @@ class Request(HTTPMessage):
@staticmethod
@defer.inlineCallbacks
def _from_sql_row(row):
def _from_sql_row(row, cust_dbpool=None, cust_cache=None):
from .http import Request
global dbpool
if cust_dbpool:
use_dbpool = cust_dbpool
use_cache = cust_cache
else:
use_dbpool = dbpool
use_cache = Request.cache
req = Request(row[0])
if row[1]:
rsp = yield Response.load_response(str(row[1]))
rsp = yield Response.load_response(str(row[1]),
cust_dbpool=cust_dbpool,
cust_cache=cust_cache)
req.response = rsp
if row[3]:
unmangled_req = yield Request.load_request(str(row[3]))
unmangled_req = yield Request.load_request(str(row[3]),
cust_dbpool=cust_dbpool,
cust_cache=cust_cache)
req.unmangled = unmangled_req
req.unmangled.is_unmangled_version = True
if row[4]:
req.time_start = datetime.datetime.fromtimestamp(row[4])
req.time_start = datetime.datetime.utcfromtimestamp(row[4])
if row[5]:
req.time_end = datetime.datetime.fromtimestamp(row[5])
req.time_end = datetime.datetime.utcfromtimestamp(row[5])
if row[6] is not None:
req.port = int(row[6])
if row[7] == 1:
@ -1719,7 +1767,7 @@ class Request(HTTPMessage):
req.reqid = str(row[2])
# tags
rows = yield dbpool.runQuery(
rows = yield use_dbpool.runQuery(
"""
SELECT tg.tag
FROM tagged tgd, tags tg
@ -1734,7 +1782,7 @@ class Request(HTTPMessage):
@staticmethod
@defer.inlineCallbacks
def load_requests_by_time(first, num):
def load_requests_by_time(first, num, cust_dbpool=None, cust_cache=None):
"""
load_requests_by_time()
Load all the requests in the data file and return them in a list.
@ -1745,23 +1793,42 @@ class Request(HTTPMessage):
from .requestcache import RequestCache
from .http import Request
starttime = RequestCache.req_times[first]
rows = yield dbpool.runQuery(
"""
SELECT %s
FROM requests
WHERE start_datetime<=? ORDER BY start_datetime desc LIMIT ?;
""" % Request._gen_sql_row(), (starttime, num)
)
global dbpool
if cust_dbpool:
use_dbpool = cust_dbpool
use_cache = cust_cache
else:
use_dbpool = dbpool
use_cache = Request.cache
if use_cache:
starttime = use_cache.req_times[first]
rows = yield use_dbpool.runQuery(
"""
SELECT %s
FROM requests
WHERE start_datetime<=? ORDER BY start_datetime desc LIMIT ?;
""" % Request._gen_sql_row(), (starttime, num)
)
else:
rows = yield use_dbpool.runQuery(
"""
SELECT %s
FROM requests r1, requests r2
WHERE r2.id=? AND
r1.start_datetime<=r2.start_datetime
ORDER BY start_datetime desc LIMIT ?;
""" % Request._gen_sql_row('r1'), (first, num)
)
reqs = []
for row in rows:
req = yield Request._from_sql_row(row)
req = yield Request._from_sql_row(row, cust_dbpool=cust_dbpool, cust_cache=cust_cache)
reqs.append(req)
defer.returnValue(reqs)
@staticmethod
@defer.inlineCallbacks
def load_requests_by_tag(tag):
def load_requests_by_tag(tag, cust_dbpool=None, cust_cache=None):
"""
load_requests_by_tag(tag)
Load all the requests in the data file with a given tag and return them in a list.
@ -1770,8 +1837,17 @@ class Request(HTTPMessage):
:rtype: twisted.internet.defer.Deferred
"""
from .http import Request
global dbpool
if cust_dbpool:
use_dbpool = cust_dbpool
use_cache = cust_cache
else:
use_dbpool = dbpool
use_cache = Request.cache
# tags
rows = yield dbpool.runQuery(
rows = yield use_dbpool.runQuery(
"""
SELECT tgd.reqid
FROM tagged tgd, tags tg
@ -1781,13 +1857,15 @@ class Request(HTTPMessage):
)
reqs = []
for row in rows:
req = Request.load_request(row[0])
req = Request.load_request(row[0],
cust_dbpool=cust_dbpool,
cust_cache=cust_cache)
reqs.append(req)
defer.returnValue(reqs)
@staticmethod
@defer.inlineCallbacks
def load_request(to_load, allow_special=True, use_cache=True):
def load_request(to_load, allow_special=True, use_cache=True, cust_dbpool=None, cust_cache=None):
"""
load_request(to_load)
Load a request with the given request id and return it.
@ -1802,7 +1880,15 @@ class Request(HTTPMessage):
"""
from .context import Context
if not dbpool:
global dbpool
if cust_dbpool:
use_dbpool = cust_dbpool
cache_to_use = cust_cache
else:
use_dbpool = dbpool
cache_to_use = Request.cache
if not use_dbpool:
raise PappyException('No database connection to load from')
if to_load == '--':
@ -1841,14 +1927,14 @@ class Request(HTTPMessage):
return r
# Get it through the cache
if use_cache:
if use_cache and cache_to_use:
# If it's not cached, load_request will be called again and be told
# not to use the cache.
r = yield Request.cache.get(loadid)
r = yield cache_to_use.get(loadid)
defer.returnValue(retreq(r))
# Load it from the data file
rows = yield dbpool.runQuery(
rows = yield use_dbpool.runQuery(
"""
SELECT %s
FROM requests
@ -1858,9 +1944,10 @@ class Request(HTTPMessage):
)
if len(rows) != 1:
raise PappyException("Request with id %s does not exist" % loadid)
req = yield Request._from_sql_row(rows[0])
req = yield Request._from_sql_row(rows[0], cust_dbpool=cust_dbpool, cust_cache=cust_cache)
assert req.reqid == loadid
Request.cache.add(req)
if cache_to_use:
cache_to_use.add(req)
defer.returnValue(retreq(req))
######################
@ -1953,6 +2040,16 @@ class Response(HTTPMessage):
# After message init so that other instance vars are initialized
self._set_dict_callbacks()
def __copy__(self):
if not self.complete:
raise PappyException("Cannot copy incomplete http messages")
retrsp = self.__class__(self.full_message)
retrsp.set_metadata(self.get_metadata())
retrsp.rspid = None
if self.unmangled:
retrsp.unmangled = self.unmangled.copy()
return retrsp
@property
def raw_headers(self):
"""
@ -2188,7 +2285,7 @@ class Response(HTTPMessage):
## Database interaction
@defer.inlineCallbacks
def async_save(self):
def async_save(self, cust_dbpool=None, cust_cache=None):
"""
async_save()
Save/update the just request in the data file. Returns a twisted deferred which
@ -2197,15 +2294,22 @@ class Response(HTTPMessage):
:rtype: twisted.internet.defer.Deferred
"""
assert(dbpool)
global dbpool
if cust_dbpool:
use_dbpool = cust_dbpool
use_cache = cust_cache
else:
use_dbpool = dbpool
use_cache = Request.cache
assert(use_dbpool)
try:
# Check for intyness
_ = int(self.rspid)
# If we have rspid, we're updating
yield dbpool.runInteraction(self._update)
yield use_dbpool.runInteraction(self._update)
except (ValueError, TypeError):
yield dbpool.runInteraction(self._insert)
yield use_dbpool.runInteraction(self._insert)
assert(self.rspid is not None)
# Right now responses without requests are unviewable
@ -2246,7 +2350,7 @@ class Response(HTTPMessage):
""" % (','.join(colnames), ','.join(['?']*len(colvals))),
tuple(colvals)
)
self.rspid = txn.lastrowid
self.rspid = str(txn.lastrowid)
assert(self.rspid is not None)
@defer.inlineCallbacks
@ -2262,14 +2366,22 @@ class Response(HTTPMessage):
@staticmethod
@defer.inlineCallbacks
def load_response(respid):
def load_response(respid, cust_dbpool=None, cust_cache=None):
"""
Load a response from its response id. Returns a deferred. I don't suggest you use this.
:rtype: twisted.internet.defer.Deferred
"""
assert(dbpool)
rows = yield dbpool.runQuery(
global dbpool
if cust_dbpool:
use_dbpool = cust_dbpool
use_cache = cust_cache
else:
use_dbpool = dbpool
use_cache = Request.cache
assert(use_dbpool)
rows = yield use_dbpool.runQuery(
"""
SELECT full_response, id, unmangled_id
FROM responses
@ -2283,7 +2395,9 @@ class Response(HTTPMessage):
resp = Response(full_response)
resp.rspid = str(rows[0][1])
if rows[0][2]:
unmangled_response = yield Response.load_response(int(rows[0][2]))
unmangled_response = yield Response.load_response(int(rows[0][2]),
cust_dbpool=cust_dbpool,
cust_cache=cust_cache)
resp.unmangled = unmangled_response
defer.returnValue(resp)

10
pappyproxy/pappy.py
View file

@ -67,7 +67,6 @@ def main():
global plugin_loader
global cons
settings = parse_args()
load_start = datetime.datetime.now()
if settings['lite']:
conf_settings = config.get_default_config()
@ -100,7 +99,7 @@ def main():
print 'Exiting...'
reactor.stop()
http.init(dbpool)
yield requestcache.RequestCache.load_ids()
yield http.Request.cache.load_ids()
context.reset_context_caches()
# Run the proxy
@ -136,13 +135,6 @@ def main():
yield context.load_scope(http.dbpool)
context.reset_to_scope(main_context)
# Apologize for slow start times
load_end = datetime.datetime.now()
load_time = (load_end - load_start)
if load_time.total_seconds() > 20:
print 'Startup was slow (%s)! Sorry!' % load_time
print 'Database has {0} requests (~{1:.2f}ms per request)'.format(len(main_context.active_requests), ((load_time.total_seconds()/len(main_context.active_requests))*1000))
sys.argv = [sys.argv[0]] # cmd2 tries to parse args
cons = ProxyCmd()
plugin_loader = plugin.PluginLoader(cons)

230
pappyproxy/plugins/decode.py Normal file
View file

@ -0,0 +1,230 @@
import StringIO
import base64
import clipboard
import gzip
import shlex
import string
import urllib
from pappyproxy.util import PappyException, hexdump
def print_maybe_bin(s):
binary = False
for c in s:
if c not in string.printable:
binary = True
break
if binary:
print hexdump(s)
else:
print s
def asciihex_encode_helper(s):
return ''.join('{0:x}'.format(ord(c)) for c in s)
def asciihex_decode_helper(s):
ret = []
try:
for a, b in zip(s[0::2], s[1::2]):
c = a+b
ret.append(chr(int(c, 16)))
return ''.join(ret)
except Exception as e:
raise PappyException(e)
def gzip_encode_helper(s):
out = StringIO.StringIO()
with gzip.GzipFile(fileobj=out, mode="w") as f:
f.write(s)
return out.getvalue()
def gzip_decode_helper(s):
dec_data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(s))
dec_data = dec_data.read()
return dec_data
def _code_helper(line, func, copy=True):
args = shlex.split(line)
if not args:
s = clipboard.paste()
s = func(s)
if copy:
try:
clipboard.copy(s)
except:
print 'Result cannot be copied to the clipboard. Result not copied.'
return s
else:
s = func(args[0].strip())
if copy:
try:
clipboard.copy(s)
except:
print 'Result cannot be copied to the clipboard. Result not copied.'
return s
def base64_decode(line):
"""
Base64 decode a string.
If no string is given, will decode the contents of the clipboard.
Results are copied to the clipboard.
"""
print_maybe_bin(_code_helper(line, base64.b64decode))
def base64_encode(line):
"""
Base64 encode a string.
If no string is given, will encode the contents of the clipboard.
Results are copied to the clipboard.
"""
print_maybe_bin(_code_helper(line, base64.b64encode))
def url_decode(line):
"""
URL decode a string.
If no string is given, will decode the contents of the clipboard.
Results are copied to the clipboard.
"""
print_maybe_bin(_code_helper(line, urllib.unquote))
def url_encode(line):
"""
URL encode special characters in a string.
If no string is given, will encode the contents of the clipboard.
Results are copied to the clipboard.
"""
print_maybe_bin(_code_helper(line, urllib.quote_plus))
def asciihex_decode(line):
"""
Decode an ascii hex string.
If no string is given, will decode the contents of the clipboard.
Results are copied to the clipboard.
"""
print_maybe_bin(_code_helper(line, asciihex_decode_helper))
def asciihex_encode(line):
"""
Convert all the characters in a line to hex and combine them.
If no string is given, will encode the contents of the clipboard.
Results are copied to the clipboard.
"""
print_maybe_bin(_code_helper(line, asciihex_encode_helper))
def gzip_decode(line):
"""
Un-gzip a string.
If no string is given, will decompress the contents of the clipboard.
Results are copied to the clipboard.
"""
print_maybe_bin(_code_helper(line, gzip_decode_helper))
def gzip_encode(line):
"""
Gzip a string.
If no string is given, will decompress the contents of the clipboard.
Results are NOT copied to the clipboard.
"""
print_maybe_bin(_code_helper(line, gzip_encode_helper, copy=False))
def base64_decode_raw(line):
"""
Same as base64_decode but the output will never be printed as a hex dump and
results will not be copied. It is suggested you redirect the output
to a file.
"""
print _code_helper(line, base64.b64decode, copy=False)
def base64_encode_raw(line):
"""
Same as base64_encode but the output will never be printed as a hex dump and
results will not be copied. It is suggested you redirect the output
to a file.
"""
print _code_helper(line, base64.b64encode, copy=False)
def url_decode_raw(line):
"""
Same as url_decode but the output will never be printed as a hex dump and
results will not be copied. It is suggested you redirect the output
to a file.
"""
print _code_helper(line, urllib.unquote, copy=False)
def url_encode_raw(line):
"""
Same as url_encode but the output will never be printed as a hex dump and
results will not be copied. It is suggested you redirect the output
to a file.
"""
print _code_helper(line, urllib.quote_plus, copy=False)
def asciihex_decode_raw(line):
"""
Same as asciihex_decode but the output will never be printed as a hex dump and
results will not be copied. It is suggested you redirect the output
to a file.
"""
print _code_helper(line, asciihex_decode_helper, copy=False)
def asciihex_encode_raw(line):
"""
Same as asciihex_encode but the output will never be printed as a hex dump and
results will not be copied. It is suggested you redirect the output
to a file.
"""
print _code_helper(line, asciihex_encode_helper, copy=False)
def gzip_decode_raw(line):
"""
Same as gzip_decode but the output will never be printed as a hex dump and
results will not be copied. It is suggested you redirect the output
to a file.
"""
print _code_helper(line, gzip_decode_helper, copy=False)
def gzip_encode_raw(line):
"""
Same as gzip_encode but the output will never be printed as a hex dump and
results will not be copied. It is suggested you redirect the output
to a file.
"""
print _code_helper(line, gzip_encode_helper, copy=False)
def load_cmds(cmd):
cmd.set_cmds({
'base64_decode': (base64_decode, None),
'base64_encode': (base64_encode, None),
'asciihex_decode': (asciihex_decode, None),
'asciihex_encode': (asciihex_encode, None),
'url_decode': (url_decode, None),
'url_encode': (url_encode, None),
'gzip_decode': (gzip_decode, None),
'gzip_encode': (gzip_encode, None),
'base64_decode_raw': (base64_decode_raw, None),
'base64_encode_raw': (base64_encode_raw, None),
'asciihex_decode_raw': (asciihex_decode_raw, None),
'asciihex_encode_raw': (asciihex_encode_raw, None),
'url_decode_raw': (url_decode_raw, None),
'url_encode_raw': (url_encode_raw, None),
'gzip_decode_raw': (gzip_decode_raw, None),
'gzip_encode_raw': (gzip_encode_raw, None),
})
cmd.add_aliases([
('base64_decode', 'b64d'),
('base64_encode', 'b64e'),
('asciihex_decode', 'ahd'),
('asciihex_encode', 'ahe'),
('url_decode', 'urld'),
('url_encode', 'urle'),
('gzip_decode', 'gzd'),
('gzip_encode', 'gze'),
('base64_decode_raw', 'b64dr'),
('base64_encode_raw', 'b64er'),
('asciihex_decode_raw', 'ahdr'),
('asciihex_encode_raw', 'aher'),
('url_decode_raw', 'urldr'),
('url_encode_raw', 'urler'),
('gzip_decode_raw', 'gzdr'),
('gzip_encode_raw', 'gzer'),
])

5
pappyproxy/plugins/filter.py
View file

@ -143,7 +143,6 @@ def filter_prune(line):
CANNOT BE UNDONE!! Be careful!
Usage: filter_prune
"""
from pappyproxy.requestcache import RequestCache
# Delete filtered items from datafile
print ''
print 'Currently active filters:'
@ -152,8 +151,8 @@ def filter_prune(line):
# We copy so that we're not removing items from a set we're iterating over
act_reqs = yield pappyproxy.pappy.main_context.get_reqs()
inact_reqs = RequestCache.all_ids.difference(set(act_reqs))
inact_reqs = inact_reqs.difference(set(RequestCache.unmangled_ids))
inact_reqs = Request.cache.all_ids.difference(set(act_reqs))
inact_reqs = inact_reqs.difference(set(Request.cache.unmangled_ids))
message = 'This will delete %d/%d requests. You can NOT undo this!! Continue?' % (len(inact_reqs), (len(inact_reqs) + len(act_reqs)))
if not confirm(message, 'n'):
defer.returnValue(None)

33
pappyproxy/plugins/misc.py
View file

@ -5,7 +5,9 @@ import shlex
from pappyproxy.console import confirm, load_reqlist
from pappyproxy.util import PappyException
from pappyproxy.http import Request
from pappyproxy.requestcache import RequestCache
from twisted.internet import defer
from twisted.enterprise import adbapi
@crochet.wait_for(timeout=None)
@defer.inlineCallbacks
@ -14,7 +16,7 @@ def clrmem(line):
Delete all in-memory only requests
Usage: clrmem
"""
to_delete = list(pappyproxy.requestcache.RequestCache.inmem_reqs)
to_delete = list(pappyproxy.http.Request.cache.inmem_reqs)
for r in to_delete:
yield r.deep_delete()
@ -85,6 +87,34 @@ def export(line):
except PappyException as e:
print 'Unable to export %s: %s' % (req.reqid, e)
@crochet.wait_for(timeout=None)
@defer.inlineCallbacks
def merge_datafile(line):
"""
Add all the requests/responses from another data file to the current one
"""
def set_text_factory(conn):
conn.text_factory = str
line = line.strip()
other_dbpool = adbapi.ConnectionPool("sqlite3", line,
check_same_thread=False,
cp_openfun=set_text_factory,
cp_max=1)
try:
count = 0
other_cache = RequestCache(cust_dbpool=other_dbpool)
yield other_cache.load_ids()
for req_d in other_cache.req_it():
count += 1
req = yield req_d
r = req.copy()
yield r.async_deep_save()
print 'Added %d requests' % count
finally:
other_dbpool.close()
def load_cmds(cmd):
cmd.set_cmds({
'clrmem': (clrmem, None),
@ -92,6 +122,7 @@ def load_cmds(cmd):
'sv': (save, None),
'export': (export, None),
'log': (log, None),
'merge': (merge_datafile, None)
})
cmd.add_aliases([
#('rpy', ''),

5
pappyproxy/plugins/view.py
View file

@ -4,7 +4,7 @@ import pappyproxy
import shlex
from pappyproxy.console import load_reqlist, print_table, print_request_rows, get_req_data_row
from pappyproxy.util import PappyException
from pappyproxy.util import PappyException, utc2local
from pappyproxy.http import Request
from twisted.internet import defer
from pappyproxy.plugin import main_context_ids
@ -57,7 +57,8 @@ def print_request_extended(request):
is_ssl = 'NO'
if request.time_start:
time_made_str = request.time_start.strftime('%a, %b %d, %Y, %I:%M:%S %p')
dtobj = utc2local(request.time_start)
time_made_str = dtobj.strftime('%a, %b %d, %Y, %I:%M:%S %p')
else:
time_made_str = '--'

10
pappyproxy/proxy.py
View file

@ -128,7 +128,7 @@ class ProxyClient(LineReceiver):
if self.factory.save_all:
# It isn't the actual time, but this should work in case
# we do an 'ls' before it gets a real time saved
self.request.time_start = datetime.datetime.now()
self.request.time_start = datetime.datetime.utcnow()
if self.factory.stream_response and not to_mangle:
self.request.async_deep_save()
else:
@ -157,13 +157,13 @@ class ProxyClient(LineReceiver):
if sendreq != self.request:
sendreq.unmangled = self.request
if self.factory.save_all:
sendreq.time_start = datetime.datetime.now()
sendreq.time_start = datetime.datetime.utcnow()
yield sendreq.async_deep_save()
else:
self.log("Request out of scope, passing along unmangled")
if not self._sent:
self.factory.start_time = datetime.datetime.now()
self.factory.start_time = datetime.datetime.utcnow()
self.transport.write(sendreq.full_request)
self.request = sendreq
self.request.submitted = True
@ -190,7 +190,7 @@ class ProxyClientFactory(ClientFactory):
self.request = request
self.connection_id = -1
self.data_defer = defer.Deferred()
self.start_time = datetime.datetime.now()
self.start_time = datetime.datetime.utcnow()
self.end_time = None
self.save_all = save_all
self.stream_response = stream_response
@ -213,7 +213,7 @@ class ProxyClientFactory(ClientFactory):
@defer.inlineCallbacks
def return_request_pair(self, request):
self.end_time = datetime.datetime.now()
self.end_time = datetime.datetime.utcnow()
log_request(printable_data(request.response.full_response), id=self.connection_id, symbol='<m', verbosity_level=3)
request.time_start = self.start_time

72
pappyproxy/requestcache.py
View file

@ -16,15 +16,7 @@ class RequestCache(object):
:type cache_size: int
"""
_next_in_mem_id = 1
_preload_limit = 10
all_ids = set()
unmangled_ids = set()
ordered_ids = SortedCollection(key=lambda x: -RequestCache.req_times[x])
inmem_reqs = set()
req_times = {}
def __init__(self, cache_size=100):
def __init__(self, cache_size=100, cust_dbpool=None):
self._cache_size = cache_size
if cache_size >= 100:
RequestCache._preload_limit = int(cache_size * 0.30)
@ -33,6 +25,14 @@ class RequestCache(object):
self._min_time = None
self.hits = 0
self.misses = 0
self.dbpool = cust_dbpool
self._next_in_mem_id = 1
self._preload_limit = 10
self.all_ids = set()
self.unmangled_ids = set()
self.ordered_ids = SortedCollection(key=lambda x: -self.req_times[x])
self.inmem_reqs = set()
self.req_times = {}
@property
def hit_ratio(self):
@ -40,37 +40,37 @@ class RequestCache(object):
return 0
return float(self.hits)/float(self.hits + self.misses)
@staticmethod
def get_memid():
i = 'm%d' % RequestCache._next_in_mem_id
RequestCache._next_in_mem_id += 1
def get_memid(self):
i = 'm%d' % self._next_in_mem_id
self._next_in_mem_id += 1
return i
@staticmethod
@defer.inlineCallbacks
def load_ids():
rows = yield pappyproxy.http.dbpool.runQuery(
def load_ids(self):
if not self.dbpool:
self.dbpool = pappyproxy.http.dbpool
rows = yield self.dbpool.runQuery(
"""
SELECT id, start_datetime FROM requests;
"""
)
for row in rows:
if row[1]:
RequestCache.req_times[str(row[0])] = row[1]
self.req_times[str(row[0])] = row[1]
else:
RequestCache.req_times[str(row[0])] = 0
if str(row[0]) not in RequestCache.all_ids:
RequestCache.ordered_ids.insert(str(row[0]))
RequestCache.all_ids.add(str(row[0]))
self.req_times[str(row[0])] = 0
if str(row[0]) not in self.all_ids:
self.ordered_ids.insert(str(row[0]))
self.all_ids.add(str(row[0]))
rows = yield pappyproxy.http.dbpool.runQuery(
rows = yield self.dbpool.runQuery(
"""
SELECT unmangled_id FROM requests
WHERE unmangled_id is NOT NULL;
"""
)
for row in rows:
RequestCache.unmangled_ids.add(str(row[0]))
self.unmangled_ids.add(str(row[0]))
def resize(self, size):
if size >= self._cache_size or size == -1:
@ -107,7 +107,7 @@ class RequestCache(object):
Add a request to the cache
"""
if not req.reqid:
req.reqid = RequestCache.get_memid()
req.reqid = self.get_memid()
if req.reqid[0] == 'm':
self.inmem_reqs.add(req)
if req.is_unmangled_version:
@ -116,10 +116,10 @@ class RequestCache(object):
self.unmangled_ids.add(req.unmangled.reqid)
self._cached_reqs[req.reqid] = req
self._update_last_used(req.reqid)
RequestCache.req_times[req.reqid] = req.sort_time
if req.reqid not in RequestCache.all_ids:
RequestCache.ordered_ids.insert(req.reqid)
RequestCache.all_ids.add(req.reqid)
self.req_times[req.reqid] = req.sort_time
if req.reqid not in self.all_ids:
self.ordered_ids.insert(req.reqid)
self.all_ids.add(req.reqid)
if len(self._cached_reqs) > self._cache_size and self._cache_size != -1:
self._evict_single()
@ -142,7 +142,7 @@ class RequestCache(object):
"""
Load a number of requests after an id into the cache
"""
reqs = yield pappyproxy.http.Request.load_requests_by_time(first, num)
reqs = yield pappyproxy.http.Request.load_requests_by_time(first, num, cust_dbpool=self.dbpool, cust_cache=self)
for r in reqs:
self.add(r)
# Bulk loading is faster, so let's just say that loading 10 requests is
@ -162,22 +162,22 @@ class RequestCache(object):
req = yield self.get(reqid)
defer.returnValue(req)
over = list(RequestCache.ordered_ids)
over = list(self.ordered_ids)
for reqid in over:
if ids is not None and reqid not in ids:
continue
if not include_unmangled and reqid in RequestCache.unmangled_ids:
if not include_unmangled and reqid in self.unmangled_ids:
continue
do_load = True
if reqid in RequestCache.all_ids:
if count % RequestCache._preload_limit == 0:
if reqid in self.all_ids:
if count % self._preload_limit == 0:
do_load = True
if do_load and not self.check(reqid):
do_load = False
if (num - count) < RequestCache._preload_limit and num != -1:
if (num - count) < self._preload_limit and num != -1:
loadnum = num - count
else:
loadnum = RequestCache._preload_limit
loadnum = self._preload_limit
yield def_wrapper(reqid, load=True, num=loadnum)
else:
yield def_wrapper(reqid)
@ -187,7 +187,7 @@ class RequestCache(object):
@defer.inlineCallbacks
def load_by_tag(tag):
reqs = yield load_requests_by_tag(tag)
reqs = yield load_requests_by_tag(tag, cust_cache=self, cust_dbpool=self.dbpool)
for req in reqs:
self.add(req)
defer.returnValue(reqs)

8
pappyproxy/tests/test_requestcache.py
View file

@ -106,7 +106,7 @@ def test_cache_inmem_evict():
assert cache.check(reqs[3].reqid)
# Testing the implementation
assert reqs[0] in RequestCache.inmem_reqs
assert reqs[1] in RequestCache.inmem_reqs
assert reqs[2] in RequestCache.inmem_reqs
assert reqs[3] in RequestCache.inmem_reqs
assert reqs[0] in cache.inmem_reqs
assert reqs[1] in cache.inmem_reqs
assert reqs[2] in cache.inmem_reqs
assert reqs[3] in cache.inmem_reqs

19
pappyproxy/util.py
View file

@ -1,4 +1,6 @@
import string
import time
import datetime
class PappyException(Exception):
"""
@ -22,3 +24,20 @@ def printable_data(data):
else:
chars += '.'
return ''.join(chars)
# Taken from http://stackoverflow.com/questions/4770297/python-convert-utc-datetime-string-to-local-datetime
def utc2local(utc):
epoch = time.mktime(utc.timetuple())
offset = datetime.datetime.fromtimestamp(epoch) - datetime.datetime.utcfromtimestamp(epoch)
return utc + offset
# Taken from https://gist.github.com/sbz/1080258
def hexdump(src, length=16):
FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.' for x in range(256)])
lines = []
for c in xrange(0, len(src), length):
chars = src[c:c+length]
hex = ' '.join(["%02x" % ord(x) for x in chars])
printable = ''.join(["%s" % ((ord(x) <= 127 and FILTER[ord(x)]) or '.') for x in chars])
lines.append("%04x %-*s %s\n" % (c, length*3, hex, printable))
return ''.join(lines)