Version 0.2.1

pappyproxy/plugins/filter.py

@@ -3,6 +3,7 @@ import pappyproxy
 
 from pappyproxy.console import confirm
 from pappyproxy.util import PappyException
+from pappyproxy.http import Request
 from twisted.internet import defer
 
 class BuiltinFilters(object):
@@ -72,7 +73,7 @@ def builtin_filter(line):
     filters_to_add = yield BuiltinFilters.get(line)
     for f in filters_to_add:
         print f.filter_string
-        pappyproxy.pappy.main_context.add_filter(f)
+        yield pappyproxy.pappy.main_context.add_filter(f)
     defer.returnValue(None)
 
 def filter_up(line):
@@ -82,15 +83,12 @@ def filter_up(line):
     """
     pappyproxy.pappy.main_context.filter_up()
 
-@crochet.wait_for(timeout=None)
-@defer.inlineCallbacks
 def filter_clear(line):
     """
     Reset the context so that it contains no filters (ignores scope)
     Usage: filter_clear
     """
-    pappyproxy.pappy.main_context.active_filters = []
-    yield pappyproxy.context.reload_from_storage()
+    pappyproxy.pappy.main_context.set_filters([])
 
 def filter_list(line):
     """
@@ -111,12 +109,14 @@ def scope_save(line):
     pappyproxy.context.save_scope(pappyproxy.pappy.main_context)
     yield pappyproxy.context.store_scope(pappyproxy.http.dbpool)
 
+@crochet.wait_for(timeout=None)
+@defer.inlineCallbacks
 def scope_reset(line):
     """
     Set the context to be the scope (view in-scope items)
     Usage: scope_reset
     """
-    pappyproxy.context.reset_to_scope(pappyproxy.pappy.main_context)
+    yield pappyproxy.context.reset_to_scope(pappyproxy.pappy.main_context)
 
 @crochet.wait_for(timeout=None)
 @defer.inlineCallbacks
@@ -143,6 +143,7 @@ def filter_prune(line):
     CANNOT BE UNDONE!! Be careful!
     Usage: filter_prune
     """
+    from pappyproxy.requestcache import RequestCache
     # Delete filtered items from datafile
     print ''
     print 'Currently active filters:'
@@ -150,15 +151,20 @@ def filter_prune(line):
         print '> %s' % f.filter_string
 
     # We copy so that we're not removing items from a set we're iterating over
-    reqs = list(pappyproxy.pappy.main_context.inactive_requests)
-    act_reqs = list(pappyproxy.pappy.main_context.active_requests)
-    message = 'This will delete %d/%d requests. You can NOT undo this!! Continue?' % (len(reqs), (len(reqs) + len(act_reqs)))
+    act_reqs = yield pappyproxy.pappy.main_context.get_reqs()
+    inact_reqs = RequestCache.all_ids.difference(set(act_reqs))
+    inact_reqs = inact_reqs.difference(set(RequestCache.unmangled_ids))
+    message = 'This will delete %d/%d requests. You can NOT undo this!! Continue?' % (len(inact_reqs), (len(inact_reqs) + len(act_reqs)))
     if not confirm(message, 'n'):
         defer.returnValue(None)
     
-    for r in reqs:
-        yield r.deep_delete()
-    print 'Deleted %d requests' % len(reqs)
+    for reqid in inact_reqs:
+        try:
+            req = yield pappyproxy.http.Request.load_request(reqid)
+            yield req.deep_delete()
+        except PappyException as e:
+            print e
+    print 'Deleted %d requests' % len(inact_reqs)
     defer.returnValue(None)
 
 ###############

pappyproxy/plugins/misc.py

@@ -4,16 +4,19 @@ import shlex
 
 from pappyproxy.console import confirm, load_reqlist
 from pappyproxy.util import PappyException
+from pappyproxy.http import Request
 from twisted.internet import defer
 
+@crochet.wait_for(timeout=None)
+@defer.inlineCallbacks
 def clrmem(line):
     """
     Delete all in-memory only requests
     Usage: clrmem
     """
-    to_delete = list(pappyproxy.context.Context.in_memory_requests)
+    to_delete = list(pappyproxy.requestcache.RequestCache.inmem_reqs)
     for r in to_delete:
-        pappyproxy.context.Context.remove_request(r)
+        yield r.deep_delete()
 
 def gencerts(line):
     """
@@ -42,6 +45,15 @@ def log(line):
     raw_input()
     pappyproxy.config.DEBUG_VERBOSITY = 0
 
+@crochet.wait_for(timeout=None)
+@defer.inlineCallbacks
+def save(line):
+    args = shlex.split(line)
+    reqids = args[0]
+    reqs = yield load_reqlist(reqids)
+    for req in reqs:
+        yield req.async_deep_save()
+    
 @crochet.wait_for(timeout=None)
 @defer.inlineCallbacks
 def export(line):
@@ -77,6 +89,7 @@ def load_cmds(cmd):
     cmd.set_cmds({
         'clrmem': (clrmem, None),
         'gencerts': (gencerts, None),
+        'sv': (save, None),
         'export': (export, None),
         'log': (log, None),
     })

pappyproxy/plugins/tagcmds.py

@@ -2,10 +2,11 @@ import crochet
 import pappyproxy
 import shlex
 
-from pappyproxy.plugin import main_context
+from pappyproxy.plugin import main_context_ids
 from pappyproxy.console import load_reqlist
 from pappyproxy.util import PappyException
 from twisted.internet import defer
+from pappyproxy.http import Request
 
 @crochet.wait_for(timeout=None)
 @defer.inlineCallbacks
@@ -22,19 +23,18 @@ def tag(line):
     tag = args[0]
 
     if len(args) > 1:
-        reqs = yield load_reqlist(args[1], False)
-        ids = [r.reqid for r in reqs]
-        print 'Tagging %s with %s' % (', '.join(ids), tag)
+        reqids = yield load_reqlist(args[1], False, ids_only=True)
+        print 'Tagging %s with %s' % (', '.join(reqids), tag)
     else:
         print "Tagging all in-context requests with %s" % tag
-        reqs = main_context().active_requests
+        reqids = yield main_context_ids()
 
-    for req in reqs:
+    for reqid in reqids:
+        req = yield Request.load_request(reqid)
         if tag not in req.tags:
             req.tags.append(tag)
             if req.saved:
                 yield req.async_save()
-            add_req(req)
         else:
             print 'Request %s already has tag %s' % (req.reqid, tag)
 
@@ -55,13 +55,14 @@ def untag(line):
 
     ids = []
     if len(args) > 1:
-        reqs = yield load_reqlist(args[1], False)
-        ids = [r.reqid for r in reqs]
+        reqids = yield load_reqlist(args[1], False, ids_only=True)
+        print 'Removing tag %s from %s' % (tag, ', '.join(reqids))
     else:
-        print "Untagging all in-context requests with tag %s" % tag
-        reqs = main_context().active_requests
+        print "Removing tag %s from all in-context requests" % tag
+        reqids = yield main_context_ids()
 
-    for req in reqs:
+    for reqid in reqids:
+        req = yield Request.load_request(reqid)
         if tag in req.tags:
             req.tags.remove(tag)
             if req.saved:

pappyproxy/plugins/view.py

@@ -3,11 +3,11 @@ import datetime
 import pappyproxy
 import shlex
 
-from pappyproxy.console import load_reqlist, print_table, print_requests
+from pappyproxy.console import load_reqlist, print_table, print_request_rows, get_req_data_row
 from pappyproxy.util import PappyException
-from pappyproxy.plugin import main_context
 from pappyproxy.http import Request
 from twisted.internet import defer
+from pappyproxy.plugin import main_context_ids
 
 ###################
 ## Helper functions
@@ -78,14 +78,6 @@ def print_request_extended(request):
     if request.plugin_data:
         print 'Plugin Data: %s' % (request.plugin_data)
 
-def get_site_map(reqs):
-    # Takes in a list of requests and returns a tree representing the site map
-    paths_set = set()
-    for req in reqs:
-        paths_set.add(req.path_tuple)
-    paths = sorted(list(paths_set))
-    return paths
-
 def print_tree(tree):
     # Prints a tree. Takes in a sorted list of path tuples
     _print_tree_helper(tree, 0, [])
@@ -142,6 +134,8 @@ def _print_tree_helper(tree, depth, print_bars):
 ####################
 ## Command functions
     
+@crochet.wait_for(timeout=None)
+@defer.inlineCallbacks
 def list_reqs(line):
     """
     List the most recent in-context requests. By default shows the most recent 25
@@ -163,16 +157,12 @@ def list_reqs(line):
     else:
         print_count = 25
 
-    def key_reqtime(req):
-        if req.time_start is None:
-            return -1
-        else:
-            return (req.time_start-datetime.datetime(1970,1,1)).total_seconds()
-
-    to_print = sorted(main_context().active_requests, key=key_reqtime, reverse=True)
-    if print_count > 0:
-        to_print = to_print[:print_count]
-    print_requests(to_print)
+    rows = []
+    ids = yield main_context_ids(print_count)
+    for i in ids:
+        req = yield Request.load_request(i)
+        rows.append(get_req_data_row(req))
+    print_request_rows(rows)
 
 @crochet.wait_for(timeout=None)
 @defer.inlineCallbacks
@@ -292,13 +282,20 @@ def dump_response(line):
         f.write(rsp.body)
     print 'Response data written to %s' % fname
 
+@crochet.wait_for(timeout=None)
+@defer.inlineCallbacks
 def site_map(line):
     """
     Print the site map. Only includes requests in the current context.
     Usage: site_map
     """
-    to_print = [r for r in main_context().active_requests if not r.response or r.response.response_code != 404]
-    tree = get_site_map(to_print)
+    ids = yield main_context_ids()
+    paths_set = set()
+    for reqid in ids:
+        req = yield Request.load_request(reqid)
+        if req.response and req.response.response_code != 404:
+            paths_set.add(req.path_tuple)
+    tree = sorted(list(paths_set))
     print_tree(tree)