A fork of pappy proxy
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

627 lines
18 KiB

import crochet
9 years ago
import pappyproxy
import re
9 years ago
import shlex
9 years ago
9 years ago
from .http import Request, RepeatableDict
from .requestcache import RequestCache
9 years ago
from twisted.internet import defer
from util import PappyException
9 years ago
"""
context.py
Functions and classes involved with managing the current context and filters
"""
scope = []
9 years ago
_BARE_COMPARERS = ('ex','nex')
class Context(object):
"""
A class representing a set of requests that pass a set of filters
:ivar active_filters: Filters that are currently applied to the context
:vartype active_filters: List of functions that takes one :class:`pappyproxy.http.Request` and returns either true or false.
:ivar active_requests: Requests which pass all the filters applied to the context
:type active_requests: Request
:ivar inactive_requests: Requests which do not pass all the filters applied to the context
:type inactive_requests: Request
"""
def __init__(self):
self.active_filters = []
9 years ago
self.complete = True
self.active_requests = []
@staticmethod
9 years ago
def get_memid():
i = 'm%d' % Context._next_in_mem_id
Context._next_in_mem_id += 1
return i
9 years ago
def cache_reset(self):
self.active_requests = []
self.complete = False
9 years ago
def add_filter(self, filt):
"""
Add a filter to the context. This will remove any requests that do not pass
the filter from the ``active_requests`` set.
:param filt: The filter to add
:type filt: Function that takes one :class:`pappyproxy.http.Request` and returns either true or false. (or a :class:`pappyproxy.context.Filter`)
"""
self.active_filters.append(filt)
9 years ago
self.cache_reset()
9 years ago
def filter_up(self):
"""
Removes the last filter that was applied to the context.
"""
# Deletes the last filter of the context
if self.active_filters:
self.active_filters = self.active_filters[:-1]
9 years ago
self.cache_reset()
9 years ago
def set_filters(self, filters):
"""
Set the list of filters for the context.
"""
self.active_filters = filters[:]
9 years ago
self.cache_reset()
9 years ago
9 years ago
@defer.inlineCallbacks
def get_reqs(self, n=-1):
# This is inefficient but I want it to work for now, and as long as we
# don't put the full requests in memory I don't care.
ids = self.active_requests
if (len(ids) >= n and n != -1) or self.complete == True:
if n == -1:
defer.returnValue(ids)
else:
defer.returnValue(ids[:n])
ids = []
for req_d in Request.cache.req_it():
r = yield req_d
passed = True
for filt in self.active_filters:
if not filt(r):
passed = False
break
if passed:
self.active_requests.append(r.reqid)
ids.append(r.reqid)
if len(ids) >= n and n != -1:
defer.returnValue(ids[:n])
self.complete = True
defer.returnValue(ids)
9 years ago
class FilterParseError(PappyException):
pass
class Filter(object):
9 years ago
"""
A class representing a filter. Its claim to fame is that you can use
:func:`pappyproxy.context.Filter.from_filter_string` to generate a
filter from a filter string.
"""
9 years ago
def __init__(self, filter_string):
self.filter_string = filter_string
def __call__(self, *args, **kwargs):
return self.filter_func(*args, **kwargs)
def __repr__(self):
return '<Filter "%s">' % self.filter_string
9 years ago
@defer.inlineCallbacks
def generate(self):
self.filter_func = yield self.from_filter_string(self.filter_string)
9 years ago
@staticmethod
9 years ago
@defer.inlineCallbacks
9 years ago
def from_filter_string(filter_string):
9 years ago
"""
from_filter_string(filter_string)
Create a filter from a filter string.
:rtype: Deferred that returns a :class:`pappyproxy.context.Filter`
"""
9 years ago
args = shlex.split(filter_string)
9 years ago
if len(args) == 0:
raise PappyException('Field is required')
9 years ago
field = args[0]
new_filter = None
9 years ago
field_args = args[1:]
9 years ago
if field in ("all",):
9 years ago
new_filter = gen_filter_by_all(field_args)
9 years ago
elif field in ("host", "domain", "hs", "dm"):
9 years ago
new_filter = gen_filter_by_host(field_args)
9 years ago
elif field in ("path", "pt"):
9 years ago
new_filter = gen_filter_by_path(field_args)
9 years ago
elif field in ("body", "bd", "data", "dt"):
9 years ago
new_filter = gen_filter_by_body(field_args)
9 years ago
elif field in ("verb", "vb"):
9 years ago
new_filter = gen_filter_by_verb(field_args)
9 years ago
elif field in ("param", "pm"):
9 years ago
new_filter = gen_filter_by_params(field_args)
9 years ago
elif field in ("header", "hd"):
9 years ago
new_filter = gen_filter_by_headers(field_args)
9 years ago
elif field in ("rawheaders", "rh"):
9 years ago
new_filter = gen_filter_by_raw_headers(field_args)
9 years ago
elif field in ("sentcookie", "sck"):
9 years ago
new_filter = gen_filter_by_submitted_cookies(field_args)
9 years ago
elif field in ("setcookie", "stck"):
9 years ago
new_filter = gen_filter_by_set_cookies(field_args)
9 years ago
elif field in ("statuscode", "sc", "responsecode"):
9 years ago
new_filter = gen_filter_by_response_code(field_args)
9 years ago
elif field in ("responsetime", "rt"):
9 years ago
raise PappyException('Not implemented yet, sorry!')
elif field in ("tag", "tg"):
9 years ago
new_filter = gen_filter_by_tag(field_args)
elif field in ("saved", "svd"):
9 years ago
new_filter = gen_filter_by_saved(field_args)
elif field in ("before", "b4", "bf"):
new_filter = yield gen_filter_by_before(field_args)
elif field in ("after", "af"):
new_filter = yield gen_filter_by_after(field_args)
9 years ago
else:
raise FilterParseError("%s is not a valid field" % field)
9 years ago
if new_filter is None:
9 years ago
raise FilterParseError("Error creating filter")
9 years ago
# dirty hack to get it to work if we don't generate any deferreds
# d = defer.Deferred()
# d.callback(None)
# yield d
defer.returnValue(new_filter)
9 years ago
def cmp_is(a, b):
return str(a) == str(b)
def cmp_contains(a, b):
return (b.lower() in a.lower())
def cmp_exists(a, b=None):
return (a is not None and a != [])
9 years ago
def cmp_len_eq(a, b):
return (len(a) == int(b))
def cmp_len_gt(a, b):
return (len(a) > int(b))
def cmp_len_lt(a, b):
return (len(a) < int(b))
def cmp_eq(a, b):
return (int(a) == int(b))
def cmp_gt(a, b):
return (int(a) > int(b))
def cmp_lt(a, b):
return (int(a) < int(b))
def cmp_containsr(a, b):
try:
if re.search(b, a):
return True
return False
except re.error as e:
raise PappyException('Invalid regexp: %s' % e)
9 years ago
def relation_from_text(s, val=''):
# Gets the relation function associated with the string
# Returns none if not found
def negate_func(func):
def f(*args, **kwargs):
return not func(*args, **kwargs)
return f
negate = False
if s[0] == 'n':
negate = True
s = s[1:]
9 years ago
9 years ago
if s in ("is",):
retfunc = cmp_is
elif s in ("contains", "ct"):
retfunc = cmp_contains
elif s in ("containsr", "ctr"):
validate_regexp(val)
retfunc = cmp_containsr
elif s in ("exists", "ex"):
retfunc = cmp_exists
elif s in ("Leq",):
retfunc = cmp_len_eq
elif s in ("Lgt",):
retfunc = cmp_len_gt
elif s in ("Llt",):
retfunc = cmp_len_lt
elif s in ("eq",):
retfunc = cmp_eq
elif s in ("gt",):
retfunc = cmp_gt
elif s in ("lt",):
retfunc = cmp_lt
else:
raise FilterParseError("Invalid relation: %s" % s)
if negate:
return negate_func(retfunc)
else:
return retfunc
def compval_from_args(args):
9 years ago
"""
NOINDEX
9 years ago
returns a function that compares to a value from text.
ie compval_from_text('ct foo') will return a function that returns true
if the passed in string contains foo.
9 years ago
"""
9 years ago
if len(args) == 0:
raise PappyException('Invalid number of arguments')
if args[0] in _BARE_COMPARERS:
if len(args) != 1:
raise PappyException('Invalid number of arguments')
comparer = relation_from_text(args[0], None)
value = None
else:
if len(args) != 2:
raise PappyException('Invalid number of arguments')
comparer = relation_from_text(args[0], args[1])
value = args[1]
9 years ago
9 years ago
def retfunc(s):
return comparer(s, value)
9 years ago
9 years ago
return retfunc
9 years ago
9 years ago
def compval_from_args_repdict(args):
"""
NOINDEX
Similar to compval_from_args but checks a repeatable dict with up to 2
comparers and values.
"""
if len(args) == 0:
raise PappyException('Invalid number of arguments')
nextargs = args[:]
value = None
if args[0] in _BARE_COMPARERS:
comparer = relation_from_text(args[0], None)
if len(args) > 1:
nextargs = args[1:]
else:
if len(args) == 1:
raise PappyException('Invalid number of arguments')
comparer = relation_from_text(args[0], args[1])
value = args[1]
nextargs = args[2:]
comparer2 = None
value2 = None
if nextargs:
if nextargs[0] in _BARE_COMPARERS:
comparer2 = relation_from_text(nextargs[0], None)
9 years ago
else:
9 years ago
if len(nextargs) == 1:
raise PappyException('Invalid number of arguments')
comparer2 = relation_from_text(nextargs[0], nextargs[1])
value2 = nextargs[1]
def retfunc(d):
for k, v in d.all_pairs():
if comparer2 is None:
if comparer(k, value) or comparer(v, value):
return True
else:
if comparer(k, value) and comparer2(v, value2):
return True
return False
9 years ago
9 years ago
return retfunc
def gen_filter_by_all(args):
compval_from_args(args) # try and throw an error
def f(req):
compval = compval_from_args(args)
if args[0][0] == 'n':
return compval(req.full_message) and (not req.response or compval(req.response.full_message))
9 years ago
else:
9 years ago
return compval(req.full_message) or (req.response and compval(req.response.full_message))
return f
9 years ago
9 years ago
def gen_filter_by_host(args):
compval_from_args(args) # try and throw an error
def f(req):
compval = compval_from_args(args)
return compval(req.host)
9 years ago
return f
9 years ago
def gen_filter_by_body(args):
compval_from_args(args) # try and throw an error
9 years ago
def f(req):
9 years ago
compval = compval_from_args(args)
if args[0][0] == 'n':
return compval(req.body) and (not req.response or compval(req.response.body))
9 years ago
else:
9 years ago
return compval(req.body) or (req.response and compval(req.response.body))
9 years ago
return f
9 years ago
def gen_filter_by_raw_headers(args):
compval_from_args(args) # try and throw an error
9 years ago
def f(req):
9 years ago
compval = compval_from_args(args)
if args[0][0] == 'n':
return compval(req.headers_section) and (not req.response or compval(req.response.headers_section))
9 years ago
else:
9 years ago
return compval(req.headers_section) or (req.response and compval(req.response.headers_section))
return f
9 years ago
9 years ago
def gen_filter_by_response_code(args):
compval_from_args(args) # try and throw an error
def f(req):
if not req.response:
return False
compval = compval_from_args(args)
return compval(req.response.response_code)
9 years ago
return f
9 years ago
def gen_filter_by_path(args):
compval_from_args(args)
9 years ago
def f(req):
9 years ago
compval = compval_from_args(args)
return compval(req.path)
9 years ago
return f
9 years ago
def gen_filter_by_responsetime(args):
compval_from_args(args)
9 years ago
def f(req):
9 years ago
compval = compval_from_args(args)
return compval(req.rsptime)
9 years ago
return f
9 years ago
def gen_filter_by_verb(args):
compval_from_args(args)
9 years ago
def f(req):
9 years ago
compval = compval_from_args(args)
return compval(req.verb)
9 years ago
return f
9 years ago
def gen_filter_by_tag(args):
compval_from_args(args)
def f(req):
9 years ago
compval = compval_from_args(args)
for tag in req.tags:
9 years ago
if compval(tag):
return True
return False
return f
9 years ago
def gen_filter_by_saved(args):
if len(args) != 0:
raise PappyException('Invalid number of arguments')
def f(req):
if req.saved:
9 years ago
return True
else:
9 years ago
return False
return f
9 years ago
@defer.inlineCallbacks
def gen_filter_by_before(args):
if len(args) != 1:
raise PappyException('Invalid number of arguments')
r = yield http.Request.load_request(args[0])
def f(req):
if req.time_start is None:
return False
if r.time_start is None:
return False
return req.time_start <= r.time_start
defer.returnValue(f)
9 years ago
@defer.inlineCallbacks
def gen_filter_by_after(reqid, negate=False):
if len(args) != 1:
raise PappyException('Invalid number of arguments')
r = yield http.Request.load_request(args[0])
def f(req):
if req.time_start is None:
return False
if r.time_start is None:
return False
return req.time_start >= r.time_start
defer.returnValue(f)
9 years ago
9 years ago
def gen_filter_by_headers(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
9 years ago
if args[0][0] == 'n':
return comparer(req.headers) and (not req.response or comparer(req.response.headers))
9 years ago
else:
9 years ago
return comparer(req.headers) and (req.response and comparer(req.response.headers))
9 years ago
return f
9 years ago
def gen_filter_by_submitted_cookies(args):
comparer = compval_from_args_repdict(args)
def f(req):
return comparer(req.cookies)
return f
def gen_filter_by_set_cookies(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
if not req.response:
return False
9 years ago
checkdict = RepeatableDict()
9 years ago
for k, v in req.response.cookies.all_pairs():
checkdict[k] = v.cookie_str
return comparer(checkdict)
9 years ago
return f
9 years ago
def gen_filter_by_url_params(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
9 years ago
return comparer(req.url_params)
9 years ago
return f
9 years ago
def gen_filter_by_post_params(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
9 years ago
return comparer(req.post_params)
9 years ago
return f
9 years ago
def gen_filter_by_params(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
9 years ago
return comparer(req.url_params) or comparer(req.post_params)
9 years ago
return f
@defer.inlineCallbacks
9 years ago
def filter_reqs(reqids, filters):
9 years ago
to_delete = set()
# Could definitely be more efficient, but it stays like this until
# it impacts performance
9 years ago
requests = []
for reqid in reqids:
r = yield Request.load_request(reqid)
requests.append(r)
9 years ago
for req in requests:
for filt in filters:
if not filt(req):
to_delete.add(req)
9 years ago
retreqs = []
retdel = []
for r in requests:
if r in to_delete:
retdel.append(r.reqid)
else:
retreqs.append(r.reqid)
defer.returnValue((retreqs, retdel))
9 years ago
9 years ago
def passes_filters(request, filters):
for filt in filters:
if not filt(request):
return False
return True
def in_scope(request):
global scope
9 years ago
passes = passes_filters(request, scope)
return passes
9 years ago
def set_scope(filters):
global scope
scope = filters
9 years ago
def save_scope(context):
9 years ago
global scope
9 years ago
scope = context.active_filters[:]
9 years ago
9 years ago
def reset_to_scope(context):
9 years ago
global scope
9 years ago
context.active_filters = scope[:]
9 years ago
context.cache_reset()
9 years ago
def print_scope():
global scope
for f in scope:
print f.filter_string
@defer.inlineCallbacks
def store_scope(dbpool):
# Delete the old scope
yield dbpool.runQuery(
"""
DELETE FROM scope
"""
);
# Insert the new scope
i = 0
for f in scope:
yield dbpool.runQuery(
"""
INSERT INTO scope (filter_order, filter_string) VALUES (?, ?);
""",
(i, f.filter_string)
);
i += 1
@defer.inlineCallbacks
def load_scope(dbpool):
global scope
rows = yield dbpool.runQuery(
"""
SELECT filter_order, filter_string FROM scope;
""",
)
rows = sorted(rows, key=lambda r: int(r[0]))
new_scope = []
for row in rows:
new_filter = Filter(row[1])
9 years ago
yield new_filter.generate()
9 years ago
new_scope.append(new_filter)
scope = new_scope
@defer.inlineCallbacks
def clear_tag(tag):
# Remove a tag from every request
9 years ago
reqs = yield Request.cache.load_by_tag(tag)
for req in reqs:
req.tags.remove(tag)
if req.saved:
yield req.async_save()
9 years ago
reset_context_caches()
@defer.inlineCallbacks
def async_set_tag(tag, reqs):
"""
async_set_tag(tag, reqs)
Remove the tag from every request then add the given requests to memory and
9 years ago
give them the tag. The async version.
:param tag: The tag to set
:type tag: String
:param reqs: The requests to assign to the tag
:type reqs: List of Requests
"""
yield clear_tag(tag)
for req in reqs:
req.tags.append(tag)
9 years ago
Request.cache.add(req)
reset_context_caches()
@crochet.wait_for(timeout=180.0)
@defer.inlineCallbacks
def set_tag(tag, reqs):
9 years ago
"""
set_tag(tag, reqs)
Remove the tag from every request then add the given requests to memory and
give them the tag. The non-async version.
:param tag: The tag to set
:type tag: String
:param reqs: The requests to assign to the tag
:type reqs: List of Requests
"""
yield async_set_tag(tag, reqs)
def validate_regexp(r):
try:
re.compile(r)
except re.error as e:
raise PappyException('Invalid regexp: %s' % e)
9 years ago
def reset_context_caches():
9 years ago
import pappyproxy.pappy
for c in pappyproxy.pappy.all_contexts:
9 years ago
c.cache_reset()