A fork of pappy proxy
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

720 lines
20 KiB

import crochet
import re
9 years ago
import shlex
9 years ago
9 years ago
from .http import Request, RepeatableDict
9 years ago
from twisted.internet import defer
from util import PappyException
9 years ago
"""
context.py
Functions and classes involved with managing the current context and filters
"""
scope = []
9 years ago
_BARE_COMPARERS = ('ex','nex')
class Context(object):
"""
A class representing a set of requests that pass a set of filters
:ivar active_filters: Filters that are currently applied to the context
:vartype active_filters: List of functions that takes one :class:`pappyproxy.http.Request` and returns either true or false.
:ivar active_requests: Requests which pass all the filters applied to the context
:type active_requests: Request
:ivar inactive_requests: Requests which do not pass all the filters applied to the context
:type inactive_requests: Request
"""
def __init__(self):
self.active_filters = []
9 years ago
self.complete = True
self.active_requests = []
@staticmethod
9 years ago
def get_memid():
i = 'm%d' % Context._next_in_mem_id
Context._next_in_mem_id += 1
return i
9 years ago
def cache_reset(self):
self.active_requests = []
self.complete = False
9 years ago
def add_filter(self, filt):
"""
Add a filter to the context. This will remove any requests that do not pass
the filter from the ``active_requests`` set.
:param filt: The filter to add
:type filt: Function that takes one :class:`pappyproxy.http.Request` and returns either true or false. (or a :class:`pappyproxy.context.Filter`)
"""
self.active_filters.append(filt)
9 years ago
self.cache_reset()
9 years ago
def filter_up(self):
"""
Removes the last filter that was applied to the context.
"""
# Deletes the last filter of the context
if self.active_filters:
self.active_filters = self.active_filters[:-1]
9 years ago
self.cache_reset()
9 years ago
def set_filters(self, filters):
"""
Set the list of filters for the context.
"""
self.active_filters = filters[:]
9 years ago
self.cache_reset()
9 years ago
9 years ago
@defer.inlineCallbacks
def get_reqs(self, n=-1):
# This is inefficient but I want it to work for now, and as long as we
# don't put the full requests in memory I don't care.
ids = self.active_requests
if (len(ids) >= n and n != -1) or self.complete == True:
if n == -1:
defer.returnValue(ids)
else:
defer.returnValue(ids[:n])
ids = []
for req_d in Request.cache.req_it():
r = yield req_d
passed = True
for filt in self.active_filters:
if not filt(r):
passed = False
break
if passed:
self.active_requests.append(r.reqid)
ids.append(r.reqid)
if len(ids) >= n and n != -1:
defer.returnValue(ids[:n])
self.complete = True
defer.returnValue(ids)
9 years ago
class FilterParseError(PappyException):
pass
def cmp_is(a, b):
9 years ago
if a is None or b is None:
return False
9 years ago
return str(a) == str(b)
def cmp_contains(a, b):
9 years ago
if a is None or b is None:
return False
9 years ago
return (b.lower() in a.lower())
def cmp_exists(a, b=None):
9 years ago
if a is None or b is None:
return False
return (a is not None and a != [])
9 years ago
def cmp_len_eq(a, b):
9 years ago
if a is None or b is None:
return False
9 years ago
return (len(a) == int(b))
def cmp_len_gt(a, b):
9 years ago
if a is None or b is None:
return False
9 years ago
return (len(a) > int(b))
def cmp_len_lt(a, b):
9 years ago
if a is None or b is None:
return False
9 years ago
return (len(a) < int(b))
def cmp_eq(a, b):
9 years ago
if a is None or b is None:
return False
9 years ago
return (int(a) == int(b))
def cmp_gt(a, b):
9 years ago
if a is None or b is None:
return False
9 years ago
return (int(a) > int(b))
def cmp_lt(a, b):
9 years ago
if a is None or b is None:
return False
9 years ago
return (int(a) < int(b))
def cmp_containsr(a, b):
9 years ago
if a is None or b is None:
return False
try:
if re.search(b, a):
return True
return False
except re.error as e:
raise PappyException('Invalid regexp: %s' % e)
9 years ago
def relation_from_text(s, val=''):
# Gets the relation function associated with the string
# Returns none if not found
def negate_func(func):
def f(*args, **kwargs):
return not func(*args, **kwargs)
return f
negate = False
if s[0] == 'n':
negate = True
s = s[1:]
9 years ago
9 years ago
if s in ("is",):
retfunc = cmp_is
elif s in ("contains", "ct"):
retfunc = cmp_contains
elif s in ("containsr", "ctr"):
validate_regexp(val)
retfunc = cmp_containsr
elif s in ("exists", "ex"):
retfunc = cmp_exists
elif s in ("Leq",):
retfunc = cmp_len_eq
elif s in ("Lgt",):
retfunc = cmp_len_gt
elif s in ("Llt",):
retfunc = cmp_len_lt
elif s in ("eq",):
retfunc = cmp_eq
elif s in ("gt",):
retfunc = cmp_gt
elif s in ("lt",):
retfunc = cmp_lt
else:
raise FilterParseError("Invalid relation: %s" % s)
if negate:
return negate_func(retfunc)
else:
return retfunc
def compval_from_args(args):
9 years ago
"""
NOINDEX
9 years ago
returns a function that compares to a value from text.
ie compval_from_text('ct foo') will return a function that returns true
if the passed in string contains foo.
9 years ago
"""
9 years ago
if len(args) == 0:
raise PappyException('Invalid number of arguments')
if args[0] in _BARE_COMPARERS:
if len(args) != 1:
raise PappyException('Invalid number of arguments')
comparer = relation_from_text(args[0], None)
value = None
else:
if len(args) != 2:
raise PappyException('Invalid number of arguments')
comparer = relation_from_text(args[0], args[1])
value = args[1]
9 years ago
9 years ago
def retfunc(s):
return comparer(s, value)
9 years ago
9 years ago
return retfunc
9 years ago
9 years ago
def compval_from_args_repdict(args):
"""
NOINDEX
Similar to compval_from_args but checks a repeatable dict with up to 2
comparers and values.
"""
if len(args) == 0:
raise PappyException('Invalid number of arguments')
nextargs = args[:]
value = None
if args[0] in _BARE_COMPARERS:
comparer = relation_from_text(args[0], None)
if len(args) > 1:
nextargs = args[1:]
else:
if len(args) == 1:
raise PappyException('Invalid number of arguments')
comparer = relation_from_text(args[0], args[1])
value = args[1]
nextargs = args[2:]
comparer2 = None
value2 = None
if nextargs:
if nextargs[0] in _BARE_COMPARERS:
comparer2 = relation_from_text(nextargs[0], None)
9 years ago
else:
9 years ago
if len(nextargs) == 1:
raise PappyException('Invalid number of arguments')
comparer2 = relation_from_text(nextargs[0], nextargs[1])
value2 = nextargs[1]
def retfunc(d):
for k, v in d.all_pairs():
if comparer2 is None:
if comparer(k, value) or comparer(v, value):
return True
else:
if comparer(k, value) and comparer2(v, value2):
return True
return False
9 years ago
9 years ago
return retfunc
def gen_filter_by_all(args):
9 years ago
compval = compval_from_args(args)
9 years ago
def f(req):
if args[0][0] == 'n':
9 years ago
return compval(req.full_message) and ((not req.response) or compval(req.response.full_message))
9 years ago
else:
9 years ago
return compval(req.full_message) or (req.response and compval(req.response.full_message))
return f
9 years ago
9 years ago
def gen_filter_by_host(args):
9 years ago
compval = compval_from_args(args)
9 years ago
def f(req):
return compval(req.host)
9 years ago
return f
9 years ago
def gen_filter_by_body(args):
9 years ago
compval = compval_from_args(args)
9 years ago
def f(req):
9 years ago
if args[0][0] == 'n':
9 years ago
return compval(req.body) and ((not req.response) or compval(req.response.body))
9 years ago
else:
9 years ago
return compval(req.body) or (req.response and compval(req.response.body))
9 years ago
return f
9 years ago
def gen_filter_by_req_body(args):
compval = compval_from_args(args)
def f(req):
return compval(req.body)
return f
def gen_filter_by_rsp_body(args):
compval = compval_from_args(args)
def f(req):
if args[0][0] == 'n':
return (not req.response) or compval(req.response.body)
else:
return req.response and compval(req.response.body)
return f
9 years ago
def gen_filter_by_raw_headers(args):
9 years ago
compval = compval_from_args(args)
9 years ago
def f(req):
9 years ago
if args[0][0] == 'n':
9 years ago
# compval already negates comparison
return compval(req.headers_section) and ((not req.response) or compval(req.response.headers_section))
9 years ago
else:
9 years ago
return compval(req.headers_section) or (req.response and compval(req.response.headers_section))
return f
9 years ago
9 years ago
def gen_filter_by_response_code(args):
compval_from_args(args) # try and throw an error
def f(req):
if not req.response:
return False
compval = compval_from_args(args)
return compval(req.response.response_code)
9 years ago
return f
9 years ago
def gen_filter_by_path(args):
9 years ago
compval = compval_from_args(args)
9 years ago
def f(req):
9 years ago
return compval(req.path)
9 years ago
return f
9 years ago
def gen_filter_by_responsetime(args):
9 years ago
compval = compval_from_args(args)
9 years ago
def f(req):
9 years ago
return compval(req.rsptime)
9 years ago
return f
9 years ago
def gen_filter_by_verb(args):
9 years ago
compval = compval_from_args(args)
9 years ago
def f(req):
9 years ago
return compval(req.verb)
9 years ago
return f
9 years ago
def gen_filter_by_tag(args):
9 years ago
compval = compval_from_args(args)
def f(req):
for tag in req.tags:
9 years ago
if compval(tag):
return True
return False
return f
9 years ago
def gen_filter_by_saved(args):
if len(args) != 0:
raise PappyException('Invalid number of arguments')
def f(req):
if req.saved:
9 years ago
return True
else:
9 years ago
return False
return f
9 years ago
@defer.inlineCallbacks
def gen_filter_by_before(args):
if len(args) != 1:
raise PappyException('Invalid number of arguments')
9 years ago
r = yield Request.load_request(args[0])
9 years ago
def f(req):
if req.time_start is None:
return False
if r.time_start is None:
return False
return req.time_start <= r.time_start
defer.returnValue(f)
9 years ago
@defer.inlineCallbacks
9 years ago
def gen_filter_by_after(args, negate=False):
9 years ago
if len(args) != 1:
raise PappyException('Invalid number of arguments')
9 years ago
r = yield Request.load_request(args[0])
9 years ago
def f(req):
if req.time_start is None:
return False
if r.time_start is None:
return False
return req.time_start >= r.time_start
defer.returnValue(f)
9 years ago
9 years ago
def gen_filter_by_headers(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
9 years ago
if args[0][0] == 'n':
9 years ago
return comparer(req.headers) and ((not req.response) or comparer(req.response.headers))
9 years ago
else:
9 years ago
return comparer(req.headers) or (req.response and comparer(req.response.headers))
9 years ago
return f
9 years ago
def gen_filter_by_request_headers(args):
comparer = compval_from_args_repdict(args)
def f(req):
return comparer(req.headers)
return f
def gen_filter_by_response_headers(args):
comparer = compval_from_args_repdict(args)
def f(req):
if args[0][0] == 'n':
return (not req.response) or comparer(req.response.headers)
else:
return req.response and comparer(req.response.headers)
return f
9 years ago
def gen_filter_by_submitted_cookies(args):
comparer = compval_from_args_repdict(args)
def f(req):
return comparer(req.cookies)
return f
def gen_filter_by_set_cookies(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
if not req.response:
return False
9 years ago
checkdict = RepeatableDict()
9 years ago
for k, v in req.response.cookies.all_pairs():
checkdict[k] = v.cookie_str
return comparer(checkdict)
9 years ago
return f
9 years ago
def gen_filter_by_url_params(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
9 years ago
return comparer(req.url_params)
9 years ago
return f
9 years ago
def gen_filter_by_post_params(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
9 years ago
return comparer(req.post_params)
9 years ago
return f
9 years ago
def gen_filter_by_params(args):
comparer = compval_from_args_repdict(args)
9 years ago
def f(req):
9 years ago
return comparer(req.url_params) or comparer(req.post_params)
9 years ago
return f
9 years ago
@defer.inlineCallbacks
def gen_filter_by_inverse(args):
filt = yield Filter.from_filter_string(parsed_args=args)
def f(req):
return not filt(req)
defer.returnValue(f)
9 years ago
@defer.inlineCallbacks
9 years ago
def filter_reqs(reqids, filters):
9 years ago
to_delete = set()
# Could definitely be more efficient, but it stays like this until
# it impacts performance
9 years ago
requests = []
for reqid in reqids:
r = yield Request.load_request(reqid)
requests.append(r)
9 years ago
for req in requests:
for filt in filters:
if not filt(req):
to_delete.add(req)
9 years ago
retreqs = []
retdel = []
for r in requests:
if r in to_delete:
retdel.append(r.reqid)
else:
retreqs.append(r.reqid)
defer.returnValue((retreqs, retdel))
9 years ago
9 years ago
def passes_filters(request, filters):
for filt in filters:
if not filt(request):
return False
return True
def in_scope(request):
global scope
9 years ago
passes = passes_filters(request, scope)
return passes
9 years ago
def set_scope(filters):
global scope
scope = filters
9 years ago
def save_scope(context):
9 years ago
global scope
9 years ago
scope = context.active_filters[:]
9 years ago
9 years ago
def reset_to_scope(context):
9 years ago
global scope
9 years ago
context.active_filters = scope[:]
9 years ago
context.cache_reset()
9 years ago
def print_scope():
global scope
for f in scope:
print f.filter_string
@defer.inlineCallbacks
def store_scope(dbpool):
# Delete the old scope
yield dbpool.runQuery(
"""
DELETE FROM scope
"""
);
# Insert the new scope
i = 0
for f in scope:
yield dbpool.runQuery(
"""
INSERT INTO scope (filter_order, filter_string) VALUES (?, ?);
""",
(i, f.filter_string)
);
i += 1
@defer.inlineCallbacks
def load_scope(dbpool):
global scope
rows = yield dbpool.runQuery(
"""
SELECT filter_order, filter_string FROM scope;
""",
)
rows = sorted(rows, key=lambda r: int(r[0]))
new_scope = []
for row in rows:
new_filter = Filter(row[1])
9 years ago
yield new_filter.generate()
9 years ago
new_scope.append(new_filter)
scope = new_scope
@defer.inlineCallbacks
def clear_tag(tag):
# Remove a tag from every request
9 years ago
reqs = yield Request.cache.load_by_tag(tag)
for req in reqs:
9 years ago
req.tags.discard(tag)
if req.saved:
yield req.async_save()
9 years ago
reset_context_caches()
@defer.inlineCallbacks
def async_set_tag(tag, reqs):
"""
async_set_tag(tag, reqs)
Remove the tag from every request then add the given requests to memory and
9 years ago
give them the tag. The async version.
:param tag: The tag to set
:type tag: String
:param reqs: The requests to assign to the tag
:type reqs: List of Requests
"""
yield clear_tag(tag)
for req in reqs:
9 years ago
req.tags.add(tag)
9 years ago
Request.cache.add(req)
reset_context_caches()
@crochet.wait_for(timeout=180.0)
@defer.inlineCallbacks
def set_tag(tag, reqs):
9 years ago
"""
set_tag(tag, reqs)
Remove the tag from every request then add the given requests to memory and
give them the tag. The non-async version.
:param tag: The tag to set
:type tag: String
:param reqs: The requests to assign to the tag
:type reqs: List of Requests
"""
yield async_set_tag(tag, reqs)
def validate_regexp(r):
try:
re.compile(r)
except re.error as e:
raise PappyException('Invalid regexp: %s' % e)
9 years ago
def reset_context_caches():
9 years ago
import pappyproxy.pappy
for c in pappyproxy.pappy.all_contexts:
9 years ago
c.cache_reset()
9 years ago
class Filter(object):
"""
A class representing a filter. Its claim to fame is that you can use
:func:`pappyproxy.context.Filter.from_filter_string` to generate a
filter from a filter string.
"""
_filter_functions = {
"all": gen_filter_by_all,
"host": gen_filter_by_host,
"domain": gen_filter_by_host,
"hs": gen_filter_by_host,
"dm": gen_filter_by_host,
"path": gen_filter_by_path,
"pt": gen_filter_by_path,
"body": gen_filter_by_body,
"bd": gen_filter_by_body,
"data": gen_filter_by_body,
"dt": gen_filter_by_body,
"reqbody": gen_filter_by_req_body,
"qbd": gen_filter_by_req_body,
"reqdata": gen_filter_by_req_body,
"qdt": gen_filter_by_req_body,
"rspbody": gen_filter_by_rsp_body,
"sbd": gen_filter_by_rsp_body,
"qspdata": gen_filter_by_rsp_body,
"sdt": gen_filter_by_rsp_body,
"verb": gen_filter_by_verb,
"vb": gen_filter_by_verb,
"param": gen_filter_by_params,
"pm": gen_filter_by_params,
"header": gen_filter_by_headers,
"hd": gen_filter_by_headers,
"reqheader": gen_filter_by_request_headers,
"qhd": gen_filter_by_request_headers,
"rspheader": gen_filter_by_response_headers,
"shd": gen_filter_by_response_headers,
"rawheaders": gen_filter_by_raw_headers,
"rh": gen_filter_by_raw_headers,
"sentcookie": gen_filter_by_submitted_cookies,
"sck": gen_filter_by_submitted_cookies,
"setcookie": gen_filter_by_set_cookies,
"stck": gen_filter_by_set_cookies,
"statuscode": gen_filter_by_response_code,
"sc": gen_filter_by_response_code,
"responsecode": gen_filter_by_response_code,
"tag": gen_filter_by_tag,
"tg": gen_filter_by_tag,
"saved": gen_filter_by_saved,
"svd": gen_filter_by_saved,
}
_async_filter_functions = {
"before": gen_filter_by_before,
"b4": gen_filter_by_before,
"bf": gen_filter_by_before,
"after": gen_filter_by_after,
"af": gen_filter_by_after,
"inv": gen_filter_by_inverse,
}
def __init__(self, filter_string):
self.filter_string = filter_string
def __call__(self, *args, **kwargs):
return self.filter_func(*args, **kwargs)
def __repr__(self):
return '<Filter "%s">' % self.filter_string
@defer.inlineCallbacks
def generate(self):
self.filter_func = yield self.from_filter_string(self.filter_string)
@staticmethod
@defer.inlineCallbacks
def from_filter_string(filter_string=None, parsed_args=None):
"""
from_filter_string(filter_string)
Create a filter from a filter string. If passed a list of arguments, they
will be used instead of parsing the string.
:rtype: Deferred that returns a :class:`pappyproxy.context.Filter`
"""
if parsed_args is not None:
args = parsed_args
else:
args = shlex.split(filter_string)
if len(args) == 0:
raise PappyException('Field is required')
field = args[0]
new_filter = None
field_args = args[1:]
if field in Filter._filter_functions:
new_filter = Filter._filter_functions[field](field_args)
elif field in Filter._async_filter_functions:
new_filter = yield Filter._async_filter_functions[field](field_args)
else:
raise FilterParseError("%s is not a valid field" % field)
if new_filter is None:
raise FilterParseError("Error creating filter")
defer.returnValue(new_filter)