jab2870/PappyProxy
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Version 0.2.11

Browse source
This commit is contained in:
Rob Glew 2016-04-14 17:25:59 -05:00
parent 9d274de709
commit 992edab315
17 changed files with 551 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

2
pappyproxy/__init__.py
View file

@ -1 +1 @@
__version__ = '0.2.10'
__version__ = '0.2.11'

24
pappyproxy/config.py
View file

@ -46,14 +46,14 @@ class PappyConfig(object):
:Default: None
.. data: listeners
.. data:: listeners
The list of active listeners. It is a list of tuples of the format (port, interface)
Not modifiable after startup. Configured in the ``config.json`` file for the project.
:Default: ``[(8000, '127.0.0.1')]``
.. data: socks_proxy
.. data:: socks_proxy
Details for a SOCKS proxy. It is a dict with the following key/values::
@ -66,7 +66,7 @@ class PappyConfig(object):
:Default: ``null``
.. data: http_proxy
.. data:: http_proxy
Details for an upstream HTTP proxy. It is a dict with the following key/values::
@ -77,37 +77,37 @@ class PappyConfig(object):
If null, no proxy will be used.
.. data: plugin_dirs
.. data:: plugin_dirs
List of directories that plugins are loaded from. Not modifiable.
:Default: ``['{DATA_DIR}/plugins', '{PAPPY_DIR}/plugins']``
.. data: save_history
.. data:: save_history
Whether command history should be saved to a file/loaded at startup.
:Default: True
.. data: config_dict
.. data:: config_dict
The dictionary read from config.json. When writing plugins, use this to load
configuration options for your plugin.
.. data: global_config_dict
.. data:: global_config_dict
The dictionary from ~/.pappy/global_config.json. It contains settings for
Pappy that are specific to the current computer. Avoid putting settings here,
especially if it involves specific projects.
.. data: archive
.. data:: archive
Project archive compressed as a ``tar.bz2`` archive if libraries available on the system,
otherwise falls back to zip archive.
:Default: ``project.archive``
.. data: crypt_dir
.. data:: crypt_dir
Temporary working directory to unpack an encrypted project archive. Directory
will contain copies of normal startup files, e.g. conifg.json, cmdhistory, etc.
@ -117,20 +117,20 @@ class PappyConfig(object):
:Default: ``crypt``
.. data: crypt_file
.. data:: crypt_file
Encrypted archive of the temporary working directory ``crypt_dir``. Compressed as a
tar.bz2 archive if libraries available on the system, otherwise falls back to zip.
:Default: ``project.crypt``
.. data: crypt_session
.. data:: crypt_session
Boolean variable to determine whether pappy started in crypto mode
:Default: False
.. data: salt_len
.. data:: salt_len
Length of the nonce-salt value appended to the end of `crypt_file`

53
pappyproxy/http.py
View file

@ -129,6 +129,59 @@ def repeatable_parse_qs(s):
def request_by_id(reqid):
req = yield Request.load_request(str(reqid))
defer.returnValue(req)
@defer.inlineCallbacks
def async_submit_requests(reqs, mangle=False, save=False):
"""
async_submit_requests(reqs, mangle=False)
:param mangle: Whether to pass the requests through intercepting macros
:type mangle: Bool
:rtype: DeferredList
Submits a list of requests at the same time asynchronously.
Responses/unmangled versions will be attached to the request objects in the list.
Prints progress to stdout.
"""
print 'Submitting %d request(s)' % len(reqs)
dones = 0
errors = 0
list_deferred = defer.Deferred()
deferreds = []
for r in reqs:
d = r.async_submit(mangle=mangle)
deferreds.append(d)
# Really not the best way to do this. If one request hangs forever the whole thing will
# just hang in the middle
for d in deferreds:
try:
yield d
dones += 1
except Exception as e:
errors += 1
print e
finished = dones+errors
if finished % 30 == 0 or finished == len(reqs):
if errors > 0:
print '{0}/{1} complete with {3} errors ({2:.2f}%)'.format(finished, len(reqs), (float(finished)/len(reqs))*100, errors)
else:
print '{0}/{1} complete ({2:.2f}%)'.format(finished, len(reqs), (float(finished)/len(reqs))*100)
if finished == len(reqs):
list_deferred.callback(None)
if save:
for r in reqs:
yield r.async_deep_save()
@crochet.wait_for(timeout=180.0)
@defer.inlineCallbacks
def submit_requests(*args, **kwargs):
ret = yield async_submit_requests(*args, **kwargs)
defer.returnValue(ret)
##########
## Classes

102
pappyproxy/macros.py
View file

@ -7,9 +7,29 @@ import stat
from jinja2 import Environment, FileSystemLoader
from pappyproxy.pappy import session
from pappyproxy.util import PappyException
from pappyproxy.util import PappyException, load_reqlist
from twisted.internet import defer
## Template generating functions
# Must be declared before MacroTemplate class
@defer.inlineCallbacks
def gen_template_args_macro(args):
if len(args) > 0:
reqids = args[0]
reqs = yield load_reqlist(reqids)
else:
reqs = []
defer.returnValue(macro_from_requests(reqs))
def gen_template_generator_noargs(name):
def f(args):
subs = {}
subs['macro_name'] = 'Macro %d' % random.randint(1,99999999)
subs['short_name'] = ''
return MacroTemplate.fill_template(name, subs)
return f
class Macro(object):
"""
A class representing a macro that can perform a series of requests and add
@ -196,6 +216,66 @@ class FileInterceptMacro(InterceptMacro):
rsp = yield self.source.async_mangle_response(request)
defer.returnValue(rsp)
defer.returnValue(request.response)
class MacroTemplate(object):
_template_data = {
'macro': ('macro.py.template',
'Generic macro template',
'[reqids]',
'macro_{fname}.py',
gen_template_args_macro),
'intmacro': ('intmacro.py.template',
'Generic intercepting macro template',
'',
'int_{fname}.py',
gen_template_generator_noargs('intmacro')),
'modheader': ('macro_header.py.template',
'Modify a header in the request and the response if it exists.',
'',
'int_{fname}.py',
gen_template_generator_noargs('modheader')),
'resubmit': ('macro_resubmit.py.template',
'Resubmit all in-context requests',
'',
'macro_{fname}.py',
gen_template_generator_noargs('resubmit')),
}
@classmethod
def fill_template(cls, template, subs):
loader = FileSystemLoader(session.config.pappy_dir+'/templates')
env = Environment(loader=loader)
template = env.get_template(cls._template_data[template][0])
return template.render(zip=zip, **subs)
@classmethod
@defer.inlineCallbacks
def fill_template_args(cls, template, args=[]):
ret = cls._template_data[template][4](args)
if isinstance(ret, defer.Deferred):
ret = yield ret
defer.returnValue(ret)
@classmethod
def template_filename(cls, template, fname):
return cls._template_data[template][3].format(fname=fname)
@classmethod
def template_list(cls):
return [k for k, v in cls._template_data.iteritems()]
@classmethod
def template_description(cls, template):
return cls._template_data[template][1]
@classmethod
def template_argstring(cls, template):
return cls._template_data[template][2]
## Other functions
def load_macros(loc):
"""
@ -279,26 +359,8 @@ def macro_from_requests(reqs, short_name='', long_name=''):
subs['req_lines'] = req_lines
subs['req_params'] = req_params
loader = FileSystemLoader(session.config.pappy_dir+'/templates')
env = Environment(loader=loader)
template = env.get_template('macro.py.template')
return template.render(zip=zip, **subs)
return MacroTemplate.fill_template('macro', subs)
def gen_imacro(short_name='', long_name=''):
subs = {}
if long_name:
subs['macro_name'] = long_name
else:
random.seed()
subs['macro_name'] = 'Macro %d' % random.randint(1,99999999)
subs['short_name'] = short_name
loader = FileSystemLoader(session.config.pappy_dir+'/templates')
env = Environment(loader=loader)
template = env.get_template('intmacro.py.template')
return template.render(**subs)
@defer.inlineCallbacks
def mangle_request(request, intmacros):
"""

12
pappyproxy/plugin.py
View file

@ -168,6 +168,18 @@ def main_context_ids(*args, **kwargs):
"""
ret = yield async_main_context_ids(*args, **kwargs)
defer.returnValue(ret)
def add_to_history(req):
"""
Save a request to history without saving it to the data file. The request
will only be saved in memory, so when the program is exited or `clrmem`
is run, the request will be deleted.
:param req: The request to add to history
:type req: :class:`pappyproxy.http.Request`
"""
pappyproxy.http.Request.cache.add(req)
pappyproxy.context.reset_context_caches()
def run_cmd(cmd):
"""

68
pappyproxy/plugins/macrocmds.py
View file

@ -3,7 +3,7 @@ import pappyproxy
import shlex
from pappyproxy.plugin import active_intercepting_macros, add_intercepting_macro, remove_intercepting_macro
from pappyproxy.macros import load_macros, macro_from_requests, gen_imacro
from pappyproxy.macros import load_macros, macro_from_requests, MacroTemplate
from pappyproxy.util import PappyException, load_reqlist, autocomplete_startswith
from twisted.internet import defer
@ -12,6 +12,25 @@ loaded_int_macros = []
macro_dict = {}
int_macro_dict = {}
@defer.inlineCallbacks
def gen_macro_helper(line, template=None):
args = shlex.split(line)
if template is None:
fname = args[0]
template_name = args[1]
argstart = 2
else:
fname = args[0]
template_name = template
argstart = 1
if template_name not in MacroTemplate.template_list():
raise PappyException('%s is not a valid template name' % template_name)
script_str = yield MacroTemplate.fill_template_args(template_name, args[argstart:])
fname = MacroTemplate.template_filename(template_name, fname)
with open(fname, 'wc') as f:
f.write(script_str)
print 'Wrote script to %s' % fname
def load_macros_cmd(line):
"""
Load macros from a directory. By default loads macros in the current directory.
@ -193,34 +212,37 @@ def generate_macro(line):
Generate a macro script with request objects
Usage: generate_macro <name> [reqs]
"""
if line == '':
raise PappyException('Macro name is required')
args = shlex.split(line)
name = args[0]
if len(args) > 1:
reqs = yield load_reqlist(args[1])
else:
reqs = []
script_str = macro_from_requests(reqs)
fname = 'macro_%s.py' % name
with open(fname, 'wc') as f:
f.write(script_str)
print 'Wrote script to %s' % fname
yield gen_macro_helper(line, template='macro')
@crochet.wait_for(timeout=None)
@defer.inlineCallbacks
def generate_int_macro(line):
"""
Generate an intercepting macro script
Usage: generate_int_macro <name>
"""
yield gen_macro_helper(line, template='intmacro')
@crochet.wait_for(timeout=None)
@defer.inlineCallbacks
def generate_template_macro(line):
"""
Generate a macro from a built in template
Usage: generate_template_macro <fname> <template> [args]
"""
if line == '':
raise PappyException('Macro name is required')
args = shlex.split(line)
name = args[0]
script_str = gen_imacro()
fname = 'int_%s.py' % name
with open(fname, 'wc') as f:
f.write(script_str)
print 'Wrote script to %s' % fname
print 'Usage: gtma <fname> <template> [args]'
print 'Macro templates:'
templates = MacroTemplate.template_list()
templates.sort()
for t in templates:
if MacroTemplate.template_argstring(t):
print '"%s %s" - %s' % (t, MacroTemplate.template_argstring(t), MacroTemplate.template_description(t))
else:
print '"%s" - %s' % (t, MacroTemplate.template_description(t))
else:
yield gen_macro_helper(line)
@crochet.wait_for(timeout=None)
@defer.inlineCallbacks
@ -241,6 +263,7 @@ def load_cmds(cmd):
'rpy': (rpy, None),
'generate_int_macro': (generate_int_macro, None),
'generate_macro': (generate_macro, None),
'generate_template_macro': (generate_template_macro, None),
'list_int_macros': (list_int_macros, None),
'stop_int_macro': (stop_int_macro, complete_stop_int_macro),
'run_int_macro': (run_int_macro, complete_run_int_macro),
@ -251,6 +274,7 @@ def load_cmds(cmd):
#('rpy', ''),
('generate_int_macro', 'gima'),
('generate_macro', 'gma'),
('generate_template_macro', 'gtma'),
('list_int_macros', 'lsim'),
('stop_int_macro', 'sim'),
('run_int_macro', 'rim'),

77
pappyproxy/plugins/misc.py
View file

@ -1,3 +1,4 @@
import argparse
import crochet
import pappyproxy
import shlex
@ -7,8 +8,10 @@ from pappyproxy.colors import Colors, Styles, path_formatter, host_color, scode_
from pappyproxy.util import PappyException, remove_color, confirm, load_reqlist, Capturing
from pappyproxy.macros import InterceptMacro
from pappyproxy.requestcache import RequestCache
from pappyproxy.session import Session
from pappyproxy.pappy import session
from pappyproxy.plugin import add_intercepting_macro, remove_intercepting_macro
from pappyproxy.plugin import add_intercepting_macro, remove_intercepting_macro, add_to_history
from pappyproxy.http import async_submit_requests, Request
from twisted.internet import defer
from twisted.enterprise import adbapi
@ -190,6 +193,77 @@ def run_without_color(line):
def version(line):
import pappyproxy
print pappyproxy.__version__
@crochet.wait_for(timeout=180.0)
@defer.inlineCallbacks
def submit(line):
"""
Resubmit some requests, optionally with modified headers and cookies.
Usage: submit reqids [-h] [-m] [-u] [-p] [-c [COOKIES [COOKIES ...]]] [-d [HEADERS [HEADERS ...]]]
"""
parser = argparse.ArgumentParser(prog="submit", usage=submit.__doc__)
parser.add_argument('reqids')
parser.add_argument('-m', '--inmem', action='store_true', help='Store resubmitted requests in memory without storing them in the data file')
parser.add_argument('-u', '--unique', action='store_true', help='Only resubmit one request per endpoint (different URL parameters are different endpoints)')
parser.add_argument('-p', '--uniquepath', action='store_true', help='Only resubmit one request per endpoint (ignoring URL parameters)')
parser.add_argument('-c', '--cookies', nargs='*', help='Apply a cookie to requests before submitting')
parser.add_argument('-d', '--headers', nargs='*', help='Apply a header to requests before submitting')
args = parser.parse_args(shlex.split(line))
headers = {}
cookies = {}
if args.headers:
for h in args.headers:
k, v = h.split('=', 1)
headers[k] = v
if args.cookies:
for c in args.cookies:
k, v = c.split('=', 1)
cookies[k] = v
if args.unique and args.uniquepath:
raise PappyException('Both -u and -p cannot be given as arguments')
newsession = Session(cookie_vals=cookies, header_vals=headers)
reqs = yield load_reqlist(args.reqids)
if args.unique or args.uniquepath:
endpoints = set()
new_reqs = []
for r in reqs:
if args.unique:
s = r.url
else:
s = r.path
if not s in endpoints:
new_reqs.append(r.copy())
endpoints.add(s)
reqs = new_reqs
else:
reqs = [r.copy() for r in reqs]
for req in reqs:
newsession.apply_req(req)
conf_message = "You're about to submit %d requests, continue?" % len(reqs)
if not confirm(conf_message):
defer.returnValue(None)
for r in reqs:
r.tags.add('resubmitted')
if args.inmem:
yield async_submit_requests(reqs)
for req in reqs:
add_to_history(req)
else:
yield async_submit_requests(reqs, save=True)
def load_cmds(cmd):
cmd.set_cmds({
@ -202,6 +276,7 @@ def load_cmds(cmd):
'nocolor': (run_without_color, None),
'watch': (watch_proxy, None),
'version': (version, None),
'submit': (submit, None)
})
cmd.add_aliases([
#('rpy', ''),

7
pappyproxy/plugins/view.py
View file

@ -14,6 +14,7 @@ from pappyproxy.plugin import async_main_context_ids
from pappyproxy.colors import Colors, Styles, verb_color, scode_color, path_formatter, host_color
from pygments.formatters import TerminalFormatter
from pygments.lexers.data import JsonLexer
from pygments.lexers.html import XmlLexer
###################
## Helper functions
@ -103,6 +104,8 @@ def guess_pretty_print_fmt(msg):
return 'json'
elif 'www-form' in msg.headers['content-type']:
return 'form'
elif 'application/xml' in msg.headers['content-type']:
return 'xml'
return 'text'
def pretty_print_body(fmt, body):
@ -121,6 +124,10 @@ def pretty_print_body(fmt, body):
print s
elif fmt.lower() == 'text':
print body
elif fmt.lower() == 'xml':
import xml.dom.minidom
xml = xml.dom.minidom.parseString(body)
print pygments.highlight(xml.toprettyxml(), XmlLexer(), TerminalFormatter())
else:
raise PappyException('"%s" is not a valid format' % fmt)
except PappyException as e:

19
pappyproxy/session.py
View file

@ -34,7 +34,7 @@ class Session(object):
if k not in self.headers:
self.headers.append(k)
def _cookie_obj(k, v):
def _cookie_obj(self, k, v):
"""
Returns the value as a cookie object regardless of if the cookie is a string or a ResponseCookie.
"""
@ -44,7 +44,7 @@ class Session(object):
cookie_str = '%s=%s' % (k, v)
return ResponseCookie(cookie_str)
def _cookie_val(v):
def _cookie_val(self, v):
"""
Returns the value of the cookie regardless of if the value is a string or a ResponseCookie
"""
@ -76,7 +76,7 @@ class Session(object):
"""
for k, v in self.cookie_vals.iteritems():
val = self._cookie_obj(v)
val = self._cookie_obj(k, v)
rsp.set_cookie(val)
# Don't apply headers to responses
@ -115,13 +115,14 @@ class Session(object):
if header in self.headers:
self.header_vals[header] = req.headers[header]
def save_rsp(self, rsp, cookies=None):
def save_rsp(self, rsp, cookies=None, save_all=False):
"""
save_rsp(rsp, cookies=None)
Update the state of the session from the response. Only cookies can be
updated from a response. Additional values can be added to the whitelist
by passing in a list of values for the ``cookies`` parameter.
by passing in a list of values for the ``cookies`` parameter. If save_all
is given, all set cookies will be added to the session.
"""
if cookies:
for c in cookies:
@ -136,7 +137,11 @@ class Session(object):
self.cookie_vals[cookie] = rsp.cookies[cookie]
else:
for k, v in rsp.cookies.all_pairs():
if v.key in self.cookies:
if save_all:
self.cookie_vals[v.key] = v
if not v.key in self.cookies:
self.cookies.append(v.key)
elif v.key in self.cookies:
self.cookie_vals[v.key] = v
def set_cookie(key, val):
@ -171,5 +176,5 @@ class Session(object):
if not key in self.cookie_vals:
raise KeyError('Cookie is not stored in session.')
v = self.cookie_vals[key]
return self._cookie_obj(v)
return self._cookie_obj(key, v)

7
pappyproxy/templates/macro.py.template
View file

@ -1,7 +1,4 @@
from pappyproxy.http import Request, get_request, post_request, request_by_id
from pappyproxy.plugin import main_context_ids
from pappyproxy.context import set_tag
from pappyproxy.iter import *
{% include 'macroheader.py.template' %}
## Iterator cheat sheet:
# fuzz_path_trav() - Values for fuzzing path traversal
@ -11,8 +8,6 @@ from pappyproxy.iter import *
# common_usernames() - Common usernames
# fuzz_dirs() - Common web paths (ie /wp-admin)
MACRO_NAME = '{{macro_name}}'
SHORT_NAME = '{{short_name}}'
{% if req_lines %}
###########
## Requests

27
pappyproxy/templates/macro_header.py.template Normal file
View file

@ -0,0 +1,27 @@
from pappyproxy.session import Session
MACRO_NAME = '{{macro_name}}'
SHORT_NAME = '{{short_name}}'
runargs = []
def init(args):
global runargs
runargs = args
def modify_header(msg, key, val):
"""
Modifies the header in a request or a response if it already exists in
the message
"""
if key in msg.headers:
msg.headers[key] = val
def mangle_request(request):
global runargs
modify_header(request, 'headername', 'headerval')
return request
def mangle_response(request):
global runargs
modify_header(request.response, 'headername', 'headerval')
return request.response

34
pappyproxy/templates/macro_resubmit.py.template Normal file
View file

@ -0,0 +1,34 @@
import sys
{% include 'macroheader.py.template' %}
def run_macro(args):
# Get IDs of in-context requests
reqids = main_context_ids()
reqids.reverse() # Resubmit earliest first
reqs = []
# Create session jar (uncomment jar functions to use)
#jar = Session() # Create a cookie jar
# Iterate over each request and submit it
for rid in reqids:
print rid,
sys.stdout.flush()
r = request_by_id(rid)
r = r.copy()
#jar.apply_req(r) # Apply headers/cookies from the cookie jar
#####################
# Modify request here
r.submit()
#jar.save_rsp(r.response, save_all=True) # Update the cookie jar from the response
#r.save() # Save the request to the data file
reqs.append(r)
print ''
# Store the requests in memory
set_tag('resubmit', reqs)

8
pappyproxy/templates/macroheader.py.template Normal file
View file

@ -0,0 +1,8 @@
from pappyproxy.http import Request, get_request, post_request, request_by_id
from pappyproxy.plugin import main_context_ids
from pappyproxy.context import set_tag
from pappyproxy.session import Session
from pappyproxy.iter import *
MACRO_NAME = '{{macro_name}}'
SHORT_NAME = '{{short_name}}'

25
pappyproxy/tests/test_session.py
View file

@ -48,14 +48,14 @@ def test_session_cookieobj_basic(req, rsp):
assert req.headers['auth'] == 'bar'
assert 'auth' not in rsp.headers
def test_session_get_req(req):
def test_session_save_req(req):
req.headers['BasicAuth'] = 'asdfasdf'
req.headers['Host'] = 'www.myfavoritecolor.foobar'
req.cookies['session'] = 'foobar'
req.cookies['favorite_color'] = 'blue'
s = Session()
s.get_req(req, ['session'], ['BasicAuth'])
s.save_req(req, ['session'], ['BasicAuth'])
assert s.cookies == ['session']
assert s.headers == ['BasicAuth']
assert s.cookie_vals['session'].val == 'foobar'
@ -63,14 +63,14 @@ def test_session_get_req(req):
assert 'Host' not in s.headers
assert 'favorite_color' not in s.cookies
def test_session_get_rsp(rsp):
def test_session_save_rsp(rsp):
rsp.headers['BasicAuth'] = 'asdfasdf'
rsp.headers['Host'] = 'www.myfavoritecolor.foobar'
rsp.set_cookie(ResponseCookie('session=foobar; secure; path=/'))
rsp.set_cookie(ResponseCookie('favorite_color=blue; secure; path=/'))
s = Session()
s.get_rsp(rsp, ['session'])
s.save_rsp(rsp, ['session'])
assert s.cookies == ['session']
assert s.headers == []
assert s.cookie_vals['session'].key == 'session'
@ -99,6 +99,21 @@ def test_session_mixed(req, rsp):
r.start_line = 'HTTP/1.1 200 OK'
r.set_cookie(ResponseCookie('state=bazzers'))
r.set_cookie(ResponseCookie('session=buzzers'))
s.get_rsp(r)
s.save_rsp(r)
assert s.cookie_vals['session'].val == 'buzzers'
assert s.cookie_vals['state'].val == 'bazzers'
def test_session_save_all(req, rsp):
s = Session()
rsp.set_cookie(ResponseCookie('state=bazzers'))
rsp.set_cookie(ResponseCookie('session=buzzers'))
s.save_rsp(rsp, save_all=True)
assert s.cookies == ['state', 'session']
assert not 'state' in req.cookies
assert not 'session' in req.cookies
s.apply_req(req)
assert req.cookies['state'] == 'bazzers'
assert req.cookies['session'] == 'buzzers'