From 992edab3150c5cae82f4ab9e17261b519a4819e2 Mon Sep 17 00:00:00 2001
From: Rob Glew <rglew56@gmail.com>
Date: Thu, 14 Apr 2016 17:25:59 -0500
Subject: [PATCH] Version 0.2.11

---
 README.md                                     | 153 ++++++++++++++++++
 docs/source/contributing.rst                  |   2 -
 docs/source/pappyplugins.rst                  |  15 +-
 pappyproxy/__init__.py                        |   2 +-
 pappyproxy/config.py                          |  24 +--
 pappyproxy/http.py                            |  53 ++++++
 pappyproxy/macros.py                          | 102 +++++++++---
 pappyproxy/plugin.py                          |  12 ++
 pappyproxy/plugins/macrocmds.py               |  68 +++++---
 pappyproxy/plugins/misc.py                    |  77 ++++++++-
 pappyproxy/plugins/view.py                    |   7 +
 pappyproxy/session.py                         |  19 ++-
 pappyproxy/templates/macro.py.template        |   7 +-
 pappyproxy/templates/macro_header.py.template |  27 ++++
 .../templates/macro_resubmit.py.template      |  34 ++++
 pappyproxy/templates/macroheader.py.template  |   8 +
 pappyproxy/tests/test_session.py              |  25 ++-
 17 files changed, 551 insertions(+), 84 deletions(-)
 create mode 100644 pappyproxy/templates/macro_header.py.template
 create mode 100644 pappyproxy/templates/macro_resubmit.py.template
 create mode 100644 pappyproxy/templates/macroheader.py.template

diff --git a/README.md b/README.md
index 6fbdb9a..a8374f3 100644
--- a/README.md
+++ b/README.md
@@ -44,6 +44,8 @@ Table of Contents
       * [Useful Functions](#useful-functions)
     * [Intercepting Macros](#intercepting-macros)
       * [Enabling/Disabling Intercepting Macros](#enablingdisabling-intercepting-macros)
+    * [Macro Templates](#macro-templates)
+    * [Resubmitting Groups of Requests](#resubmitting-groups-of-requests)
     * [Logging](#logging)
     * [Additional Commands and Features](#additional-commands-and-features)
       * [Response streaming](#response-streaming)
@@ -54,6 +56,7 @@ Table of Contents
     * [Using an HTTP Proxy](#using-an-http-proxy)
     * [Using a SOCKS Proxy](#using-a-socks-proxy)
     * [Transparent Host Redirection](#transparent-host-redirection)
+    * [Project File Encryption](#project-file-encryption)
     * [FAQ](#faq)
       * [Why does my request have an id of --?!?!](#why-does-my-request-have-an-id-of---)
     * [Boring, Technical Stuff](#boring-technical-stuff)
@@ -698,6 +701,13 @@ def run_macro(args):
 
 If you enter in a value for `SHORT_NAME`, you can use it as a shortcut to run that macro. So if in a macro you set `SHORT_NAME='tm'` you can run it by running `pappy> rma tm`.
 
+Remember, you can use the wildcard to generate a macro with all in-context requests:
+
+```
+# Generate a macro with all in-context requests
+pappy> gma allreqs *
+```
+
 ### Passing Arguments to Macros
 
 When you run the macro, any additional command line arguments will be passed to the run_macro function in the `args` argument. For example, if you run your macro using
@@ -802,6 +812,7 @@ def run_macro(args):
 | get_request(url, url_params={}) | Returns a Request object that contains a GET request to the given url with the given url params |
 | post_request(url, post_params={}, url_params={}) | Returns a Request object that contains a POST request to the given url with the given url and post params |
 | request_by_id(reqid) | Get a request object from its id. |
+| main_context_ids() | Returns a list of the IDs that are in the current context. Use this for macros that need to act on every in-context request. For example, it can be used in a macro to resubmit a set of requests. |
 
 Intercepting Macros
 -------------------
@@ -903,6 +914,77 @@ You can use the following commands to start/stop intercepting macros
 | `lim` | `list_int_macros`, `lsim` | List all enabled/disabled intercepting macros |
 | `gima <name>` | `generate_int_macro`, `gima` | Generate an intercepting macro with the given name. |
 
+Macro Templates
+---------------
+Pappy also includes some other templates for generating macros. They can be generated with the `gtma` command. You can then modify the generated macros to do what you want. For example, you could modify the resubmit macro to get a new session token before submitting each request. Using a template can save you from writing boilerplate for commonly created macros.
+
+Examples:
+```
+# The same as gma foo 1,2,3
+pappy> gtma foo macro 1,2,3
+Wrote script to macro_foo.py
+
+# Generate a macro that resubmits all in-context requests
+pappy> gtma suball resubmit
+Wrote script to macro_suball.py
+
+# Generate an intercepting macro that modifies headers as they pass through the proxy
+pappy> gtma headers modheader
+Wrote script to int_headers.py
+```
+
+Command information:
+| Command | Aliases | Description |
+|:--------|:--------|:------------|
+| `gtma <name> <template name> [template arguments]` | `generate_template_macro`, `gtma` | Generate a macro using a template. |
+
+Available macro templates:
+
+| Name | Arguments | Description |
+|:-----|:----------|:------------|
+| `macro` | `[reqids]` | The template used to generate macros from request IDs. |
+| `intmacro` | None | The template used to generate an intercepting macro. |
+| `modheader` | None | Create an intercepting macro that modifies a header in the request or response. |
+| `resubmit` | None | Create a macro that resubmits all in-context requests. Includes commented out code to maintain session state using a cookie jar. |
+
+Resubmitting Groups of Requests
+-------------------------------
+You can use the `submit` request to resubmit requests. It is suggested that you use this command with a heavy use of filters and using the wildcard (`*`) to submit all in-context requests. Be careful submitting everything in context, remember, if you have to Ctl-C out you will close Pappy and lose all in-memory requests!
+
+| Command | Aliases | Description |
+|:--------|:--------|:------------|
+| `submit reqids [-m] [-u] [-p] [-c [COOKIES [COOKIES ...]]] [-d [HEADERS [HEADERS ...]]]` | `submit` | Submit a given set of requests. Request IDs must be passed in as the first argument. The wildcard (`*`) selector can be very useful. Resubmitted requests are given a `resubmitted` tag. See the arguments section for information on the arguments. |
+
+### Useful Filters For Selecting Requests to Resubmit
+
+* `before` and `after` to select requests in a time range. You can use the `after` filter on the most recent request, browse the site, then use the `before` filter to select a continuous browsing session.
+* `verb` if you only want to select GET requests
+* `path ct logout` to avoid logging out
+
+### Arguments
+
+There are a few simple parameters you can pass to the command to modify requests. These behave like normal command parameters in the terminal. If you need something more complex (ie getting CSRF tokens, refreshing the session token, reacting to Set-Cookie headers, etc.) you should consider writing a macro and using the `main_context_ids` function to get in-context IDs then iterating over them and handling them however you want.
+
+| Argument | Description |
+|:---------|:------------|
+| `-c <cookie>=<val>` | Modify a cookie on each request before submitting. Can pass more than one pair to the flag to modify more than one cookie. Does not encode the cookie values in any way. |
+| `-d <header>=<val>` | Modify a header on each request before submitting. Can pass more than one pair to the flag to modify more than one header. |
+| `-m` | Store requests in memory instead of saving to the data file. |
+| `-u` | Only submit one request per endpoint. Will count requests with the same path but different url params as *different* endpoints. |
+| `-p` | Only submit one request per endpoint. Will count requests with the same path but different url params as *the same* endpoints. |
+
+Examples:
+```
+# Resubmit all in-context requests with the SESSIONID cookie set to 1234 and SESSIONSTATE set to {'admin'='true'}
+pappy> submit * -c SESSIONID=1234 SESSIONSTATE=%7B%27admin%27%3A%27true%27%7D
+
+# Resubmit all in-context requests with the User-Agent header set to "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" then store them in memory
+pappy> submit * -m -h "User-Agent=Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+
+# Submit requests 123, 124, and 125 with a new user agent and new session cookies and store the submitted requests in memory
+pappy> submit 123,124,125 -h "User-Agent=Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" -c SESSIONID=1234 SESSIONSTATE=%7B%27admin%27%3A%27true%27%7D
+```
+
 Logging
 -------
 You can watch in real-time what requests are going through the proxy. Verbosisty defaults to 1 which just states when connections are made/lost and some information on what is happening. If verbosity is set to 3, it includes all the data which is sent through the proxy and processed. It will print the raw response from the server, what it decodes it to, etc. Even if you don't run this command, all the information is stored in the dubug directory (the directory is cleared every start though!)
@@ -1110,6 +1192,72 @@ Or if you’re going to YOLO it do the same thing then listen on port 80/443 dir
 
 Pappy will automatically use this host to make the connection and forward the request to the new server.
 
+Project File Encryption
+-----------------------
+
+Pappy includes some basic features for automatically compressing and encrypting your project directory with a password. However, before I go into details on how to do this, I need to make one thing clear.
+
+**Don't rely on Pappy to encrypt confidential information. Use a dedicated encryption product to encrypt your project directory instead.**
+
+Other commercial and large open source crypto projects have had a much larger number of people look at their crypto implementations and are less likely to have errors in their implementation. However, for cases where you don't need enterprise level security or if you just want your project stored in a single password-protected file instead of a directory, Pappy's got you covered.
+
+Here is how Pappy's project encryption works:
+
+* Open a project by running Pappy with the `-c` flag
+* Pappy creates a `crypt/` directory in the current directory and changes the working directory into it
+* Do work as normal. You can use other tools in the created `crypt/` directory
+* When you quit Pappy, the file is compressed and encrypted with the provided password
+* The project directory is deleted
+
+Unfortunately, if Pappy hard crashes the files will not be cleaned up. However, if you start Pappy and it notices a `crypt/` directory, it will attempt to use it as the project directory and create a new encrypted project file upon exiting.
+
+Here is an of the usage:
+
+```
+$ pwd
+/tmp/exampleproj
+$ ls
+$ pappy -c example.proj
+Copying default config to ./config.json
+Proxy is listening on port 8000
+pappy> !pwd
+/tmp/exampleproj/crypt
+
+# Switch to another terminal window
+/templates/ $ echo "Hello World" >  /tmp/exampleproj/crypt/hello.txt
+
+# Back to Pappy
+pappy> !cat hello.txt
+Hello World
+pappy> exit
+Enter a password:
+$ ls
+example.proj
+```
+
+Then to work on the project again:
+
+```
+$ pappy -c example.proj
+Enter a password:
+Proxy is listening on port 8000
+pappy> !ls
+config.json     data.db         hello.txt
+pappy>
+```
+
+Example of recovering after crash:
+```
+$ ls
+crypt           project.archive
+$ pappy -c test.proj
+Proxy is listening on port 8000
+pappy> exit
+Enter a password:
+$ ls
+test.proj
+```
+
 FAQ
 ---
 
@@ -1130,6 +1278,11 @@ Changelog
 ---------
 The boring part of the readme
 
+* 0.2.11
+    * Project directory compression/encryption. Thanks, onizenso!
+    * Add `submit` command
+    * Add macro templates
+    * Add header replacement and resubmit in-context requests macro templates
 * 0.2.10
     * Add wildcard support for requests that can take in multiple request ids
     * Update dump_response to dump multiple requests at the same time
diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst
index 3240fd1..b6cff2f 100644
--- a/docs/source/contributing.rst
+++ b/docs/source/contributing.rst
@@ -35,8 +35,6 @@ Anyways, here's some ideas for things you could implement:
     Make it easy to pass a request to SQLMap to check for SQLi. Make sure you can configure which fields you do/don't want tested and by default just give either "yes it looks like SQLi" or "no it doesn't look like SQLi"
 * Additional macro templates
     Write some commands for generating additional types of macros. For example let people generate an intercepting macro that does search/replace or modifies a header. Save as much typing as possible for common actions.
-* Show requests/responses real-time as they go through the proxy
-    Let people watch requests as they pass through the proxy. It's fine to implement this as an intercepting macro since people watching the requests aren't going to notice response streaming being disabled.
 * Vim plugin to make editing HTTP messages easier
     Implement some functionality to make editing HTTP messages easier. It would be great to have a plugin to automatically add to vim when using the interceptor/repeater to make editing requests easier. Look at burp's request editor and try to implement anything you miss from it.
 * Request Diff
diff --git a/docs/source/pappyplugins.rst b/docs/source/pappyplugins.rst
index 182cc9d..466e815 100644
--- a/docs/source/pappyplugins.rst
+++ b/docs/source/pappyplugins.rst
@@ -272,10 +272,10 @@ Using defer.inlineCallbacks With a Command
 .. note::
    This tutorial won't tell you how to use inlineCallbacks in general. Type "twisted inline callbacks" into google to figure out what they are. This is mainly just a reminder to use the ``crochet`` wrapper for console commands and warning you that some functions may return deferreds that you may have to deal with.
 
-Since you're writing a plugin, you'll probably be using functions which return a deferred. And to keep things readable, you'll want to use the ``defer.inlineCallbacks`` function wrapper. Unfortunately, you can't bind async functions to commands. Luckily, there's a library called `crochet <https://pypi.python.org/pypi/crochet>`_ which lets you add another wrapper to the function that lets it be used like a blocking function. Rather than talking about it, let's write a plugin to call :func:`pappyproxy.console.load_reqlist` to print out some requests' hosts. Let's start by pretending it's a normal function::
+Since you're writing a plugin, you'll probably be using functions which return a deferred. And to keep things readable, you'll want to use the ``defer.inlineCallbacks`` function wrapper. Unfortunately, you can't bind async functions to commands. Luckily, there's a library called `crochet <https://pypi.python.org/pypi/crochet>`_ which lets you add another wrapper to the function that lets it be used like a blocking function. Rather than talking about it, let's write a plugin to call :func:`pappyproxy.util.load_reqlist` to print out some requests' hosts. Let's start by pretending it's a normal function::
 
     import shlex
-    from pappyproxy.console import load_reqlist
+    from pappyproxy.util import load_reqlist
     
     def print_hosts(line):
         args = shlex.split(line)
@@ -309,10 +309,10 @@ And we run it::
     iteration over non-sequence
     pappy>
   
-Iteration over a non-sequence? what? Well, :func:`pappyproxy.console.load_reqlist` doesn't actually return a list of requests. It returns a deferred which returns a list of requests. I'm not going into the details (look up some stuff on using inline callbacks with Twisted if you want more info), but the way to fix it is to slap an ``inlineCallbacks`` wrapper on the function and ``yield`` the result of the function. Now it looks like this::
+Iteration over a non-sequence? what? Well, :func:`pappyproxy.util.load_reqlist` doesn't actually return a list of requests. It returns a deferred which returns a list of requests. I'm not going into the details (look up some stuff on using inline callbacks with Twisted if you want more info), but the way to fix it is to slap an ``inlineCallbacks`` wrapper on the function and ``yield`` the result of the function. Now it looks like this::
 
     import shlex
-    from pappyproxy.console import load_reqlist
+    from pappyproxy.util import load_reqlist
     from twisted.internet import defer
     
     @defer.inlineCallbacks
@@ -336,7 +336,7 @@ However, the console assumes that any functions it calls will be blocking. As a
 
     import shlex
     import crochet
-    from pappyproxy.console import load_reqlist
+    from pappyproxy.util import load_reqlist
     from twisted.internet import defer
     
     @crochet.wait_for(timeout=None)
@@ -394,7 +394,7 @@ Here is an example plugin for storing the user-agent (if it exists) in the ``plu
     import shlex
     from twisted.internet import defer
     
-    from pappyproxy.console import load_reqlist
+    from pappyproxy.util import load_reqlist
     from pappyproxy.plugin import main_context
     from pappyproxy.util import PappyException
     
@@ -435,8 +435,7 @@ Here is an example plugin for storing the user-agent (if it exists) in the ``plu
 
 Useful Functions
 ----------------
-* Load a request by id: :func:`pappyproxy.http.Request.load_request`
-* Create a filter from a filter string: :func:`pappyproxy.context.Filter.from_filter_string`
+See :mod:`pappyproxy.plugin` and :mod:`pappyproxy.util` for useful functions
 
 Built In Plugins As Examples
 ============================
diff --git a/pappyproxy/__init__.py b/pappyproxy/__init__.py
index e913eb4..c5b683c 100644
--- a/pappyproxy/__init__.py
+++ b/pappyproxy/__init__.py
@@ -1 +1 @@
-__version__ = '0.2.10'
+__version__ = '0.2.11'
diff --git a/pappyproxy/config.py b/pappyproxy/config.py
index 32dfeae..6619c30 100644
--- a/pappyproxy/config.py
+++ b/pappyproxy/config.py
@@ -46,14 +46,14 @@ class PappyConfig(object):
     
     :Default: None
     
-    .. data: listeners
+    .. data:: listeners
     
     The list of active listeners. It is a list of tuples of the format (port, interface)
     Not modifiable after startup. Configured in the ``config.json`` file for the project.
     
     :Default: ``[(8000, '127.0.0.1')]``
     
-    .. data: socks_proxy
+    .. data:: socks_proxy
     
     Details for a SOCKS proxy. It is a dict with the following key/values::
     
@@ -66,7 +66,7 @@ class PappyConfig(object):
     
     :Default: ``null``
     
-    .. data: http_proxy
+    .. data:: http_proxy
 
     Details for an upstream HTTP proxy. It is a dict with the following key/values::
     
@@ -77,37 +77,37 @@ class PappyConfig(object):
     
     If null, no proxy will be used.
 
-    .. data: plugin_dirs
+    .. data:: plugin_dirs
     
     List of directories that plugins are loaded from. Not modifiable.
     
     :Default: ``['{DATA_DIR}/plugins', '{PAPPY_DIR}/plugins']``
     
-    .. data: save_history
+    .. data:: save_history
     
     Whether command history should be saved to a file/loaded at startup.
     
     :Default: True
     
-    .. data: config_dict
+    .. data:: config_dict
     
     The dictionary read from config.json. When writing plugins, use this to load
     configuration options for your plugin.
     
-    .. data: global_config_dict
+    .. data:: global_config_dict
     
     The dictionary from ~/.pappy/global_config.json. It contains settings for
     Pappy that are specific to the current computer. Avoid putting settings here,
     especially if it involves specific projects.
      
-    .. data: archive 
+    .. data:: archive 
 
     Project archive compressed as a ``tar.bz2`` archive if libraries available on the system, 
     otherwise falls back to zip archive.
 
     :Default: ``project.archive``
 
-    .. data: crypt_dir 
+    .. data:: crypt_dir 
 
     Temporary working directory to unpack an encrypted project archive. Directory
     will contain copies of normal startup files, e.g. conifg.json, cmdhistory, etc.
@@ -117,20 +117,20 @@ class PappyConfig(object):
 
     :Default: ``crypt``
 
-    .. data: crypt_file
+    .. data:: crypt_file
 
     Encrypted archive of the temporary working directory ``crypt_dir``. Compressed as a
     tar.bz2 archive if libraries available on the system, otherwise falls back to zip.
 
     :Default: ``project.crypt``
      
-    .. data: crypt_session
+    .. data:: crypt_session
     
     Boolean variable to determine whether pappy started in crypto mode
     
     :Default: False
 
-    .. data: salt_len
+    .. data:: salt_len
 
     Length of the nonce-salt value appended to the end of `crypt_file`
 
diff --git a/pappyproxy/http.py b/pappyproxy/http.py
index 89ceb48..0e765bd 100644
--- a/pappyproxy/http.py
+++ b/pappyproxy/http.py
@@ -129,6 +129,59 @@ def repeatable_parse_qs(s):
 def request_by_id(reqid):
     req = yield Request.load_request(str(reqid))
     defer.returnValue(req)
+    
+@defer.inlineCallbacks
+def async_submit_requests(reqs, mangle=False, save=False):
+    """
+    async_submit_requests(reqs, mangle=False)
+    :param mangle: Whether to pass the requests through intercepting macros
+    :type mangle: Bool
+    :rtype: DeferredList
+
+    Submits a list of requests at the same time asynchronously.
+    Responses/unmangled versions will be attached to the request objects in the list.
+    Prints progress to stdout.
+    """
+    print 'Submitting %d request(s)' % len(reqs)
+
+    dones = 0
+    errors = 0
+    list_deferred = defer.Deferred()
+
+    deferreds = []
+    for r in reqs:
+        d = r.async_submit(mangle=mangle)
+        deferreds.append(d)
+
+    # Really not the best way to do this. If one request hangs forever the whole thing will
+    # just hang in the middle
+    for d in deferreds:
+        try:
+            yield d
+            dones += 1
+        except Exception as e:
+            errors += 1
+            print e
+
+        finished = dones+errors
+        
+        if finished % 30 == 0 or finished == len(reqs):
+            if errors > 0:
+                print '{0}/{1} complete with {3} errors ({2:.2f}%)'.format(finished, len(reqs), (float(finished)/len(reqs))*100, errors)
+            else:
+                print '{0}/{1} complete ({2:.2f}%)'.format(finished, len(reqs), (float(finished)/len(reqs))*100)
+        if finished == len(reqs):
+            list_deferred.callback(None)
+
+    if save:
+        for r in reqs:
+            yield r.async_deep_save()
+
+@crochet.wait_for(timeout=180.0)
+@defer.inlineCallbacks
+def submit_requests(*args, **kwargs):
+    ret = yield async_submit_requests(*args, **kwargs)
+    defer.returnValue(ret)
 
 ##########
 ## Classes
diff --git a/pappyproxy/macros.py b/pappyproxy/macros.py
index b6a56f0..a91ebc1 100644
--- a/pappyproxy/macros.py
+++ b/pappyproxy/macros.py
@@ -7,9 +7,29 @@ import stat
 
 from jinja2 import Environment, FileSystemLoader
 from pappyproxy.pappy import session
-from pappyproxy.util import PappyException
+from pappyproxy.util import PappyException, load_reqlist
 from twisted.internet import defer
 
+## Template generating functions
+# Must be declared before MacroTemplate class
+    
+@defer.inlineCallbacks
+def gen_template_args_macro(args):
+    if len(args) > 0:
+        reqids = args[0]
+        reqs = yield load_reqlist(reqids)
+    else:
+        reqs = []
+    defer.returnValue(macro_from_requests(reqs))
+
+def gen_template_generator_noargs(name):
+    def f(args):
+        subs = {}
+        subs['macro_name'] = 'Macro %d' % random.randint(1,99999999)
+        subs['short_name'] = ''
+        return MacroTemplate.fill_template(name, subs)
+    return f
+
 class Macro(object):
     """
     A class representing a macro that can perform a series of requests and add
@@ -196,6 +216,66 @@ class FileInterceptMacro(InterceptMacro):
             rsp = yield self.source.async_mangle_response(request)
             defer.returnValue(rsp)
         defer.returnValue(request.response)
+        
+class MacroTemplate(object):
+    _template_data = {
+        'macro': ('macro.py.template',
+                  'Generic macro template',
+                  '[reqids]',
+                  'macro_{fname}.py',
+                  gen_template_args_macro),
+
+        'intmacro': ('intmacro.py.template',
+                     'Generic intercepting macro template',
+                     '',
+                     'int_{fname}.py',
+                     gen_template_generator_noargs('intmacro')),
+
+        'modheader': ('macro_header.py.template',
+                      'Modify a header in the request and the response if it exists.',
+                      '',
+                      'int_{fname}.py',
+                      gen_template_generator_noargs('modheader')),
+
+        'resubmit': ('macro_resubmit.py.template',
+                     'Resubmit all in-context requests',
+                     '',
+                     'macro_{fname}.py',
+                     gen_template_generator_noargs('resubmit')),
+    }
+
+    @classmethod
+    def fill_template(cls, template, subs):
+        loader = FileSystemLoader(session.config.pappy_dir+'/templates')
+        env = Environment(loader=loader)
+        template = env.get_template(cls._template_data[template][0])
+        return template.render(zip=zip, **subs)
+
+    @classmethod
+    @defer.inlineCallbacks
+    def fill_template_args(cls, template, args=[]):
+        ret = cls._template_data[template][4](args)
+        if isinstance(ret, defer.Deferred):
+            ret = yield ret
+        defer.returnValue(ret)
+
+    @classmethod
+    def template_filename(cls, template, fname):
+        return cls._template_data[template][3].format(fname=fname)
+
+    @classmethod
+    def template_list(cls):
+        return [k for k, v in cls._template_data.iteritems()]
+    
+    @classmethod
+    def template_description(cls, template):
+        return cls._template_data[template][1]
+
+    @classmethod
+    def template_argstring(cls, template):
+        return cls._template_data[template][2]
+
+## Other functions
 
 def load_macros(loc):
     """
@@ -279,26 +359,8 @@ def macro_from_requests(reqs, short_name='', long_name=''):
     subs['req_lines'] = req_lines
     subs['req_params'] = req_params
 
-    loader = FileSystemLoader(session.config.pappy_dir+'/templates')
-    env = Environment(loader=loader)
-    template = env.get_template('macro.py.template')
-    return template.render(zip=zip, **subs)
+    return MacroTemplate.fill_template('macro', subs)
 
-def gen_imacro(short_name='', long_name=''):
-    subs = {}
-    if long_name:
-        subs['macro_name'] = long_name
-    else:
-        random.seed()
-        subs['macro_name'] = 'Macro %d' % random.randint(1,99999999)
-
-    subs['short_name'] = short_name
-
-    loader = FileSystemLoader(session.config.pappy_dir+'/templates')
-    env = Environment(loader=loader)
-    template = env.get_template('intmacro.py.template')
-    return template.render(**subs)
-    
 @defer.inlineCallbacks
 def mangle_request(request, intmacros):
     """
diff --git a/pappyproxy/plugin.py b/pappyproxy/plugin.py
index 4b9fadc..0b96e16 100644
--- a/pappyproxy/plugin.py
+++ b/pappyproxy/plugin.py
@@ -168,6 +168,18 @@ def main_context_ids(*args, **kwargs):
     """
     ret = yield async_main_context_ids(*args, **kwargs)
     defer.returnValue(ret)
+    
+def add_to_history(req):
+    """
+    Save a request to history without saving it to the data file. The request
+    will only be saved in memory, so when the program is exited or `clrmem`
+    is run, the request will be deleted.
+
+    :param req: The request to add to history
+    :type req: :class:`pappyproxy.http.Request`
+    """
+    pappyproxy.http.Request.cache.add(req)
+    pappyproxy.context.reset_context_caches()
 
 def run_cmd(cmd):
     """
diff --git a/pappyproxy/plugins/macrocmds.py b/pappyproxy/plugins/macrocmds.py
index 8f1950a..6b8b108 100644
--- a/pappyproxy/plugins/macrocmds.py
+++ b/pappyproxy/plugins/macrocmds.py
@@ -3,7 +3,7 @@ import pappyproxy
 import shlex
 
 from pappyproxy.plugin import active_intercepting_macros, add_intercepting_macro, remove_intercepting_macro
-from pappyproxy.macros import load_macros, macro_from_requests, gen_imacro
+from pappyproxy.macros import load_macros, macro_from_requests, MacroTemplate
 from pappyproxy.util import PappyException, load_reqlist, autocomplete_startswith
 from twisted.internet import defer
 
@@ -12,6 +12,25 @@ loaded_int_macros = []
 macro_dict = {}
 int_macro_dict = {}
 
+@defer.inlineCallbacks
+def gen_macro_helper(line, template=None):
+    args = shlex.split(line)
+    if template is None:
+        fname = args[0]
+        template_name = args[1]
+        argstart = 2
+    else:
+        fname = args[0]
+        template_name = template
+        argstart = 1
+    if template_name not in MacroTemplate.template_list():
+        raise PappyException('%s is not a valid template name' % template_name)
+    script_str = yield MacroTemplate.fill_template_args(template_name, args[argstart:])
+    fname = MacroTemplate.template_filename(template_name, fname)
+    with open(fname, 'wc') as f:
+        f.write(script_str)
+    print 'Wrote script to %s' % fname
+
 def load_macros_cmd(line):
     """
     Load macros from a directory. By default loads macros in the current directory.
@@ -193,34 +212,37 @@ def generate_macro(line):
     Generate a macro script with request objects
     Usage: generate_macro <name> [reqs]
     """
-    if line == '':
-        raise PappyException('Macro name is required')
-    args = shlex.split(line)
-    name = args[0]
-    if len(args) > 1:
-        reqs = yield load_reqlist(args[1])
-    else:
-        reqs = []
-    script_str = macro_from_requests(reqs)
-    fname = 'macro_%s.py' % name
-    with open(fname, 'wc') as f:
-        f.write(script_str)
-    print 'Wrote script to %s' % fname
+    yield gen_macro_helper(line, template='macro')
 
+@crochet.wait_for(timeout=None)
+@defer.inlineCallbacks
 def generate_int_macro(line):
     """
     Generate an intercepting macro script
     Usage: generate_int_macro <name>
     """
+    yield gen_macro_helper(line, template='intmacro')
+
+@crochet.wait_for(timeout=None)
+@defer.inlineCallbacks
+def generate_template_macro(line):
+    """
+    Generate a macro from a built in template
+    Usage: generate_template_macro <fname> <template> [args]
+    """
     if line == '':
-        raise PappyException('Macro name is required')
-    args = shlex.split(line)
-    name = args[0]
-    script_str = gen_imacro()
-    fname = 'int_%s.py' % name
-    with open(fname, 'wc') as f:
-        f.write(script_str)
-    print 'Wrote script to %s' % fname
+        print 'Usage: gtma <fname> <template> [args]'
+        print 'Macro templates:'
+
+        templates = MacroTemplate.template_list()
+        templates.sort()
+        for t in templates:
+            if MacroTemplate.template_argstring(t):
+                print '"%s %s" - %s' % (t, MacroTemplate.template_argstring(t), MacroTemplate.template_description(t))
+            else:
+                print '"%s" - %s' % (t, MacroTemplate.template_description(t))
+    else:
+        yield gen_macro_helper(line)
 
 @crochet.wait_for(timeout=None)
 @defer.inlineCallbacks
@@ -241,6 +263,7 @@ def load_cmds(cmd):
         'rpy': (rpy, None),
         'generate_int_macro': (generate_int_macro, None),
         'generate_macro': (generate_macro, None),
+        'generate_template_macro': (generate_template_macro, None),
         'list_int_macros': (list_int_macros, None),
         'stop_int_macro': (stop_int_macro, complete_stop_int_macro),
         'run_int_macro': (run_int_macro, complete_run_int_macro),
@@ -251,6 +274,7 @@ def load_cmds(cmd):
         #('rpy', ''),
         ('generate_int_macro', 'gima'),
         ('generate_macro', 'gma'),
+        ('generate_template_macro', 'gtma'),
         ('list_int_macros', 'lsim'),
         ('stop_int_macro', 'sim'),
         ('run_int_macro', 'rim'),
diff --git a/pappyproxy/plugins/misc.py b/pappyproxy/plugins/misc.py
index 22fcbb8..32ae5c7 100644
--- a/pappyproxy/plugins/misc.py
+++ b/pappyproxy/plugins/misc.py
@@ -1,3 +1,4 @@
+import argparse
 import crochet
 import pappyproxy
 import shlex
@@ -7,8 +8,10 @@ from pappyproxy.colors import Colors, Styles, path_formatter, host_color, scode_
 from pappyproxy.util import PappyException, remove_color, confirm, load_reqlist, Capturing
 from pappyproxy.macros import InterceptMacro
 from pappyproxy.requestcache import RequestCache
+from pappyproxy.session import Session
 from pappyproxy.pappy import session
-from pappyproxy.plugin import add_intercepting_macro, remove_intercepting_macro
+from pappyproxy.plugin import add_intercepting_macro, remove_intercepting_macro, add_to_history
+from pappyproxy.http import async_submit_requests, Request
 from twisted.internet import defer
 from twisted.enterprise import adbapi
 
@@ -190,6 +193,77 @@ def run_without_color(line):
 def version(line):
     import pappyproxy
     print pappyproxy.__version__
+    
+@crochet.wait_for(timeout=180.0)
+@defer.inlineCallbacks
+def submit(line):
+    """
+    Resubmit some requests, optionally with modified headers and cookies.
+
+    Usage: submit reqids [-h] [-m] [-u] [-p] [-c [COOKIES [COOKIES ...]]] [-d [HEADERS [HEADERS ...]]]
+    """
+    
+    parser = argparse.ArgumentParser(prog="submit", usage=submit.__doc__)
+    parser.add_argument('reqids')
+    parser.add_argument('-m', '--inmem', action='store_true', help='Store resubmitted requests in memory without storing them in the data file')
+    parser.add_argument('-u', '--unique', action='store_true', help='Only resubmit one request per endpoint (different URL parameters are different endpoints)')
+    parser.add_argument('-p', '--uniquepath', action='store_true', help='Only resubmit one request per endpoint (ignoring URL parameters)')
+    parser.add_argument('-c', '--cookies', nargs='*', help='Apply a cookie to requests before submitting')
+    parser.add_argument('-d', '--headers', nargs='*', help='Apply a header to requests before submitting')
+    args = parser.parse_args(shlex.split(line))
+
+    headers = {}
+    cookies = {}
+
+    if args.headers:
+        for h in args.headers:
+            k, v = h.split('=', 1)
+            headers[k] = v
+
+    if args.cookies:
+        for c in args.cookies:
+            k, v = c.split('=', 1)
+            cookies[k] = v
+
+    if args.unique and args.uniquepath:
+        raise PappyException('Both -u and -p cannot be given as arguments')
+
+    newsession = Session(cookie_vals=cookies, header_vals=headers)
+    
+    reqs = yield load_reqlist(args.reqids)
+
+    if args.unique or args.uniquepath:
+        endpoints = set()
+        new_reqs = []
+        for r in reqs:
+            if args.unique:
+                s = r.url
+            else:
+                s = r.path
+
+            if not s in endpoints:
+                new_reqs.append(r.copy())
+                endpoints.add(s)
+        reqs = new_reqs
+    else:
+        reqs = [r.copy() for r in reqs]
+
+    for req in reqs:
+        newsession.apply_req(req)
+
+    conf_message = "You're about to submit %d requests, continue?" % len(reqs)
+    if not confirm(conf_message):
+        defer.returnValue(None)
+
+    for r in reqs:
+        r.tags.add('resubmitted')
+
+    if args.inmem:
+        yield async_submit_requests(reqs)
+        for req in reqs:
+            add_to_history(req)
+    else:
+        yield async_submit_requests(reqs, save=True)
         
 def load_cmds(cmd):
     cmd.set_cmds({
@@ -202,6 +276,7 @@ def load_cmds(cmd):
         'nocolor': (run_without_color, None),
         'watch': (watch_proxy, None),
         'version': (version, None),
+        'submit': (submit, None)
     })
     cmd.add_aliases([
         #('rpy', ''),
diff --git a/pappyproxy/plugins/view.py b/pappyproxy/plugins/view.py
index 2da7575..73e08a7 100644
--- a/pappyproxy/plugins/view.py
+++ b/pappyproxy/plugins/view.py
@@ -14,6 +14,7 @@ from pappyproxy.plugin import async_main_context_ids
 from pappyproxy.colors import Colors, Styles, verb_color, scode_color, path_formatter, host_color
 from pygments.formatters import TerminalFormatter
 from pygments.lexers.data import JsonLexer
+from pygments.lexers.html import XmlLexer
 
 ###################
 ## Helper functions
@@ -103,6 +104,8 @@ def guess_pretty_print_fmt(msg):
             return 'json'
         elif 'www-form' in msg.headers['content-type']:
             return 'form'
+        elif 'application/xml' in msg.headers['content-type']:
+            return 'xml'
     return 'text'
     
 def pretty_print_body(fmt, body):
@@ -121,6 +124,10 @@ def pretty_print_body(fmt, body):
                 print s
         elif fmt.lower() == 'text':
             print body
+        elif fmt.lower() == 'xml':
+            import xml.dom.minidom
+            xml = xml.dom.minidom.parseString(body)
+            print pygments.highlight(xml.toprettyxml(), XmlLexer(), TerminalFormatter())
         else:
             raise PappyException('"%s" is not a valid format' % fmt)
     except PappyException as e:
diff --git a/pappyproxy/session.py b/pappyproxy/session.py
index ac5b26b..0e8bb95 100644
--- a/pappyproxy/session.py
+++ b/pappyproxy/session.py
@@ -34,7 +34,7 @@ class Session(object):
                 if k not in self.headers:
                     self.headers.append(k)
 
-    def _cookie_obj(k, v):
+    def _cookie_obj(self, k, v):
         """
         Returns the value as a cookie object regardless of if the cookie is a string or a ResponseCookie.
         """
@@ -44,7 +44,7 @@ class Session(object):
             cookie_str = '%s=%s' % (k, v)
             return ResponseCookie(cookie_str)
 
-    def _cookie_val(v):
+    def _cookie_val(self, v):
         """
         Returns the value of the cookie regardless of if the value is a string or a ResponseCookie
         """
@@ -76,7 +76,7 @@ class Session(object):
         """
 
         for k, v in self.cookie_vals.iteritems():
-            val = self._cookie_obj(v)
+            val = self._cookie_obj(k, v)
             rsp.set_cookie(val)
         # Don't apply headers to responses
 
@@ -115,13 +115,14 @@ class Session(object):
                 if header in self.headers:
                     self.header_vals[header] = req.headers[header]
 
-    def save_rsp(self, rsp, cookies=None):
+    def save_rsp(self, rsp, cookies=None, save_all=False):
         """
         save_rsp(rsp, cookies=None)
 
         Update the state of the session from the response. Only cookies can be
         updated from a response. Additional values can be added to the whitelist
-        by passing in a list of values for the ``cookies`` parameter.
+        by passing in a list of values for the ``cookies`` parameter. If save_all
+        is given, all set cookies will be added to the session.
         """
         if cookies:
             for c in cookies:
@@ -136,7 +137,11 @@ class Session(object):
                     self.cookie_vals[cookie] = rsp.cookies[cookie]
         else:
             for k, v in rsp.cookies.all_pairs():
-                if v.key in self.cookies:
+                if save_all:
+                    self.cookie_vals[v.key] = v
+                    if not v.key in self.cookies:
+                        self.cookies.append(v.key)
+                elif v.key in self.cookies:
                     self.cookie_vals[v.key] = v
 
     def set_cookie(key, val):
@@ -171,5 +176,5 @@ class Session(object):
         if not key in self.cookie_vals:
             raise KeyError('Cookie is not stored in session.')
         v = self.cookie_vals[key]
-        return self._cookie_obj(v)
+        return self._cookie_obj(key, v)
 
diff --git a/pappyproxy/templates/macro.py.template b/pappyproxy/templates/macro.py.template
index e280481..6d5a8fa 100644
--- a/pappyproxy/templates/macro.py.template
+++ b/pappyproxy/templates/macro.py.template
@@ -1,7 +1,4 @@
-from pappyproxy.http import Request, get_request, post_request, request_by_id
-from pappyproxy.plugin import main_context_ids 
-from pappyproxy.context import set_tag
-from pappyproxy.iter import *
+{% include 'macroheader.py.template' %}
 
 ## Iterator cheat sheet:
 # fuzz_path_trav() - Values for fuzzing path traversal
@@ -11,8 +8,6 @@ from pappyproxy.iter import *
 # common_usernames() - Common usernames
 # fuzz_dirs() - Common web paths (ie /wp-admin)
 
-MACRO_NAME = '{{macro_name}}'
-SHORT_NAME = '{{short_name}}'
 {% if req_lines %}
 ###########
 ## Requests
diff --git a/pappyproxy/templates/macro_header.py.template b/pappyproxy/templates/macro_header.py.template
new file mode 100644
index 0000000..67c3f4c
--- /dev/null
+++ b/pappyproxy/templates/macro_header.py.template
@@ -0,0 +1,27 @@
+from pappyproxy.session import Session
+
+MACRO_NAME = '{{macro_name}}'
+SHORT_NAME = '{{short_name}}'
+runargs = []
+
+def init(args):
+    global runargs
+    runargs = args
+    
+def modify_header(msg, key, val):
+    """
+    Modifies the header in a request or a response if it already exists in
+    the message
+    """
+    if key in msg.headers:
+        msg.headers[key] = val
+
+def mangle_request(request):
+    global runargs
+    modify_header(request, 'headername', 'headerval')
+    return request
+
+def mangle_response(request):
+    global runargs
+    modify_header(request.response, 'headername', 'headerval')
+    return request.response
diff --git a/pappyproxy/templates/macro_resubmit.py.template b/pappyproxy/templates/macro_resubmit.py.template
new file mode 100644
index 0000000..27037e6
--- /dev/null
+++ b/pappyproxy/templates/macro_resubmit.py.template
@@ -0,0 +1,34 @@
+import sys
+{% include 'macroheader.py.template' %}
+
+def run_macro(args):
+    # Get IDs of in-context requests
+    reqids = main_context_ids()
+    reqids.reverse() # Resubmit earliest first
+    reqs = []
+
+    # Create session jar (uncomment jar functions to use)
+    #jar = Session() # Create a cookie jar
+
+    # Iterate over each request and submit it
+    for rid in reqids:
+        print rid,
+        sys.stdout.flush()
+        r = request_by_id(rid)
+        r = r.copy()
+
+        #jar.apply_req(r) # Apply headers/cookies from the cookie jar
+
+        #####################
+        # Modify request here
+        
+        r.submit()
+        #jar.save_rsp(r.response, save_all=True) # Update the cookie jar from the response
+
+        #r.save() # Save the request to the data file
+        reqs.append(r)
+
+    print ''
+
+    # Store the requests in memory
+    set_tag('resubmit', reqs)
diff --git a/pappyproxy/templates/macroheader.py.template b/pappyproxy/templates/macroheader.py.template
new file mode 100644
index 0000000..37a4161
--- /dev/null
+++ b/pappyproxy/templates/macroheader.py.template
@@ -0,0 +1,8 @@
+from pappyproxy.http import Request, get_request, post_request, request_by_id
+from pappyproxy.plugin import main_context_ids 
+from pappyproxy.context import set_tag
+from pappyproxy.session import Session
+from pappyproxy.iter import *
+
+MACRO_NAME = '{{macro_name}}'
+SHORT_NAME = '{{short_name}}'
diff --git a/pappyproxy/tests/test_session.py b/pappyproxy/tests/test_session.py
index 0d6847c..3863c9e 100644
--- a/pappyproxy/tests/test_session.py
+++ b/pappyproxy/tests/test_session.py
@@ -48,14 +48,14 @@ def test_session_cookieobj_basic(req, rsp):
     assert req.headers['auth'] == 'bar'
     assert 'auth' not in rsp.headers
 
-def test_session_get_req(req):
+def test_session_save_req(req):
     req.headers['BasicAuth'] = 'asdfasdf'
     req.headers['Host'] = 'www.myfavoritecolor.foobar'
     req.cookies['session'] = 'foobar'
     req.cookies['favorite_color'] = 'blue'
 
     s = Session()
-    s.get_req(req, ['session'], ['BasicAuth'])
+    s.save_req(req, ['session'], ['BasicAuth'])
     assert s.cookies == ['session']
     assert s.headers == ['BasicAuth']
     assert s.cookie_vals['session'].val == 'foobar'
@@ -63,14 +63,14 @@ def test_session_get_req(req):
     assert 'Host' not in s.headers
     assert 'favorite_color' not in s.cookies
 
-def test_session_get_rsp(rsp):
+def test_session_save_rsp(rsp):
     rsp.headers['BasicAuth'] = 'asdfasdf'
     rsp.headers['Host'] = 'www.myfavoritecolor.foobar'
     rsp.set_cookie(ResponseCookie('session=foobar; secure; path=/'))
     rsp.set_cookie(ResponseCookie('favorite_color=blue; secure; path=/'))
 
     s = Session()
-    s.get_rsp(rsp, ['session'])
+    s.save_rsp(rsp, ['session'])
     assert s.cookies == ['session']
     assert s.headers == []
     assert s.cookie_vals['session'].key == 'session'
@@ -99,6 +99,21 @@ def test_session_mixed(req, rsp):
     r.start_line = 'HTTP/1.1 200 OK'
     r.set_cookie(ResponseCookie('state=bazzers'))
     r.set_cookie(ResponseCookie('session=buzzers'))
-    s.get_rsp(r)
+    s.save_rsp(r)
     assert s.cookie_vals['session'].val == 'buzzers'
     assert s.cookie_vals['state'].val == 'bazzers'
+
+def test_session_save_all(req, rsp):
+    s = Session()
+    rsp.set_cookie(ResponseCookie('state=bazzers'))
+    rsp.set_cookie(ResponseCookie('session=buzzers'))
+    s.save_rsp(rsp, save_all=True)
+
+    assert s.cookies == ['state', 'session']
+
+    assert not 'state' in req.cookies
+    assert not 'session' in req.cookies
+    s.apply_req(req)
+    assert req.cookies['state'] == 'bazzers'
+    assert req.cookies['session'] == 'buzzers'
+