Release 0.0.1

See README diff for changes

pappyproxy/schema/schema_1.py

@@ -0,0 +1,54 @@
+from twisted.internet import defer
+
+"""
+Schema v1
+
+Description:
+The initial schema for the first version of the proxy. It includes the creation
+of the schema_meta table and other data tables.
+"""
+
+update_queries = [
+    """
+    CREATE TABLE responses (
+        id             INTEGER   PRIMARY KEY  AUTOINCREMENT,
+        full_response  BLOB  NOT NULL,
+        unmangled_id   INTEGER   REFERENCES responses(id)
+    );
+    """,
+
+    """
+    CREATE TABLE requests (
+        id                INTEGER   PRIMARY KEY  AUTOINCREMENT,
+        full_request      BLOB      NOT NULL,
+        tag               TEXT,
+        submitted         INTEGER   NOT NULL,
+        response_id       INTEGER   REFERENCES responses(id),
+        unmangled_id      INTEGER   REFERENCES requests(id),
+        start_datetime    TEXT,
+        end_datetime      TEXT
+    );
+    """,
+    
+    """
+    CREATE TABLE schema_meta (
+        version INTEGER NOT NULL
+    );
+    """,
+    
+    """
+    CREATE TABLE scope (
+        filter_order    INTEGER  NOT NULL,
+        filter_string   TEXT  NOT NULL
+    );
+    """,
+
+    """
+    INSERT INTO schema_meta (version) VALUES (1);
+    """,
+]
+
+@defer.inlineCallbacks
+def update(dbpool):
+    for query in update_queries:
+        yield dbpool.runQuery(query)

pappyproxy/schema/schema_2.py

@@ -0,0 +1,37 @@
+from pappyproxy import http
+from twisted.internet import defer
+
+"""
+Schema v2
+
+Description:
+Adds support for specifying the port of a request and specify its port. This
+lets requests that have the port/ssl settings specified in the CONNECT request
+maintain that information.
+"""
+
+update_queries = [
+    """
+    ALTER TABLE requests ADD COLUMN port INTEGER;
+    """,
+
+    """
+    ALTER TABLE requests ADD COLUMN is_ssl INTEGER;
+    """,
+
+    """
+    UPDATE schema_meta SET version=2;
+    """,
+]
+
+@defer.inlineCallbacks
+def update(dbpool):
+    for query in update_queries:
+        yield dbpool.runQuery(query)
+
+    # Load each request and save them again for any request that specified a port
+    # or protocol in the host header.
+    http.init(dbpool)
+    reqs = yield http.Request.load_from_filters([])
+    for req in reqs:
+        yield req.deep_save()

pappyproxy/schema/update.py

@@ -0,0 +1,58 @@
+import os
+import glob
+import imp
+
+from twisted.internet import reactor
+from twisted.enterprise import adbapi
+from twisted.internet import defer
+
+@defer.inlineCallbacks
+def get_schema_version(dbpool):
+    schema_exists = yield dbpool.runQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='schema_meta';")
+    if not schema_exists:
+        # If we get an empty list, we have no schema
+        defer.returnValue(0)
+    else:
+        schema_version_result = yield dbpool.runQuery("SELECT version FROM schema_meta;")
+
+        # There should only be one row in the meta table
+        assert(len(schema_version_result) == 1)
+
+        # Return the retrieved version
+        version = schema_version_result[0][0]
+        defer.returnValue(version)
+
+def add_schema_files(schemas):
+    # Finds and imports all schema_*.py files into the list
+    module_files = glob.glob(os.path.dirname(os.path.abspath(__file__)) + "/schema_*.py")
+    for mod in module_files:
+        module_name = os.path.basename(os.path.splitext(mod)[0])
+        newmod = imp.load_source('%s'%module_name, mod)
+        schemas.append( (module_name, newmod) )
+
+@defer.inlineCallbacks
+def update_schema(dbpool):
+    # Update the database schema to the latest version
+    schema_version = yield get_schema_version(dbpool)
+    if schema_version == 0:
+        verbose_update = False
+    else:
+        verbose_update = True
+    schemas = []
+    add_schema_files(schemas)
+    schemas = sorted(schemas, key=lambda tup: tup[0])
+    for i in range(schema_version, len(schemas)):
+        # schemas[0] is v1, schemas[1] is v2, etc
+        if verbose_update:
+            print "Updating datafaile schema to version %d" % (i+1)
+        yield schemas[i][1].update(dbpool)
+    
+@defer.inlineCallbacks
+def main():
+    dbpool = adbapi.ConnectionPool("sqlite3", "data.db", check_same_thread=False)
+    yield update_schema(dbpool)
+    reactor.stop()
+        
+if __name__ == '__main__':
+    reactor.callWhenRunning(main)
+    reactor.run()