Drastically restructured the compression and crypto features

Converted the crypto and compression plugins to core features, and
added the utility variables and functions to the Config class in
``config.py``. Added helper functions in PappySession class in
``pappy.py`` to enable the user to pass in an encrypted project archive.
Next moving to testing and debugging!

.gitignore

@@ -10,4 +10,5 @@ TAGS
 config.json
 build/*
 *.egg-info/*
-.#*
+.#*
+*notes*

pappyproxy/compress.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+import crochet
+import glob
+import pappyproxy
+
+import zipfile
+import tarfile
+
+# This is a gross hack, please help
+bz2 = None
+try:
+    import bz2
+except:
+    print "BZ2 not installed on your system"
+
+from base64 import b64encode, b64decode
+from os import getcwd, sep, path, urandom 
+from pappyproxy.plugins.misc import CryptoCompressUtils as ccu
+
+class Compress(object):
+    def __init__(self, sessconfig):
+        self.config = sessconfig
+        self.zip_archive = sessconfig.archive
+        self.bz2_archive = sessconfig.archive
+
+    def compress_project():
+        if bz2:
+            tar_project()
+        else:
+            zip_project()
+    
+    def decompress_project():
+        if bz2:
+            untar_project()
+        else:
+            unzip_project()
+    
+    def zip_project():
+        """
+        Zip project files
+    
+        Using append mode (mode='a') will create a zip archive
+        if none exists in the project.
+        """
+        try:
+            zf = zipfile.ZipFile(self.zip_archive, mode="a")
+            project_files = self.config.get_project_files() 
+            for pf in project_files:
+                zf.write(pf)
+            zf.close()
+        except e:
+            raise PappyException("Error creating the zipfile", e)
+        pass
+    
+    def unzip_project():
+        """
+        Extract project files from decrypted zip archive.
+        Initially checks the zip archive's magic number and
+        attempts to extract pappy.json to validate integrity 
+        of the zipfile.
+        """
+        if not zipfile.is_zipfile(self.zip_archive):
+            raise PappyException("Project archive corrupted.")
+    
+        zf = zipfile.ZipFile(self.zip_archive)
+    
+        try:
+            zf.extract("config.json")
+        except e:
+            raise PappyException("Project archive contents corrupted. Error: ", e)
+    
+        zf.extractall()
+    
+    def tar_project():
+        if tarfile.is_tarfile(self.bz2_archive):
+            archive = tarfile.open(self.bz2_archive, 'w:bz2')
+            project_files = self.config.get_project_files()
+    
+            # Read files line by line to accomodate larger files, e.g. the project database
+            for pf in project_files:
+                archive.add(pf)
+    	archive.close()
+    
+    def untar_project():
+        if tarfile.is_tarfile(self.bz2_archive):
+            # Attempt to read the first 16 bytes of the archive
+            # Raise exception if there is a failure
+    	project_files = self.config.get_project_files()
+            try:
+                with tarfile.open(self.bz2_archive, "r:bz2") as archive:
+                    for pf in project_files:
+                        archive.add(pf)
+            except e:
+                raise PappyException("Project archive contents corrupted. Error: ", e

pappyproxy/config.py

@@ -98,6 +98,51 @@ class PappyConfig(object):
     The dictionary from ~/.pappy/global_config.json. It contains settings for
     Pappy that are specific to the current computer. Avoid putting settings here,
     especially if it involves specific projects.
+
+    .. data: archive 
+
+    Project archive compressed as a ``tar.bz2`` archive if libraries available on the system, 
+    otherwise falls back to zip archive.
+
+    :Default: 'project.archive'
+
+    .. data: crypt_dir 
+
+    Temporary working directory to unpack an encrypted project archive. Directory
+    will contain copies of normal startup files, e.g. conifg.json, cmdhistory, etc.
+    On exiting pappy, entire directory will be compressed into an archive and encrypted.
+    Compressed as a tar.bz2 archive if libraries available on the system, 
+    otherwise falls back to zip.
+
+    :Default: 'crypt'
+
+    .. data: crypt_file
+
+    Encrypted archive of the temporary working directory ``crypt_dir``. Compressed as a
+    tar.bz2 archive if libraries available on the system, otherwise falls back to zip.
+
+    :Default: 'project.crypt'
+
+    .. data: crypt_mode
+
+    Boolean value to determine whether project is being decrypted or encrypted, during 
+    start-up and tear-down respectively.
+
+    .. data: salt
+
+    Nonce value used for key derivation. Generated by reading 16 bytes
+    from /dev/urandom.
+
+    :Default: ``os.urandom(16)``
+
+    .. data: salt_file
+
+    Clear-text file containing the salt generated for key derivation. A new salt
+    will be generated each time the project is encrypted. After successfully
+    decrypting the project file (``project.crypt``), the salt file (``project.salt``)
+    will be deleted.
+
+    :Default: ``project.salt``
     """
 
     def __init__(self):
@@ -125,6 +170,13 @@ class PappyConfig(object):
 
         self.config_dict = {}
         self.global_config_dict = {}
+
+        self.archive = 'project.archive' 
+        self.crypt_dir = os.path.join(os.getcwd(), 'crypt')
+        self.crypt_file = 'project.crypt'
+        self.crypt_mode = None
+        self.salt = os.urandom(16)
+        self.salt_file = 'project.salt'
     
     def get_default_config(self):
         default_config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
@@ -133,6 +185,15 @@ class PappyConfig(object):
             settings = json.load(f)
         return settings
 
+    def get_project_files(self):
+      file_glob = glob.glob('*')
+      pp = os.path.join(os.getcwd()) 
+      project_files = [pp+f for f in file_glob if os.path.isfile(pp+f)]
+      project_files.remove(self.salt_file)
+      project_files.remove(self.crypt_file)
+      return project_files 
+        
+
     @staticmethod
     def _parse_proxy_login(conf):
         proxy = {}

pappyproxy/crypto.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+
+import crochet
+import glob
+import os
+import pappyproxy
+import scrypt
+import shutil
+import twisted
+
+from . import compress
+from base64 import b64encode, b64decode
+from cryptography import Fernet
+from twisted.internet import reactor, defer
+
+class Crypto(object):
+    def __init__(self, sessconfig):
+        self.config = sessconfig
+        self.archive = self.config.archive 
+        self.compressor = compress.Compress(sessconfig)
+
+    def encrypt_project(passwd):
+        """
+        Compress and encrypt the project files, deleting clear-text files afterwards
+        """
+        # Derive the key
+        key = crypto_ramp_up(passwd)
+
+        # Instantiate the crypto module
+        fern = Fernet(key)
+   
+        # Create project archive and crypto archive 
+        self.compressor.compress_project()
+        archive_file = open(self.archive, 'rb') 
+        archive_crypt = open(self.config.crypt_file, 'wb')
+    
+        # Encrypt the archive read as a bytestring
+        crypt_token = fern.encrypt(archive_file)
+        archive_crypt.write(crypt_token)
+    
+        # Delete clear-text files
+        delete_clear_files()
+        
+        # Leave crypto working directory
+        os.chdir('../')
+    
+    @defer.inlineCallbacks
+    def decrypt_project(passwd):
+        """
+        Decrypt and decompress the project files
+        """
+
+        # Create crypto working directory 
+        crypto_path = os.path.join(os.getcwd(), pappy_config.crypt_dir)
+        os.mkdir(crypto_path)
+
+        if os.path.isfile(self.config.crypt_file):
+            # Derive the key
+            key = crypto_ramp_up(passwd)
+            fern = Fernet(key)
+
+            # Decrypt the project archive
+            archive_crypt = open(self.config.crypt_file, 'rb')
+            archive = fern.decrypt(archive_crypt)
+
+            shutil.move(archive, crypto_path)
+            os.chdir(crypto_path)
+            self.compressor.decompress_project()
+        else:
+            project_files = self.config.get_project_files()
+            for pf in project_files:
+                shutil.copy2(pf, crypto_path)
+            os.chdir(crypto_path)
+            
+    
+    def crypto_ramp_up(passwd):
+        salt = ""
+        if os.path.isfile(self.config.salt_file):
+            salt = get_salt()
+        else:
+            salt = create_salt_file()
+        key = derive_key(passwd, salt)
+        return key
+    
+    def delete_clear_files():
+        """
+        Deletes all clear-text files left in the project directory.
+        """
+        project_files = self.config.get_project_files()
+        for pf in project_files:
+    	    os.remove(pf)
+    	
+    def delete_crypt_files():
+        """
+        Deletes all encrypted-text files in the project directory.
+        Forces generation of new salt after opening and closing the project.
+        Adds security in the case of a one-time compromise of the system.
+        """
+        os.remove(self.config.salt_file)
+        os.remove(self.config.crypt_file)
+    
+    def create_salt_file():
+        self.config.salt = urandom(16)
+        salt_file = open(self.config.salt_file, 'wb')
+        salt_file.write(self.config.salt)
+        salt_file.close()
+        return salt
+    
+    def get_salt():
+        try:
+            salt_file = open(self.config.salt_file, 'rb')
+            salt = salt_file.readline()
+        except:
+            raise PappyException("Unable to read pappy.salt")
+        return salt
+    
+    def get_password():
+        """
+        Retrieve password from the user. Raise an exception if the 
+        password is not capable of utf-8 encoding.
+        """
+        encoded_passwd = ""
+        try:
+            passwd = raw_input("Enter a password: ")
+            encode_passwd = passwd.encode("utf-8")
+        except:
+            raise PappyException("Invalid password, try again")
+        return encoded_passwd
+    
+    def derive_key(passwd, salt):
+        """
+        Derive a key sufficient for use as a cryptographic key
+        used to encrypt the project (currently: cryptography.Fernet).
+    
+        cryptography.Fernet utilizes AES-CBC-128, requiring a 32-byte key.
+        Parameter notes from the py-scrypt source-code:
+        https://bitbucket.org/mhallin/py-scrypt/
+    
+        Compute scrypt(password, salt, N, r, p, buflen).
+    
+        The parameters r, p, and buflen must satisfy r * p < 2^30 and
+        buflen <= (2^32 - 1) * 32. The parameter N must be a power of 2
+        greater than 1. N, r and p must all be positive.
+    
+        Notes for Python 2:
+          - `password` and `salt` must be str instances
+          - The result will be a str instance
+    
+        Notes for Python 3:
+          - `password` and `salt` can be both str and bytes. If they are str
+            instances, they wil be encoded with utf-8.
+          - The result will be a bytes instance
+    
+        Exceptions raised:
+          - TypeError on invalid input
+          - scrypt.error if scrypt failed
+        """
+    
+        derived_key = ""
+        try:
+            dkey = scrypt.hash(passwd, salt, bufflen=32)
+        except e:
+            raise PappyException("Error deriving the key: ", e)
+        return derived_key

pappyproxy/pappy.py

@@ -21,7 +21,9 @@ import signal
 
 from . import comm
 from . import config
+from . import compress
 from . import context
+from . import crypto
 from . import http
 from .console import ProxyCmd
 from twisted.enterprise import adbapi
@@ -62,6 +64,8 @@ class PappySession(object):
         self.dbpool = None
         self.delete_data_on_quit = False
         self.ports = None
+        self.crypto = Crypto(sessconfig)
+        self.password = None
 
     @defer.inlineCallbacks
     def start(self):
@@ -138,12 +142,25 @@ class PappySession(object):
         # Add cleanup to defer
         self.complete_defer = deferToThread(self.cons.cmdloop)
         self.complete_defer.addCallback(self.cleanup)
-            
+
+    @defer.inlineCallbacks
+    def encrypt(self):
+        if self.password:
+            self.crypto.encrypt_project(self.password)     
+        else:
+            self.password = self.crypto.get_password()
+            self.crypto.encrypt_project(self.password)     
+
+    @defer.inlineCallbacks
+    def decrypt(self):
+        self.password = self.crypto.get_password()
+        self.crypto.decrypt_project(self.password)
+                
     @defer.inlineCallbacks
     def cleanup(self, ignored=None):
         for port in self.ports:
             yield port.stopListening()
-
+    
         if self.delete_data_on_quit:
             print 'Deleting temporary datafile'
             os.remove(self.config.datafile)
@@ -153,6 +170,7 @@ def parse_args():
 
     parser = argparse.ArgumentParser(description='An intercepting proxy for testing web applications.')
     parser.add_argument('-l', '--lite', help='Run the proxy in "lite" mode', action='store_true')
+    parser.add_argument('-c', '--crypt', type=str, nargs='?', help='Start pappy in "crypto" mode, optionally supply a name for the encrypted project archive [CRYPT]')
 
     args = parser.parse_args(sys.argv[1:])
     settings = {}
@@ -162,6 +180,13 @@ def parse_args():
     else:
         settings['lite'] = False
 
+    if args.crypt:
+        settings['crypt'] = args.crypt 
+    elif args.crypt == "":
+        settings['crypt'] = 'project.crypt'
+    else:
+        settings['crypt'] = None
+
     return settings
 
 def set_text_factory(conn):
@@ -189,7 +214,12 @@ def main():
     session = PappySession(pappy_config)
     signal.signal(signal.SIGINT, inturrupt_handler)
 
-    if settings['lite']:
+    if settings['crypt']:
+        session.decrypt()
+        conf_settings = pappy_config.load_from_file('./config.json')
+        pappy_config.global_load_from_file()
+        session.delete_data_on_quit = False
+    elif settings['lite']:
         conf_settings = pappy_config.get_default_config()
         conf_settings['debug_dir'] = None
         conf_settings['debug_to_file'] = False