Merge pull request #7 from onizenso/master

Encryption Feature

.gitignore

@@ -11,3 +11,4 @@ config.json
 build/*
 *.egg-info/*
 .#*
+*notes*

pappyproxy/Makefile

@@ -16,3 +16,7 @@ test-proxy:
 
 test-comm:
 	py.test -v -rw --twisted tests/test_comm.py
+
+test-crypto:
+	py.test -v -rw --twisted tests/test_crypto.py
+      

pappyproxy/compress.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+import crochet
+import glob
+import pappyproxy
+
+import zipfile
+import tarfile
+
+try:
+    import bz2
+except ImportError:
+    bz2 = None
+    print "BZ2 not installed on your system"
+
+from base64 import b64encode, b64decode
+from os import getcwd, sep, path, urandom
+
+
+class Compress(object):
+    def __init__(self, sessconfig):
+        self.config = sessconfig
+        self.zip_archive = sessconfig.archive
+        self.bz2_archive = sessconfig.archive
+
+    def compress_project(self):
+        if bz2:
+            self.tar_project()
+        else:
+            self.zip_project()
+
+    def decompress_project(self):
+        if bz2:
+            self.untar_project()
+        else:
+            self.unzip_project()
+
+    def zip_project(self):
+        """
+        Zip project files
+
+        Using append mode (mode='a') will create a zip archive
+        if none exists in the project.
+        """
+        try:
+            zf = zipfile.ZipFile(self.zip_archive, mode="a")
+            zf.write(self.config.crypt_dir)
+            zf.close()
+        except zipfile.LargeZipFile as e:
+            raise PappyException("Project zipfile too large. Error: ", e)
+
+    def unzip_project(self):
+        """
+        Extract project files from decrypted zip archive.
+        Initially checks the zip archive's magic number and
+        attempts to extract pappy.json to validate integrity
+        of the zipfile.
+        """
+        if not zipfile.is_zipfile(self.zip_archive):
+            raise PappyException("Project archive corrupted.")
+
+        zf = zipfile.ZipFile(self.zip_archive)
+
+        try:
+            zf.extract("config.json")
+        except zipfile.BadZipfile as e:
+            raise PappyException("Zip archive corrupted. Error: ", e)
+
+        zf.extractall()
+
+    def tar_project(self):
+        archive = tarfile.open(self.bz2_archive, 'w:bz2')
+
+        archive.add(self.config.crypt_dir)
+        archive.close()
+
+    def untar_project(self):
+        if tarfile.is_tarfile(self.bz2_archive):
+            # Raise exception if there is a failure
+            try:
+                with tarfile.open(self.bz2_archive, "r:bz2") as archive:
+                    archive.extractall()
+            except tarfile.ExtractError as e:
+                raise PappyException("Tar archive corrupted. Error: ", e)

pappyproxy/config.py

@@ -1,3 +1,4 @@
+import glob
 import json
 import os
 import shutil
@@ -98,6 +99,42 @@ class PappyConfig(object):
     The dictionary from ~/.pappy/global_config.json. It contains settings for
     Pappy that are specific to the current computer. Avoid putting settings here,
     especially if it involves specific projects.
+     
+    .. data: archive 
+
+    Project archive compressed as a ``tar.bz2`` archive if libraries available on the system, 
+    otherwise falls back to zip archive.
+
+    :Default: ``project.archive``
+
+    .. data: crypt_dir 
+
+    Temporary working directory to unpack an encrypted project archive. Directory
+    will contain copies of normal startup files, e.g. conifg.json, cmdhistory, etc.
+    On exiting pappy, entire directory will be compressed into an archive and encrypted.
+    Compressed as a tar.bz2 archive if libraries available on the system, 
+    otherwise falls back to zip.
+
+    :Default: ``crypt``
+
+    .. data: crypt_file
+
+    Encrypted archive of the temporary working directory ``crypt_dir``. Compressed as a
+    tar.bz2 archive if libraries available on the system, otherwise falls back to zip.
+
+    :Default: ``project.crypt``
+     
+    .. data: crypt_session
+    
+    Boolean variable to determine whether pappy started in crypto mode
+    
+    :Default: False
+
+    .. data: salt_len
+
+    Length of the nonce-salt value appended to the end of `crypt_file`
+
+    :Default: 16
     """
 
     def __init__(self):
@@ -126,6 +163,13 @@ class PappyConfig(object):
         self.config_dict = {}
         self.global_config_dict = {}
 
+        self.archive = 'project.archive' 
+        self.debug = False
+        self.crypt_dir = 'crypt'
+        self.crypt_file = 'project.crypt'
+        self.crypt_session = False
+        self.salt_len = 16
+    
     def get_default_config(self):
         default_config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                            'default_user_config.json')
@@ -133,6 +177,17 @@ class PappyConfig(object):
             settings = json.load(f)
         return settings
 
+    def get_project_files(self):
+        file_glob = glob.glob('*')
+        pp = os.getcwd() + os.sep 
+        project_files = [pp+f for f in file_glob if os.path.isfile(pp+f)]
+        
+        if self.crypt_file in project_files:
+            project_files.remove(self.crypt_file)
+        
+        return project_files 
+        
+
     @staticmethod
     def _parse_proxy_login(conf):
         proxy = {}

pappyproxy/console.py

@@ -42,13 +42,15 @@ class ProxyCmd(cmd2.Cmd):
         self._cmds = {}
         self._aliases = {}
 
-        atexit.register(self.save_histfile)
-        readline.set_history_length(self.session.config.histsize)
-        if os.path.exists('cmdhistory'):
-            if self.session.config.histsize != 0:
-                readline.read_history_file('cmdhistory')
-            else:
-                os.remove('cmdhistory')
+        # Only read and save history when not in crypto mode
+        if not self.session.config.crypt_session:
+            atexit.register(self.save_histfile)
+            readline.set_history_length(self.session.config.histsize)
+            if os.path.exists('cmdhistory'):
+                if self.session.config.histsize != 0:
+                    readline.read_history_file('cmdhistory')
+                else:
+                    os.remove('cmdhistory')
 
         cmd2.Cmd.__init__(self, *args, **kwargs)
 
@@ -110,10 +112,12 @@ class ProxyCmd(cmd2.Cmd):
         raise AttributeError(attr)
 
     def save_histfile(self):
-        # Write the command to the history file
-        if self.session.config.histsize != 0:
-            readline.set_history_length(self.session.config.histsize)
-            readline.write_history_file('cmdhistory')
+        # Only write to file if not in crypto mode
+        if not self.session.config.crypt_session:
+            # Write the command to the history file
+            if self.session.config.histsize != 0:
+                readline.set_history_length(self.session.config.histsize)
+                readline.write_history_file('cmdhistory')
     
     def get_names(self):
         # Hack to get cmd to recognize do_/etc functions as functions for things

pappyproxy/crypto.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+
+import crochet
+import getpass
+import glob
+import os
+import pappyproxy
+import scrypt
+import shutil
+import twisted
+
+from . import compress
+from .util import PappyException
+from base64 import b64encode, b64decode
+from cryptography.fernet import Fernet, InvalidToken
+from twisted.internet import reactor, defer
+
+
+class Crypto(object):
+    def __init__(self, sessconfig):
+        self.config = sessconfig
+        self.archive = sessconfig.archive
+        self.compressor = compress.Compress(sessconfig)
+        self.key = None
+        self.password = None
+        self.salt = None
+
+    def encrypt_project(self):
+        """
+        Compress and encrypt the project files,
+        deleting clear-text files afterwards
+        """
+
+        # Leave the crypto working directory
+        if self.config.crypt_dir in os.getcwd():
+            os.chdir('../')
+
+        self.compressor.compress_project()
+
+        # Get the password and salt, then derive the key
+        self.crypto_ramp_up()
+
+        # Create project and crypto archive
+        archive_file = open(self.archive, 'rb')
+        archive_crypt = open(self.config.crypt_file, 'wb')
+
+        try:
+            # Encrypt the archive read as a bytestring
+            fern = Fernet(self.key)
+            crypt_token = fern.encrypt(archive_file.read())
+            archive_crypt.write(crypt_token)
+        except InvalidToken as e:
+            raise PappyException("Error encrypting project: ", e)
+            return False
+
+        archive_file.close()
+        archive_crypt.close()
+
+        # Store the salt for the next decryption
+        self.create_salt_file()
+
+        # Delete clear-text files
+        self.delete_clear_files()
+        return True
+
+    def decrypt_project(self):
+        """
+        Decrypt and decompress the project files
+        """
+
+        # Decrypt and decompress the project if crypt_file exists
+        if os.path.isfile(self.config.crypt_file):
+            cf = self.config.crypt_file
+            sl = self.config.salt_len
+            crl = os.path.getsize(cf) - sl
+
+            archive_crypt = open(cf, 'rb').read(crl)
+            archive_file = open(self.config.archive, 'wb')
+
+            retries = 3
+            while True:
+                try:
+                    self.crypto_ramp_up()
+                    fern = Fernet(self.key)
+                    archive = fern.decrypt(archive_crypt)
+                    break
+                except InvalidToken as e:
+                    print "Invalid decryption: ", e
+                    retries -= 1
+                    # Quit pappy if user doesn't retry
+                    # or if all retries exhuasted
+                    if not self.confirm_password_retry() or retries <= 0:
+                        os.remove(self.config.archive)
+                        return False
+                    else:
+                        self.password = None
+                        self.key = None
+                        pass
+
+            archive_file.write(archive)
+            archive_file.close()
+
+            self.compressor.decompress_project()
+            self.delete_crypt_files()
+
+            os.chdir(self.config.crypt_dir)
+            return True
+        # If project exited before encrypting the working directory
+        # change to the working directory to resume the session
+	elif os.path.isdir(self.config.crypt_dir):
+            os.chdir(self.config.crypt_dir)
+            return True
+        # If project hasn't been encrypted before,
+        # setup crypt working directory
+        else:
+            os.mkdir(self.config.crypt_dir)
+
+            project_files = self.config.get_project_files()
+            for pf in project_files:
+                shutil.copy2(pf, self.config.crypt_dir)
+            os.chdir(self.config.crypt_dir)
+            return True
+
+
+    def confirm_password_retry(self):
+        answer = raw_input("Re-enter your password? (y/n): ").strip()
+        if answer[0] == "y" or answer[0] == "Y":
+            return True
+        else:
+            return False
+
+    def crypto_ramp_up(self):
+        if not self.password:
+            self.get_password()
+        if not self.salt:
+            self.set_salt()
+        self.derive_key()
+
+    def get_password(self):
+        """
+        Retrieve password from the user. Raise an exception if the
+        password is not capable of utf-8 encoding.
+        """
+        encoded_passwd = ""
+        try:
+            passwd = getpass.getpass("Enter a password: ").strip()
+            self.password = passwd.encode("utf-8")
+        except:
+            raise PappyException("Invalid password, try again")
+
+    def set_salt(self):
+        if self.config.crypt_dir in os.getcwd():
+            os.chdir('../')
+            self.set_salt_from_file()
+            os.chdir(self.config.crypt_dir)
+        elif os.path.isfile(self.config.crypt_file):
+            self.set_salt_from_file()
+        else:
+            self.salt = os.urandom(16)
+
+    def set_salt_from_file(self):
+        try:
+            # Seek to `salt_len` bytes before the EOF
+            # then read `salt_len` bytes to retrieve the salt
+
+            # WARNING: must open `crypt_file` in `rb` mode
+            # or `salt_file.seek()` will result in undefined
+            # behavior.
+            salt_file = open(self.config.crypt_file, 'rb')
+            sl = self.config.salt_len
+            # Negate the salt length to seek to the 
+            # correct position in the buffer
+            salt_file.seek(-sl, 2)
+            self.salt = salt_file.read(sl)
+            salt_file.close()
+        except:
+            cf = self.config.crypt_file
+            raise PappyException("Unable to read %s" % cf)
+
+    def create_salt_file(self):
+        salt_file = open(self.config.crypt_file, 'a')
+        salt_file.write(self.salt)
+        salt_file.close()
+
+    def derive_key(self):
+        """
+        Derive a key sufficient for use as a cryptographic key
+        used to encrypt the project (currently: cryptography.Fernet).
+
+        cryptography.Fernet utilizes AES-CBC-128, requiring a 32-byte key.
+        Parameter notes from the py-scrypt source-code:
+        https://bitbucket.org/mhallin/py-scrypt/
+
+        Compute scrypt(password, salt, N, r, p, buflen).
+
+        The parameters r, p, and buflen must satisfy r * p < 2^30 and
+        buflen <= (2^32 - 1) * 32. The parameter N must be a power of 2
+        greater than 1. N, r and p must all be positive.
+
+        Notes for Python 2:
+          - `password` and `salt` must be str instances
+          - The result will be a str instance
+
+        Notes for Python 3:
+          - `password` and `salt` can be both str and bytes. If they are str
+            instances, they wil be encoded with utf-8.
+          - The result will be a bytes instance
+
+        Exceptions raised:
+          - TypeError on invalid input
+          - scrypt.error if scrypt failed
+        """
+
+        try:
+            if not self.key:
+                shash = scrypt.hash(self.password, self.salt, buflen=32)
+                self.key = b64encode(shash)
+        except TypeError as e:
+            raise PappyException("Scrypt failed with type error: ", e)
+        except scrypt.error, e:
+            raise PappyException("Scrypt failed with internal error: ", e)
+
+    def delete_clear_files(self):
+        """
+        Deletes all clear-text files left in the project directory.
+        """
+        shutil.rmtree(self.config.crypt_dir)
+        os.remove(self.config.archive)
+
+    def delete_crypt_files(self):
+        """
+        Deletes all encrypted-text files in the project directory.
+        Forces generation of new salt after opening and closing the project.
+        Adds security in the case of a one-time compromise of the system.
+        """
+        os.remove(self.config.crypt_file)

pappyproxy/pappy.py

@@ -21,9 +21,12 @@ import signal
 
 from . import comm
 from . import config
+from . import compress
 from . import context
+from . import crypto
 from . import http
 from .console import ProxyCmd
+from .util import PappyException
 from twisted.enterprise import adbapi
 from twisted.internet import reactor, defer
 from twisted.internet.error import CannotListenError
@@ -62,11 +65,21 @@ class PappySession(object):
         self.dbpool = None
         self.delete_data_on_quit = False
         self.ports = None
+        self.crypto = crypto.Crypto(sessconfig)
 
     @defer.inlineCallbacks
     def start(self):
         from . import proxy, plugin
 
+        if self.config.crypt_session:
+            if self.decrypt():
+                self.config.load_from_file('./config.json')
+                self.config.global_load_from_file()
+                self.delete_data_on_quit = False
+            else:
+                self.complete_defer.callback(None)
+                return
+        
         # If the data file doesn't exist, create it with restricted permissions
         if not os.path.isfile(self.config.datafile):
             with os.fdopen(os.open(self.config.datafile, os.O_CREAT, 0o0600), 'r'):
@@ -139,6 +152,20 @@ class PappySession(object):
         self.complete_defer = deferToThread(self.cons.cmdloop)
         self.complete_defer.addCallback(self.cleanup)
 
+    def encrypt(self):
+        if self.crypto.encrypt_project():
+            return True
+        else:
+            return False
+
+    def decrypt(self):
+        # Attempt to decrypt project archive
+        if self.crypto.decrypt_project():
+            return True
+        # Quit pappy on failure
+        else:
+            return False
+
     @defer.inlineCallbacks
     def cleanup(self, ignored=None):
         for port in self.ports:
@@ -148,11 +175,25 @@ class PappySession(object):
             print 'Deleting temporary datafile'
             os.remove(self.config.datafile)
 
+        # Encrypt the project when in crypto mode 
+        if self.config.crypt_session:
+            self.encrypt()
+    
 def parse_args():
     # parses sys.argv and returns a settings dictionary
 
     parser = argparse.ArgumentParser(description='An intercepting proxy for testing web applications.')
     parser.add_argument('-l', '--lite', help='Run the proxy in "lite" mode', action='store_true')
+    parser.add_argument('-d', '--debug', help='Run the proxy in "debug" mode', action='store_true')
+    try:
+        hlpmsg = ''.join(['Start pappy in "crypto" mode,',
+                 'must supply a name for the encrypted',
+                 'project archive [CRYPT]'])
+        parser.add_argument('-c', '--crypt', type=str, nargs=1, help=hlpmsg)
+    except:
+        print 'Must supply a project name: pappy -c <project_name>'
+        reactor.stop()
+        defer.returnValue(None)
 
     args = parser.parse_args(sys.argv[1:])
     settings = {}
@@ -162,6 +203,16 @@ def parse_args():
     else:
         settings['lite'] = False
 
+    if args.crypt:
+        # Convert from single-item list produced by argparse `nargs=1`
+        settings['crypt'] = args.crypt[0].encode('utf-8')
+    else:
+        settings['crypt'] = None
+
+    if args.debug:
+        settings['debug'] = True
+    else:
+        settings['debug'] = False
     return settings
 
 def set_text_factory(conn):
@@ -189,7 +240,10 @@ def main():
     session = PappySession(pappy_config)
     signal.signal(signal.SIGINT, inturrupt_handler)
 
-    if settings['lite']:
+    if settings['crypt']:
+        pappy_config.crypt_file = settings['crypt']
+        pappy_config.crypt_session = True
+    elif settings['lite']:
         conf_settings = pappy_config.get_default_config()
         conf_settings['debug_dir'] = None
         conf_settings['debug_to_file'] = False
@@ -205,6 +259,9 @@ def main():
         pappy_config.global_load_from_file()
         session.delete_data_on_quit = False
 
+    if settings['debug']:
+        pappy_config.debug = True
+
     yield session.start()
 
     session.complete_defer.addCallback(lambda ignored: reactor.stop())

pappyproxy/tests/test_crypto.py

@@ -0,0 +1,62 @@
+import os
+import pytest
+import random
+import string
+from pappyproxy.session import Session
+from pappyproxy.crypto import Crypto
+from pappyproxy.config import PappyConfig
+
+@pytest.fixture
+def conf():
+    c = PappyConfig()
+    return c
+
+@pytest.fixture
+def crypt():
+    c = Crypto(conf())
+    return c
+
+@pytest.fixture
+def tmpname():
+    cns = string.ascii_lowercase + string.ascii_uppercase + string.digits
+    tn = ''
+    for i in xrange(8):
+        tn += cns[random.randint(0,len(cns)-1)]
+    return tn
+
+tmpdir = '/tmp/test_crypto'+tmpname()
+tmpfiles = ['cmdhistory', 'config.json', 'data.db']
+tmp_pass = 'fugyeahbruh'
+
+def stub_files():
+    enter_tmpdir()
+    for sf in tmpfiles:
+        with os.fdopen(os.open(sf, os.O_CREAT, 0o0600), 'r'):
+            pass
+
+def enter_tmpdir():
+    if not os.path.isdir(tmpdir):
+        os.mkdir(tmpdir)
+    os.chdir(tmpdir)
+
+def test_decrypt_tmpdir():
+    enter_tmpdir()
+    c = crypt()
+     
+    # Stub out the password, working with stdout is a pain with pytest
+    c.password = tmp_pass
+    
+    c.decrypt_project()
+    assert os.path.isdir(os.path.join(os.getcwd(), '../crypt'))
+
+def test_decrypt_copy_files():
+    enter_tmpdir()
+    stub_files()
+    c = crypt()
+     
+    # Stub out the password, working with stdout is a pain with pytest
+    c.password = tmp_pass
+    
+    c.decrypt_project()
+    for tf in tmpfiles:
+        assert os.path.isfile(os.path.join(os.getcwd(),tf))

setup.py

@@ -33,6 +33,7 @@ setup(name='pappyproxy',
           'pytest-mock>=0.9.0',
           'pytest-twisted>=1.5',
           'pytest>=2.8.3',
+          'scrypt>=0.7.1',
           'service_identity>=14.0.0',
           'twisted>=15.4.0',
           'txsocksx>=1.15.0.2'