diff --git a/.gitignore b/.gitignore index d1d307d..b8b31e7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ TAGS config.json build/* *.egg-info/* -.#* \ No newline at end of file +.#* +*notes* diff --git a/pappyproxy/Makefile b/pappyproxy/Makefile index 2e74f8d..2faa546 100644 --- a/pappyproxy/Makefile +++ b/pappyproxy/Makefile @@ -16,3 +16,7 @@ test-proxy: test-comm: py.test -v -rw --twisted tests/test_comm.py + +test-crypto: + py.test -v -rw --twisted tests/test_crypto.py + diff --git a/pappyproxy/compress.py b/pappyproxy/compress.py new file mode 100644 index 0000000..2204088 --- /dev/null +++ b/pappyproxy/compress.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +import crochet +import glob +import pappyproxy + +import zipfile +import tarfile + +try: + import bz2 +except ImportError: + bz2 = None + print "BZ2 not installed on your system" + +from base64 import b64encode, b64decode +from os import getcwd, sep, path, urandom + + +class Compress(object): + def __init__(self, sessconfig): + self.config = sessconfig + self.zip_archive = sessconfig.archive + self.bz2_archive = sessconfig.archive + + def compress_project(self): + if bz2: + self.tar_project() + else: + self.zip_project() + + def decompress_project(self): + if bz2: + self.untar_project() + else: + self.unzip_project() + + def zip_project(self): + """ + Zip project files + + Using append mode (mode='a') will create a zip archive + if none exists in the project. + """ + try: + zf = zipfile.ZipFile(self.zip_archive, mode="a") + zf.write(self.config.crypt_dir) + zf.close() + except zipfile.LargeZipFile as e: + raise PappyException("Project zipfile too large. Error: ", e) + + def unzip_project(self): + """ + Extract project files from decrypted zip archive. + Initially checks the zip archive's magic number and + attempts to extract pappy.json to validate integrity + of the zipfile. + """ + if not zipfile.is_zipfile(self.zip_archive): + raise PappyException("Project archive corrupted.") + + zf = zipfile.ZipFile(self.zip_archive) + + try: + zf.extract("config.json") + except zipfile.BadZipfile as e: + raise PappyException("Zip archive corrupted. Error: ", e) + + zf.extractall() + + def tar_project(self): + archive = tarfile.open(self.bz2_archive, 'w:bz2') + + archive.add(self.config.crypt_dir) + archive.close() + + def untar_project(self): + if tarfile.is_tarfile(self.bz2_archive): + # Raise exception if there is a failure + try: + with tarfile.open(self.bz2_archive, "r:bz2") as archive: + archive.extractall() + except tarfile.ExtractError as e: + raise PappyException("Tar archive corrupted. Error: ", e) diff --git a/pappyproxy/config.py b/pappyproxy/config.py index 9181c50..32dfeae 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -1,3 +1,4 @@ +import glob import json import os import shutil @@ -98,6 +99,42 @@ class PappyConfig(object): The dictionary from ~/.pappy/global_config.json. It contains settings for Pappy that are specific to the current computer. Avoid putting settings here, especially if it involves specific projects. + + .. data: archive + + Project archive compressed as a ``tar.bz2`` archive if libraries available on the system, + otherwise falls back to zip archive. + + :Default: ``project.archive`` + + .. data: crypt_dir + + Temporary working directory to unpack an encrypted project archive. Directory + will contain copies of normal startup files, e.g. conifg.json, cmdhistory, etc. + On exiting pappy, entire directory will be compressed into an archive and encrypted. + Compressed as a tar.bz2 archive if libraries available on the system, + otherwise falls back to zip. + + :Default: ``crypt`` + + .. data: crypt_file + + Encrypted archive of the temporary working directory ``crypt_dir``. Compressed as a + tar.bz2 archive if libraries available on the system, otherwise falls back to zip. + + :Default: ``project.crypt`` + + .. data: crypt_session + + Boolean variable to determine whether pappy started in crypto mode + + :Default: False + + .. data: salt_len + + Length of the nonce-salt value appended to the end of `crypt_file` + + :Default: 16 """ def __init__(self): @@ -125,6 +162,13 @@ class PappyConfig(object): self.config_dict = {} self.global_config_dict = {} + + self.archive = 'project.archive' + self.debug = False + self.crypt_dir = 'crypt' + self.crypt_file = 'project.crypt' + self.crypt_session = False + self.salt_len = 16 def get_default_config(self): default_config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), @@ -133,6 +177,17 @@ class PappyConfig(object): settings = json.load(f) return settings + def get_project_files(self): + file_glob = glob.glob('*') + pp = os.getcwd() + os.sep + project_files = [pp+f for f in file_glob if os.path.isfile(pp+f)] + + if self.crypt_file in project_files: + project_files.remove(self.crypt_file) + + return project_files + + @staticmethod def _parse_proxy_login(conf): proxy = {} diff --git a/pappyproxy/console.py b/pappyproxy/console.py index 7701da2..b670ba2 100644 --- a/pappyproxy/console.py +++ b/pappyproxy/console.py @@ -42,13 +42,15 @@ class ProxyCmd(cmd2.Cmd): self._cmds = {} self._aliases = {} - atexit.register(self.save_histfile) - readline.set_history_length(self.session.config.histsize) - if os.path.exists('cmdhistory'): - if self.session.config.histsize != 0: - readline.read_history_file('cmdhistory') - else: - os.remove('cmdhistory') + # Only read and save history when not in crypto mode + if not self.session.config.crypt_session: + atexit.register(self.save_histfile) + readline.set_history_length(self.session.config.histsize) + if os.path.exists('cmdhistory'): + if self.session.config.histsize != 0: + readline.read_history_file('cmdhistory') + else: + os.remove('cmdhistory') cmd2.Cmd.__init__(self, *args, **kwargs) @@ -110,10 +112,12 @@ class ProxyCmd(cmd2.Cmd): raise AttributeError(attr) def save_histfile(self): - # Write the command to the history file - if self.session.config.histsize != 0: - readline.set_history_length(self.session.config.histsize) - readline.write_history_file('cmdhistory') + # Only write to file if not in crypto mode + if not self.session.config.crypt_session: + # Write the command to the history file + if self.session.config.histsize != 0: + readline.set_history_length(self.session.config.histsize) + readline.write_history_file('cmdhistory') def get_names(self): # Hack to get cmd to recognize do_/etc functions as functions for things diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py new file mode 100644 index 0000000..2789471 --- /dev/null +++ b/pappyproxy/crypto.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python + +import crochet +import getpass +import glob +import os +import pappyproxy +import scrypt +import shutil +import twisted + +from . import compress +from .util import PappyException +from base64 import b64encode, b64decode +from cryptography.fernet import Fernet, InvalidToken +from twisted.internet import reactor, defer + + +class Crypto(object): + def __init__(self, sessconfig): + self.config = sessconfig + self.archive = sessconfig.archive + self.compressor = compress.Compress(sessconfig) + self.key = None + self.password = None + self.salt = None + + def encrypt_project(self): + """ + Compress and encrypt the project files, + deleting clear-text files afterwards + """ + + # Leave the crypto working directory + if self.config.crypt_dir in os.getcwd(): + os.chdir('../') + + self.compressor.compress_project() + + # Get the password and salt, then derive the key + self.crypto_ramp_up() + + # Create project and crypto archive + archive_file = open(self.archive, 'rb') + archive_crypt = open(self.config.crypt_file, 'wb') + + try: + # Encrypt the archive read as a bytestring + fern = Fernet(self.key) + crypt_token = fern.encrypt(archive_file.read()) + archive_crypt.write(crypt_token) + except InvalidToken as e: + raise PappyException("Error encrypting project: ", e) + return False + + archive_file.close() + archive_crypt.close() + + # Store the salt for the next decryption + self.create_salt_file() + + # Delete clear-text files + self.delete_clear_files() + return True + + def decrypt_project(self): + """ + Decrypt and decompress the project files + """ + + # Decrypt and decompress the project if crypt_file exists + if os.path.isfile(self.config.crypt_file): + cf = self.config.crypt_file + sl = self.config.salt_len + crl = os.path.getsize(cf) - sl + + archive_crypt = open(cf, 'rb').read(crl) + archive_file = open(self.config.archive, 'wb') + + retries = 3 + while True: + try: + self.crypto_ramp_up() + fern = Fernet(self.key) + archive = fern.decrypt(archive_crypt) + break + except InvalidToken as e: + print "Invalid decryption: ", e + retries -= 1 + # Quit pappy if user doesn't retry + # or if all retries exhuasted + if not self.confirm_password_retry() or retries <= 0: + os.remove(self.config.archive) + return False + else: + self.password = None + self.key = None + pass + + archive_file.write(archive) + archive_file.close() + + self.compressor.decompress_project() + self.delete_crypt_files() + + os.chdir(self.config.crypt_dir) + return True + # If project exited before encrypting the working directory + # change to the working directory to resume the session + elif os.path.isdir(self.config.crypt_dir): + os.chdir(self.config.crypt_dir) + return True + # If project hasn't been encrypted before, + # setup crypt working directory + else: + os.mkdir(self.config.crypt_dir) + + project_files = self.config.get_project_files() + for pf in project_files: + shutil.copy2(pf, self.config.crypt_dir) + os.chdir(self.config.crypt_dir) + return True + + + def confirm_password_retry(self): + answer = raw_input("Re-enter your password? (y/n): ").strip() + if answer[0] == "y" or answer[0] == "Y": + return True + else: + return False + + def crypto_ramp_up(self): + if not self.password: + self.get_password() + if not self.salt: + self.set_salt() + self.derive_key() + + def get_password(self): + """ + Retrieve password from the user. Raise an exception if the + password is not capable of utf-8 encoding. + """ + encoded_passwd = "" + try: + passwd = getpass.getpass("Enter a password: ").strip() + self.password = passwd.encode("utf-8") + except: + raise PappyException("Invalid password, try again") + + def set_salt(self): + if self.config.crypt_dir in os.getcwd(): + os.chdir('../') + self.set_salt_from_file() + os.chdir(self.config.crypt_dir) + elif os.path.isfile(self.config.crypt_file): + self.set_salt_from_file() + else: + self.salt = os.urandom(16) + + def set_salt_from_file(self): + try: + # Seek to `salt_len` bytes before the EOF + # then read `salt_len` bytes to retrieve the salt + + # WARNING: must open `crypt_file` in `rb` mode + # or `salt_file.seek()` will result in undefined + # behavior. + salt_file = open(self.config.crypt_file, 'rb') + sl = self.config.salt_len + # Negate the salt length to seek to the + # correct position in the buffer + salt_file.seek(-sl, 2) + self.salt = salt_file.read(sl) + salt_file.close() + except: + cf = self.config.crypt_file + raise PappyException("Unable to read %s" % cf) + + def create_salt_file(self): + salt_file = open(self.config.crypt_file, 'a') + salt_file.write(self.salt) + salt_file.close() + + def derive_key(self): + """ + Derive a key sufficient for use as a cryptographic key + used to encrypt the project (currently: cryptography.Fernet). + + cryptography.Fernet utilizes AES-CBC-128, requiring a 32-byte key. + Parameter notes from the py-scrypt source-code: + https://bitbucket.org/mhallin/py-scrypt/ + + Compute scrypt(password, salt, N, r, p, buflen). + + The parameters r, p, and buflen must satisfy r * p < 2^30 and + buflen <= (2^32 - 1) * 32. The parameter N must be a power of 2 + greater than 1. N, r and p must all be positive. + + Notes for Python 2: + - `password` and `salt` must be str instances + - The result will be a str instance + + Notes for Python 3: + - `password` and `salt` can be both str and bytes. If they are str + instances, they wil be encoded with utf-8. + - The result will be a bytes instance + + Exceptions raised: + - TypeError on invalid input + - scrypt.error if scrypt failed + """ + + try: + if not self.key: + shash = scrypt.hash(self.password, self.salt, buflen=32) + self.key = b64encode(shash) + except TypeError as e: + raise PappyException("Scrypt failed with type error: ", e) + except scrypt.error, e: + raise PappyException("Scrypt failed with internal error: ", e) + + def delete_clear_files(self): + """ + Deletes all clear-text files left in the project directory. + """ + shutil.rmtree(self.config.crypt_dir) + os.remove(self.config.archive) + + def delete_crypt_files(self): + """ + Deletes all encrypted-text files in the project directory. + Forces generation of new salt after opening and closing the project. + Adds security in the case of a one-time compromise of the system. + """ + os.remove(self.config.crypt_file) diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index eeb8bff..fd20b21 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -21,9 +21,12 @@ import signal from . import comm from . import config +from . import compress from . import context +from . import crypto from . import http from .console import ProxyCmd +from .util import PappyException from twisted.enterprise import adbapi from twisted.internet import reactor, defer from twisted.internet.error import CannotListenError @@ -62,10 +65,20 @@ class PappySession(object): self.dbpool = None self.delete_data_on_quit = False self.ports = None + self.crypto = crypto.Crypto(sessconfig) @defer.inlineCallbacks def start(self): from . import proxy, plugin + + if self.config.crypt_session: + if self.decrypt(): + self.config.load_from_file('./config.json') + self.config.global_load_from_file() + self.delete_data_on_quit = False + else: + self.complete_defer.callback(None) + return # If the data file doesn't exist, create it with restricted permissions if not os.path.isfile(self.config.datafile): @@ -138,21 +151,49 @@ class PappySession(object): # Add cleanup to defer self.complete_defer = deferToThread(self.cons.cmdloop) self.complete_defer.addCallback(self.cleanup) - + + def encrypt(self): + if self.crypto.encrypt_project(): + return True + else: + return False + + def decrypt(self): + # Attempt to decrypt project archive + if self.crypto.decrypt_project(): + return True + # Quit pappy on failure + else: + return False + @defer.inlineCallbacks def cleanup(self, ignored=None): for port in self.ports: yield port.stopListening() - + if self.delete_data_on_quit: print 'Deleting temporary datafile' os.remove(self.config.datafile) + + # Encrypt the project when in crypto mode + if self.config.crypt_session: + self.encrypt() def parse_args(): # parses sys.argv and returns a settings dictionary parser = argparse.ArgumentParser(description='An intercepting proxy for testing web applications.') parser.add_argument('-l', '--lite', help='Run the proxy in "lite" mode', action='store_true') + parser.add_argument('-d', '--debug', help='Run the proxy in "debug" mode', action='store_true') + try: + hlpmsg = ''.join(['Start pappy in "crypto" mode,', + 'must supply a name for the encrypted', + 'project archive [CRYPT]']) + parser.add_argument('-c', '--crypt', type=str, nargs=1, help=hlpmsg) + except: + print 'Must supply a project name: pappy -c ' + reactor.stop() + defer.returnValue(None) args = parser.parse_args(sys.argv[1:]) settings = {} @@ -162,6 +203,16 @@ def parse_args(): else: settings['lite'] = False + if args.crypt: + # Convert from single-item list produced by argparse `nargs=1` + settings['crypt'] = args.crypt[0].encode('utf-8') + else: + settings['crypt'] = None + + if args.debug: + settings['debug'] = True + else: + settings['debug'] = False return settings def set_text_factory(conn): @@ -189,7 +240,10 @@ def main(): session = PappySession(pappy_config) signal.signal(signal.SIGINT, inturrupt_handler) - if settings['lite']: + if settings['crypt']: + pappy_config.crypt_file = settings['crypt'] + pappy_config.crypt_session = True + elif settings['lite']: conf_settings = pappy_config.get_default_config() conf_settings['debug_dir'] = None conf_settings['debug_to_file'] = False @@ -205,6 +259,9 @@ def main(): pappy_config.global_load_from_file() session.delete_data_on_quit = False + if settings['debug']: + pappy_config.debug = True + yield session.start() session.complete_defer.addCallback(lambda ignored: reactor.stop()) diff --git a/pappyproxy/tests/test_crypto.py b/pappyproxy/tests/test_crypto.py new file mode 100644 index 0000000..9939cd0 --- /dev/null +++ b/pappyproxy/tests/test_crypto.py @@ -0,0 +1,62 @@ +import os +import pytest +import random +import string +from pappyproxy.session import Session +from pappyproxy.crypto import Crypto +from pappyproxy.config import PappyConfig + +@pytest.fixture +def conf(): + c = PappyConfig() + return c + +@pytest.fixture +def crypt(): + c = Crypto(conf()) + return c + +@pytest.fixture +def tmpname(): + cns = string.ascii_lowercase + string.ascii_uppercase + string.digits + tn = '' + for i in xrange(8): + tn += cns[random.randint(0,len(cns)-1)] + return tn + +tmpdir = '/tmp/test_crypto'+tmpname() +tmpfiles = ['cmdhistory', 'config.json', 'data.db'] +tmp_pass = 'fugyeahbruh' + +def stub_files(): + enter_tmpdir() + for sf in tmpfiles: + with os.fdopen(os.open(sf, os.O_CREAT, 0o0600), 'r'): + pass + +def enter_tmpdir(): + if not os.path.isdir(tmpdir): + os.mkdir(tmpdir) + os.chdir(tmpdir) + +def test_decrypt_tmpdir(): + enter_tmpdir() + c = crypt() + + # Stub out the password, working with stdout is a pain with pytest + c.password = tmp_pass + + c.decrypt_project() + assert os.path.isdir(os.path.join(os.getcwd(), '../crypt')) + +def test_decrypt_copy_files(): + enter_tmpdir() + stub_files() + c = crypt() + + # Stub out the password, working with stdout is a pain with pytest + c.password = tmp_pass + + c.decrypt_project() + for tf in tmpfiles: + assert os.path.isfile(os.path.join(os.getcwd(),tf)) diff --git a/setup.py b/setup.py index d621644..5bec257 100755 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ setup(name='pappyproxy', 'pytest-mock>=0.9.0', 'pytest-twisted>=1.5', 'pytest>=2.8.3', + 'scrypt>=0.7.1', 'service_identity>=14.0.0', 'twisted>=15.4.0', 'txsocksx>=1.15.0.2'