From 6a792092244e5ddfab5ab91204978ef63ede1910 Mon Sep 17 00:00:00 2001 From: onizenso Date: Wed, 23 Mar 2016 14:27:11 +0000 Subject: [PATCH 01/21] Cleaned up local repository Cleaned up the huge mess I created, pushing back to catch everything up. Redoing crypto plugin as main component instead of plugin. Likely compressing existing project files on first crypto run, then storing the encrypted archive blob in the project directory. When pappy started again, it will see the encrypted blob, extract files to working directory, do all of its work in that directory, then exit the directory, and clean up all files. A lot of work, but worth the end result! --- pappyproxy/plugins/compress/__init__.py | 0 pappyproxy/plugins/compress/compress.py | 89 ++++++++++++ pappyproxy/plugins/crypto/__init__.py | 0 pappyproxy/plugins/crypto/crypto.py | 182 ++++++++++++++++++++++++ pappyproxy/plugins/misc.py | 22 +++ 5 files changed, 293 insertions(+) create mode 100644 pappyproxy/plugins/compress/__init__.py create mode 100644 pappyproxy/plugins/compress/compress.py create mode 100644 pappyproxy/plugins/crypto/__init__.py create mode 100644 pappyproxy/plugins/crypto/crypto.py diff --git a/pappyproxy/plugins/compress/__init__.py b/pappyproxy/plugins/compress/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pappyproxy/plugins/compress/compress.py b/pappyproxy/plugins/compress/compress.py new file mode 100644 index 0000000..18a320e --- /dev/null +++ b/pappyproxy/plugins/compress/compress.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +import crochet +import glob +import pappyproxy + +import zipfile +import tarfile + +# This is a gross hack, please help +bz2 = None +try: + import bz2 +except: + print "BZ2 not installed on your system" + +from base64 import b64encode, b64decode +from os import getcwd, sep, path, urandom +from pappyproxy.plugins.misc import CryptoCompressUtils as ccu + +def compress_project(): + if bz2: + tar_project() + else: + zip_project() + +def decompress_project(): + if bz2: + untar_project() + else: + unzip_project() + +def zip_project(): + """ + Zip project files + + Using append mode (mode='a') will create a zip archive + if none exists in the project. + """ + try: + zf = zipfile.ZipFile(ZIPFILE, mode="a") + project_files = ccu.get_project_files() + for pf in project_files: + zf.write(pf) + zf.close() + except e: + raise PappyException("Error creating the zipfile", e) + pass + +def unzip_project(): + """ + Extract project files from decrypted zip archive. + Initially checks the zip archive's magic number and + attempts to extract pappy.json to validate integrity + of the zipfile. + """ + if not zipfile.is_zipfile(ZIPFILE): + raise PappyException("Project archive corrupted.") + + zf = zipfile.ZipFile(ZIPFILE) + + try: + zf.extract("config.json") + except e: + raise PappyException("Project archive contents corrupted. Error: ", e) + + zf.extractall() + +def tar_project(): + if tarfile.is_tarfile(BZ2FILE): + archive = tarfile.open(ccu.BZ2FILE, 'w:bz2') + project_files = ccu.get_project_files() + + # Read files line by line to accomodate larger files, e.g. the project database + for pf in project_files: + archive.add(pf) + archive.close() + +def untar_project(): + if tarfile.is_tarfile(BZ2FILE): + # Attempt to read the first 16 bytes of the archive + # Raise exception if there is a failure + project_files = ccu.get_project_files() + try: + with tarfile.open(BZ2FILE, "r:bz2") as archive: + for pf in project_files: + archive.add(pf) + except e: + raise PappyException("Project archive contents corrupted. Error: ", e diff --git a/pappyproxy/plugins/crypto/__init__.py b/pappyproxy/plugins/crypto/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pappyproxy/plugins/crypto/crypto.py b/pappyproxy/plugins/crypto/crypto.py new file mode 100644 index 0000000..a63d929 --- /dev/null +++ b/pappyproxy/plugins/crypto/crypto.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python + +import crochet +import glob +import pappyproxy + +import scrypt +import twisted + +from base64 import b64encode, b64decode +from cryptography import Fernet +from os import getcwd, sep, path, remove, urandom +from pappyproxy.plugins import compress +from pappyproxy.plugins.misc import CryptoCompressUtils as ccu + + +def encrypt_project(passwd): + """ + Compress and encrypt the project files, deleting clear-text files afterwards + """ + # Derive the key + key = crypto_ramp_up(passwd) + # Instantiate the crypto module + fern = Fernet(key) + + compress.compress_project() + archive = None + if path.is_file(ZIPFILE): + archive = open(ccu.ZIPFILE, 'rb') + else: + archive = open(ccu.BZ2FILE, 'rb') + archive_crypt = open(ccu.CRYPTFILE, 'wb') + + # Encrypt the archive read as a bytestring + crypt_token = fern.encrypt(archive) + archive_crypt.write(crypt_token) + + # Delete clear-text files + delete_clear_files() + +def decrypt_project(passwd): + """ + Decompress and decrypt the project files + """ + # Derive the key + key = crypto_ramp_up(passwd) + fern = Fernet(key) + archive_crypt = open(ccu.CRYPTFILE, 'rb') + archive = fern.decrypt(archive_crypt) + compress.decompress_project() + delete_crypt_files() + + +def crypto_ramp_up(passwd): + salt = "" + if path.isfile(ccu.SALTFILE): + salt = get_salt() + else: + salt = create_salt() + key = derive_key(passwd, salt) + return key + +def delete_clear_files(): + """ + Deletes all clear-text files left in the project directory. + """ + project_files = ccu.get_project_files() + for pf in project_files: + os.remove(pf) + +def delete_crypt_files(): + """ + Deletes all encrypted-text files in the project directory. + Forces generation of new salt after opening and closing the project. + Adds security in the case of a one-time compromise of the system. + """ + os.remove(ccu.SALTFILE) + os.remove(ccu.CRYPTFILE) + +def create_salt(): + salt = b64encode(urandom(16)) + salt_file = open(ccu.SALTFILE, 'wb') + salt_file.write(salt) + salt_file.close() + return salt + +def get_salt(): + try: + salt_file = open(ccu.SALTFILE, 'rb') + salt = b64decode(salt_file.readline()) + except: + raise PappyException("Unable to read pappy.salt") + return salt + +def get_password(): + """ + Retrieve password from the user. Raise an exception if the + password is not capable of base64 encoding. + """ + encode_passwd = "" + try: + passwd = raw_input("Enter a password: ") + encode_passwd = b64encode(passwd.encode("utf-8")) + except: + raise PappyException("Invalid password, try again") + return encode_passwd + +def derive_key(passwd, salt): + """ + Derive a key sufficient for use as a cryptographic key + used to encrypt the project (currently: cryptography.Fernet). + + cryptography.Fernet utilizes AES-CBC-128, requiring a 32-byte key. + Parameter notes from the py-scrypt source-code: + https://bitbucket.org/mhallin/py-scrypt/ + + Compute scrypt(password, salt, N, r, p, buflen). + + The parameters r, p, and buflen must satisfy r * p < 2^30 and + buflen <= (2^32 - 1) * 32. The parameter N must be a power of 2 + greater than 1. N, r and p must all be positive. + + Notes for Python 2: + - `password` and `salt` must be str instances + - The result will be a str instance + + Notes for Python 3: + - `password` and `salt` can be both str and bytes. If they are str + instances, they wil be encoded with utf-8. + - The result will be a bytes instance + + Exceptions raised: + - TypeError on invalid input + - scrypt.error if scrypt failed + """ + + derived_key = "" + try: + dkey = scrypt.hash(passwd, salt, bufflen=32) + except e: + raise PappyException("Error deriving the key: ", e) + return derived_key + + +@crochet.wait_for(timeout=None) +@defer.inlineCallbacks +def cryptocmd(line): + """ + Encrypt/Decrypt local project directory + Usage: pappy -e + Details: + Pappy will create a compressed archive of local project files. + + The archive file is encrypted using the cryptography.Fernet module, + a user-supplied password and the scrypt key-derivation function. + + cryptography.Fernet uses AES-CBC-128 with HMAC256. This is merely + a starting point, and any help implementing a stronger crypto-system + is very welcome. Development is geared toward using + AES-256-GCM as the AEAD encryption mode to eliminate the need for Fernet and HMAC256. + SCrypt will still be used as the key derivation function until a public-key encryption + scheme is developed. + + See Encryption section of README.md for more information. + """ + + if isinstance(line, str): + args = crochet.split(line) + ## Encryption mode (Encrypt=0, Decrypt=1) + ## Set internally depending if plugin is called during pappy startup or shutdown + mode = args[0] + + ## Request the pasword from the user + passwd = get_passwd() + + if mode == ccu.ENCRYPT: + encrypt_project(passwd) + elif mode == ccu.DECRYPT: + decrypt_project(passwd) + else: + raise PappyException("Incorrect crypto mode") + diff --git a/pappyproxy/plugins/misc.py b/pappyproxy/plugins/misc.py index 87934de..fb9bcd3 100644 --- a/pappyproxy/plugins/misc.py +++ b/pappyproxy/plugins/misc.py @@ -201,3 +201,25 @@ def load_cmds(cmd): cmd.add_aliases([ #('rpy', ''), ]) + +class CryptoCompressUtils(): + # Constants + ENCRYPT = 0 + DECRYPT = 1 + PROJECT_PATH = getcwd() + sep + ZIPFILE = PROJECT_PATH + "pappy.zip" + BZ2FILE = PROJECT_PATH + "pappy.bz2" + CRYPTFILE = "" + if path.isfile(ZIPFILE): + CRYPTFILE = ZIPFILE + ".crypt" + elsif path.isfile(BZ2FILE): + CRYPTFILE = BZ2FILE + ".crypt" + SALTFILE = PROJECT_PATH + "pappy.salt" + + def get_project_files(): + file_glob = glob.glob('*') + pp = PROJECT_PATH + project_files = [pp+f for f in file_glob if path.isfile(pp+f)] + project_files.remove(SALTFILE) + project_files.remove(CRYPTFILE) + return project_files From 5be69b205dbffbc312681d67868fb98ba9fb576a Mon Sep 17 00:00:00 2001 From: onizenso Date: Thu, 24 Mar 2016 20:48:40 +0000 Subject: [PATCH 02/21] Drastically restructured the compression and crypto features Converted the crypto and compression plugins to core features, and added the utility variables and functions to the Config class in ``config.py``. Added helper functions in PappySession class in ``pappy.py`` to enable the user to pass in an encrypted project archive. Next moving to testing and debugging! --- .gitignore | 3 +- pappyproxy/compress.py | 95 ++++++++++++++++++++++++ pappyproxy/config.py | 61 +++++++++++++++ pappyproxy/crypto.py | 164 +++++++++++++++++++++++++++++++++++++++++ pappyproxy/pappy.py | 36 ++++++++- 5 files changed, 355 insertions(+), 4 deletions(-) create mode 100644 pappyproxy/compress.py create mode 100644 pappyproxy/crypto.py diff --git a/.gitignore b/.gitignore index d1d307d..b8b31e7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ TAGS config.json build/* *.egg-info/* -.#* \ No newline at end of file +.#* +*notes* diff --git a/pappyproxy/compress.py b/pappyproxy/compress.py new file mode 100644 index 0000000..ec4370d --- /dev/null +++ b/pappyproxy/compress.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +import crochet +import glob +import pappyproxy + +import zipfile +import tarfile + +# This is a gross hack, please help +bz2 = None +try: + import bz2 +except: + print "BZ2 not installed on your system" + +from base64 import b64encode, b64decode +from os import getcwd, sep, path, urandom +from pappyproxy.plugins.misc import CryptoCompressUtils as ccu + +class Compress(object): + def __init__(self, sessconfig): + self.config = sessconfig + self.zip_archive = sessconfig.archive + self.bz2_archive = sessconfig.archive + + def compress_project(): + if bz2: + tar_project() + else: + zip_project() + + def decompress_project(): + if bz2: + untar_project() + else: + unzip_project() + + def zip_project(): + """ + Zip project files + + Using append mode (mode='a') will create a zip archive + if none exists in the project. + """ + try: + zf = zipfile.ZipFile(self.zip_archive, mode="a") + project_files = self.config.get_project_files() + for pf in project_files: + zf.write(pf) + zf.close() + except e: + raise PappyException("Error creating the zipfile", e) + pass + + def unzip_project(): + """ + Extract project files from decrypted zip archive. + Initially checks the zip archive's magic number and + attempts to extract pappy.json to validate integrity + of the zipfile. + """ + if not zipfile.is_zipfile(self.zip_archive): + raise PappyException("Project archive corrupted.") + + zf = zipfile.ZipFile(self.zip_archive) + + try: + zf.extract("config.json") + except e: + raise PappyException("Project archive contents corrupted. Error: ", e) + + zf.extractall() + + def tar_project(): + if tarfile.is_tarfile(self.bz2_archive): + archive = tarfile.open(self.bz2_archive, 'w:bz2') + project_files = self.config.get_project_files() + + # Read files line by line to accomodate larger files, e.g. the project database + for pf in project_files: + archive.add(pf) + archive.close() + + def untar_project(): + if tarfile.is_tarfile(self.bz2_archive): + # Attempt to read the first 16 bytes of the archive + # Raise exception if there is a failure + project_files = self.config.get_project_files() + try: + with tarfile.open(self.bz2_archive, "r:bz2") as archive: + for pf in project_files: + archive.add(pf) + except e: + raise PappyException("Project archive contents corrupted. Error: ", e diff --git a/pappyproxy/config.py b/pappyproxy/config.py index 9181c50..3b8f565 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -98,6 +98,51 @@ class PappyConfig(object): The dictionary from ~/.pappy/global_config.json. It contains settings for Pappy that are specific to the current computer. Avoid putting settings here, especially if it involves specific projects. + + .. data: archive + + Project archive compressed as a ``tar.bz2`` archive if libraries available on the system, + otherwise falls back to zip archive. + + :Default: 'project.archive' + + .. data: crypt_dir + + Temporary working directory to unpack an encrypted project archive. Directory + will contain copies of normal startup files, e.g. conifg.json, cmdhistory, etc. + On exiting pappy, entire directory will be compressed into an archive and encrypted. + Compressed as a tar.bz2 archive if libraries available on the system, + otherwise falls back to zip. + + :Default: 'crypt' + + .. data: crypt_file + + Encrypted archive of the temporary working directory ``crypt_dir``. Compressed as a + tar.bz2 archive if libraries available on the system, otherwise falls back to zip. + + :Default: 'project.crypt' + + .. data: crypt_mode + + Boolean value to determine whether project is being decrypted or encrypted, during + start-up and tear-down respectively. + + .. data: salt + + Nonce value used for key derivation. Generated by reading 16 bytes + from /dev/urandom. + + :Default: ``os.urandom(16)`` + + .. data: salt_file + + Clear-text file containing the salt generated for key derivation. A new salt + will be generated each time the project is encrypted. After successfully + decrypting the project file (``project.crypt``), the salt file (``project.salt``) + will be deleted. + + :Default: ``project.salt`` """ def __init__(self): @@ -125,6 +170,13 @@ class PappyConfig(object): self.config_dict = {} self.global_config_dict = {} + + self.archive = 'project.archive' + self.crypt_dir = os.path.join(os.getcwd(), 'crypt') + self.crypt_file = 'project.crypt' + self.crypt_mode = None + self.salt = os.urandom(16) + self.salt_file = 'project.salt' def get_default_config(self): default_config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), @@ -133,6 +185,15 @@ class PappyConfig(object): settings = json.load(f) return settings + def get_project_files(self): + file_glob = glob.glob('*') + pp = os.path.join(os.getcwd()) + project_files = [pp+f for f in file_glob if os.path.isfile(pp+f)] + project_files.remove(self.salt_file) + project_files.remove(self.crypt_file) + return project_files + + @staticmethod def _parse_proxy_login(conf): proxy = {} diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py new file mode 100644 index 0000000..70a914b --- /dev/null +++ b/pappyproxy/crypto.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python + +import crochet +import glob +import os +import pappyproxy +import scrypt +import shutil +import twisted + +from . import compress +from base64 import b64encode, b64decode +from cryptography import Fernet +from twisted.internet import reactor, defer + +class Crypto(object): + def __init__(self, sessconfig): + self.config = sessconfig + self.archive = self.config.archive + self.compressor = compress.Compress(sessconfig) + + def encrypt_project(passwd): + """ + Compress and encrypt the project files, deleting clear-text files afterwards + """ + # Derive the key + key = crypto_ramp_up(passwd) + + # Instantiate the crypto module + fern = Fernet(key) + + # Create project archive and crypto archive + self.compressor.compress_project() + archive_file = open(self.archive, 'rb') + archive_crypt = open(self.config.crypt_file, 'wb') + + # Encrypt the archive read as a bytestring + crypt_token = fern.encrypt(archive_file) + archive_crypt.write(crypt_token) + + # Delete clear-text files + delete_clear_files() + + # Leave crypto working directory + os.chdir('../') + + @defer.inlineCallbacks + def decrypt_project(passwd): + """ + Decrypt and decompress the project files + """ + + # Create crypto working directory + crypto_path = os.path.join(os.getcwd(), pappy_config.crypt_dir) + os.mkdir(crypto_path) + + if os.path.isfile(self.config.crypt_file): + # Derive the key + key = crypto_ramp_up(passwd) + fern = Fernet(key) + + # Decrypt the project archive + archive_crypt = open(self.config.crypt_file, 'rb') + archive = fern.decrypt(archive_crypt) + + shutil.move(archive, crypto_path) + os.chdir(crypto_path) + self.compressor.decompress_project() + else: + project_files = self.config.get_project_files() + for pf in project_files: + shutil.copy2(pf, crypto_path) + os.chdir(crypto_path) + + + def crypto_ramp_up(passwd): + salt = "" + if os.path.isfile(self.config.salt_file): + salt = get_salt() + else: + salt = create_salt_file() + key = derive_key(passwd, salt) + return key + + def delete_clear_files(): + """ + Deletes all clear-text files left in the project directory. + """ + project_files = self.config.get_project_files() + for pf in project_files: + os.remove(pf) + + def delete_crypt_files(): + """ + Deletes all encrypted-text files in the project directory. + Forces generation of new salt after opening and closing the project. + Adds security in the case of a one-time compromise of the system. + """ + os.remove(self.config.salt_file) + os.remove(self.config.crypt_file) + + def create_salt_file(): + self.config.salt = urandom(16) + salt_file = open(self.config.salt_file, 'wb') + salt_file.write(self.config.salt) + salt_file.close() + return salt + + def get_salt(): + try: + salt_file = open(self.config.salt_file, 'rb') + salt = salt_file.readline() + except: + raise PappyException("Unable to read pappy.salt") + return salt + + def get_password(): + """ + Retrieve password from the user. Raise an exception if the + password is not capable of utf-8 encoding. + """ + encoded_passwd = "" + try: + passwd = raw_input("Enter a password: ") + encode_passwd = passwd.encode("utf-8") + except: + raise PappyException("Invalid password, try again") + return encoded_passwd + + def derive_key(passwd, salt): + """ + Derive a key sufficient for use as a cryptographic key + used to encrypt the project (currently: cryptography.Fernet). + + cryptography.Fernet utilizes AES-CBC-128, requiring a 32-byte key. + Parameter notes from the py-scrypt source-code: + https://bitbucket.org/mhallin/py-scrypt/ + + Compute scrypt(password, salt, N, r, p, buflen). + + The parameters r, p, and buflen must satisfy r * p < 2^30 and + buflen <= (2^32 - 1) * 32. The parameter N must be a power of 2 + greater than 1. N, r and p must all be positive. + + Notes for Python 2: + - `password` and `salt` must be str instances + - The result will be a str instance + + Notes for Python 3: + - `password` and `salt` can be both str and bytes. If they are str + instances, they wil be encoded with utf-8. + - The result will be a bytes instance + + Exceptions raised: + - TypeError on invalid input + - scrypt.error if scrypt failed + """ + + derived_key = "" + try: + dkey = scrypt.hash(passwd, salt, bufflen=32) + except e: + raise PappyException("Error deriving the key: ", e) + return derived_key diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index eeb8bff..ec6fb85 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -21,7 +21,9 @@ import signal from . import comm from . import config +from . import compress from . import context +from . import crypto from . import http from .console import ProxyCmd from twisted.enterprise import adbapi @@ -62,6 +64,8 @@ class PappySession(object): self.dbpool = None self.delete_data_on_quit = False self.ports = None + self.crypto = Crypto(sessconfig) + self.password = None @defer.inlineCallbacks def start(self): @@ -138,12 +142,25 @@ class PappySession(object): # Add cleanup to defer self.complete_defer = deferToThread(self.cons.cmdloop) self.complete_defer.addCallback(self.cleanup) - + + @defer.inlineCallbacks + def encrypt(self): + if self.password: + self.crypto.encrypt_project(self.password) + else: + self.password = self.crypto.get_password() + self.crypto.encrypt_project(self.password) + + @defer.inlineCallbacks + def decrypt(self): + self.password = self.crypto.get_password() + self.crypto.decrypt_project(self.password) + @defer.inlineCallbacks def cleanup(self, ignored=None): for port in self.ports: yield port.stopListening() - + if self.delete_data_on_quit: print 'Deleting temporary datafile' os.remove(self.config.datafile) @@ -153,6 +170,7 @@ def parse_args(): parser = argparse.ArgumentParser(description='An intercepting proxy for testing web applications.') parser.add_argument('-l', '--lite', help='Run the proxy in "lite" mode', action='store_true') + parser.add_argument('-c', '--crypt', type=str, nargs='?', help='Start pappy in "crypto" mode, optionally supply a name for the encrypted project archive [CRYPT]') args = parser.parse_args(sys.argv[1:]) settings = {} @@ -162,6 +180,13 @@ def parse_args(): else: settings['lite'] = False + if args.crypt: + settings['crypt'] = args.crypt + elif args.crypt == "": + settings['crypt'] = 'project.crypt' + else: + settings['crypt'] = None + return settings def set_text_factory(conn): @@ -189,7 +214,12 @@ def main(): session = PappySession(pappy_config) signal.signal(signal.SIGINT, inturrupt_handler) - if settings['lite']: + if settings['crypt']: + session.decrypt() + conf_settings = pappy_config.load_from_file('./config.json') + pappy_config.global_load_from_file() + session.delete_data_on_quit = False + elif settings['lite']: conf_settings = pappy_config.get_default_config() conf_settings['debug_dir'] = None conf_settings['debug_to_file'] = False From 870d2abbe80ccfab45191992059bb7b098929696 Mon Sep 17 00:00:00 2001 From: onizenso Date: Thu, 24 Mar 2016 20:56:35 +0000 Subject: [PATCH 03/21] Removed old compress and crypto plugins Got rid of the old plugin structure for the crypto and compress features. --- pappyproxy/plugins/compress/__init__.py | 0 pappyproxy/plugins/compress/compress.py | 89 ------------ pappyproxy/plugins/crypto/__init__.py | 0 pappyproxy/plugins/crypto/crypto.py | 182 ------------------------ pappyproxy/plugins/misc.py | 22 --- 5 files changed, 293 deletions(-) delete mode 100644 pappyproxy/plugins/compress/__init__.py delete mode 100644 pappyproxy/plugins/compress/compress.py delete mode 100644 pappyproxy/plugins/crypto/__init__.py delete mode 100644 pappyproxy/plugins/crypto/crypto.py diff --git a/pappyproxy/plugins/compress/__init__.py b/pappyproxy/plugins/compress/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pappyproxy/plugins/compress/compress.py b/pappyproxy/plugins/compress/compress.py deleted file mode 100644 index 18a320e..0000000 --- a/pappyproxy/plugins/compress/compress.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python - -import crochet -import glob -import pappyproxy - -import zipfile -import tarfile - -# This is a gross hack, please help -bz2 = None -try: - import bz2 -except: - print "BZ2 not installed on your system" - -from base64 import b64encode, b64decode -from os import getcwd, sep, path, urandom -from pappyproxy.plugins.misc import CryptoCompressUtils as ccu - -def compress_project(): - if bz2: - tar_project() - else: - zip_project() - -def decompress_project(): - if bz2: - untar_project() - else: - unzip_project() - -def zip_project(): - """ - Zip project files - - Using append mode (mode='a') will create a zip archive - if none exists in the project. - """ - try: - zf = zipfile.ZipFile(ZIPFILE, mode="a") - project_files = ccu.get_project_files() - for pf in project_files: - zf.write(pf) - zf.close() - except e: - raise PappyException("Error creating the zipfile", e) - pass - -def unzip_project(): - """ - Extract project files from decrypted zip archive. - Initially checks the zip archive's magic number and - attempts to extract pappy.json to validate integrity - of the zipfile. - """ - if not zipfile.is_zipfile(ZIPFILE): - raise PappyException("Project archive corrupted.") - - zf = zipfile.ZipFile(ZIPFILE) - - try: - zf.extract("config.json") - except e: - raise PappyException("Project archive contents corrupted. Error: ", e) - - zf.extractall() - -def tar_project(): - if tarfile.is_tarfile(BZ2FILE): - archive = tarfile.open(ccu.BZ2FILE, 'w:bz2') - project_files = ccu.get_project_files() - - # Read files line by line to accomodate larger files, e.g. the project database - for pf in project_files: - archive.add(pf) - archive.close() - -def untar_project(): - if tarfile.is_tarfile(BZ2FILE): - # Attempt to read the first 16 bytes of the archive - # Raise exception if there is a failure - project_files = ccu.get_project_files() - try: - with tarfile.open(BZ2FILE, "r:bz2") as archive: - for pf in project_files: - archive.add(pf) - except e: - raise PappyException("Project archive contents corrupted. Error: ", e diff --git a/pappyproxy/plugins/crypto/__init__.py b/pappyproxy/plugins/crypto/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pappyproxy/plugins/crypto/crypto.py b/pappyproxy/plugins/crypto/crypto.py deleted file mode 100644 index a63d929..0000000 --- a/pappyproxy/plugins/crypto/crypto.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python - -import crochet -import glob -import pappyproxy - -import scrypt -import twisted - -from base64 import b64encode, b64decode -from cryptography import Fernet -from os import getcwd, sep, path, remove, urandom -from pappyproxy.plugins import compress -from pappyproxy.plugins.misc import CryptoCompressUtils as ccu - - -def encrypt_project(passwd): - """ - Compress and encrypt the project files, deleting clear-text files afterwards - """ - # Derive the key - key = crypto_ramp_up(passwd) - # Instantiate the crypto module - fern = Fernet(key) - - compress.compress_project() - archive = None - if path.is_file(ZIPFILE): - archive = open(ccu.ZIPFILE, 'rb') - else: - archive = open(ccu.BZ2FILE, 'rb') - archive_crypt = open(ccu.CRYPTFILE, 'wb') - - # Encrypt the archive read as a bytestring - crypt_token = fern.encrypt(archive) - archive_crypt.write(crypt_token) - - # Delete clear-text files - delete_clear_files() - -def decrypt_project(passwd): - """ - Decompress and decrypt the project files - """ - # Derive the key - key = crypto_ramp_up(passwd) - fern = Fernet(key) - archive_crypt = open(ccu.CRYPTFILE, 'rb') - archive = fern.decrypt(archive_crypt) - compress.decompress_project() - delete_crypt_files() - - -def crypto_ramp_up(passwd): - salt = "" - if path.isfile(ccu.SALTFILE): - salt = get_salt() - else: - salt = create_salt() - key = derive_key(passwd, salt) - return key - -def delete_clear_files(): - """ - Deletes all clear-text files left in the project directory. - """ - project_files = ccu.get_project_files() - for pf in project_files: - os.remove(pf) - -def delete_crypt_files(): - """ - Deletes all encrypted-text files in the project directory. - Forces generation of new salt after opening and closing the project. - Adds security in the case of a one-time compromise of the system. - """ - os.remove(ccu.SALTFILE) - os.remove(ccu.CRYPTFILE) - -def create_salt(): - salt = b64encode(urandom(16)) - salt_file = open(ccu.SALTFILE, 'wb') - salt_file.write(salt) - salt_file.close() - return salt - -def get_salt(): - try: - salt_file = open(ccu.SALTFILE, 'rb') - salt = b64decode(salt_file.readline()) - except: - raise PappyException("Unable to read pappy.salt") - return salt - -def get_password(): - """ - Retrieve password from the user. Raise an exception if the - password is not capable of base64 encoding. - """ - encode_passwd = "" - try: - passwd = raw_input("Enter a password: ") - encode_passwd = b64encode(passwd.encode("utf-8")) - except: - raise PappyException("Invalid password, try again") - return encode_passwd - -def derive_key(passwd, salt): - """ - Derive a key sufficient for use as a cryptographic key - used to encrypt the project (currently: cryptography.Fernet). - - cryptography.Fernet utilizes AES-CBC-128, requiring a 32-byte key. - Parameter notes from the py-scrypt source-code: - https://bitbucket.org/mhallin/py-scrypt/ - - Compute scrypt(password, salt, N, r, p, buflen). - - The parameters r, p, and buflen must satisfy r * p < 2^30 and - buflen <= (2^32 - 1) * 32. The parameter N must be a power of 2 - greater than 1. N, r and p must all be positive. - - Notes for Python 2: - - `password` and `salt` must be str instances - - The result will be a str instance - - Notes for Python 3: - - `password` and `salt` can be both str and bytes. If they are str - instances, they wil be encoded with utf-8. - - The result will be a bytes instance - - Exceptions raised: - - TypeError on invalid input - - scrypt.error if scrypt failed - """ - - derived_key = "" - try: - dkey = scrypt.hash(passwd, salt, bufflen=32) - except e: - raise PappyException("Error deriving the key: ", e) - return derived_key - - -@crochet.wait_for(timeout=None) -@defer.inlineCallbacks -def cryptocmd(line): - """ - Encrypt/Decrypt local project directory - Usage: pappy -e - Details: - Pappy will create a compressed archive of local project files. - - The archive file is encrypted using the cryptography.Fernet module, - a user-supplied password and the scrypt key-derivation function. - - cryptography.Fernet uses AES-CBC-128 with HMAC256. This is merely - a starting point, and any help implementing a stronger crypto-system - is very welcome. Development is geared toward using - AES-256-GCM as the AEAD encryption mode to eliminate the need for Fernet and HMAC256. - SCrypt will still be used as the key derivation function until a public-key encryption - scheme is developed. - - See Encryption section of README.md for more information. - """ - - if isinstance(line, str): - args = crochet.split(line) - ## Encryption mode (Encrypt=0, Decrypt=1) - ## Set internally depending if plugin is called during pappy startup or shutdown - mode = args[0] - - ## Request the pasword from the user - passwd = get_passwd() - - if mode == ccu.ENCRYPT: - encrypt_project(passwd) - elif mode == ccu.DECRYPT: - decrypt_project(passwd) - else: - raise PappyException("Incorrect crypto mode") - diff --git a/pappyproxy/plugins/misc.py b/pappyproxy/plugins/misc.py index fb9bcd3..87934de 100644 --- a/pappyproxy/plugins/misc.py +++ b/pappyproxy/plugins/misc.py @@ -201,25 +201,3 @@ def load_cmds(cmd): cmd.add_aliases([ #('rpy', ''), ]) - -class CryptoCompressUtils(): - # Constants - ENCRYPT = 0 - DECRYPT = 1 - PROJECT_PATH = getcwd() + sep - ZIPFILE = PROJECT_PATH + "pappy.zip" - BZ2FILE = PROJECT_PATH + "pappy.bz2" - CRYPTFILE = "" - if path.isfile(ZIPFILE): - CRYPTFILE = ZIPFILE + ".crypt" - elsif path.isfile(BZ2FILE): - CRYPTFILE = BZ2FILE + ".crypt" - SALTFILE = PROJECT_PATH + "pappy.salt" - - def get_project_files(): - file_glob = glob.glob('*') - pp = PROJECT_PATH - project_files = [pp+f for f in file_glob if path.isfile(pp+f)] - project_files.remove(SALTFILE) - project_files.remove(CRYPTFILE) - return project_files From 022783026381f54ef87d45b404bda48aa6b698dd Mon Sep 17 00:00:00 2001 From: onizenso Date: Fri, 25 Mar 2016 16:28:02 +0000 Subject: [PATCH 04/21] Minor typo and syntax fixes --- pappyproxy/compress.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pappyproxy/compress.py b/pappyproxy/compress.py index ec4370d..662f30b 100644 --- a/pappyproxy/compress.py +++ b/pappyproxy/compress.py @@ -16,7 +16,6 @@ except: from base64 import b64encode, b64decode from os import getcwd, sep, path, urandom -from pappyproxy.plugins.misc import CryptoCompressUtils as ccu class Compress(object): def __init__(self, sessconfig): @@ -86,10 +85,10 @@ class Compress(object): if tarfile.is_tarfile(self.bz2_archive): # Attempt to read the first 16 bytes of the archive # Raise exception if there is a failure - project_files = self.config.get_project_files() + project_files = self.config.get_project_files() try: with tarfile.open(self.bz2_archive, "r:bz2") as archive: for pf in project_files: archive.add(pf) except e: - raise PappyException("Project archive contents corrupted. Error: ", e + raise PappyException("Project archive contents corrupted. Error: ", e) From ad37727c6bc9580ef9b56b2d5effb28aa9daa6c1 Mon Sep 17 00:00:00 2001 From: onizenso Date: Fri, 25 Mar 2016 16:33:25 +0000 Subject: [PATCH 05/21] Added scrypt to dependencies --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6fc998c..12fcdeb 100755 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ setup(name='pappyproxy', 'pytest-mock>=0.9.0', 'pytest-twisted>=1.5', 'pytest>=2.8.3', + 'scrypt>=0.7.1', 'service_identity>=14.0.0', 'twisted>=15.4.0', 'txsocksx>=1.15.0.2' From b56bb835586aa3ac99d6933f656bb4477ca3d7f5 Mon Sep 17 00:00:00 2001 From: onizenso Date: Fri, 25 Mar 2016 19:43:43 +0000 Subject: [PATCH 06/21] Debugging Crypto config and temp directory creation Attempting to get stub file creation and copying working. Fixed syntax errors, and now attempting to get password reading working in the test environment. --- pappyproxy/Makefile | 4 ++ pappyproxy/config.py | 17 ++++--- pappyproxy/crypto.py | 81 ++++++++++++++++++--------------- pappyproxy/pappy.py | 11 ++--- pappyproxy/tests/test_crypto.py | 51 +++++++++++++++++++++ 5 files changed, 112 insertions(+), 52 deletions(-) create mode 100644 pappyproxy/tests/test_crypto.py diff --git a/pappyproxy/Makefile b/pappyproxy/Makefile index 2e74f8d..2faa546 100644 --- a/pappyproxy/Makefile +++ b/pappyproxy/Makefile @@ -16,3 +16,7 @@ test-proxy: test-comm: py.test -v -rw --twisted tests/test_comm.py + +test-crypto: + py.test -v -rw --twisted tests/test_crypto.py + diff --git a/pappyproxy/config.py b/pappyproxy/config.py index 3b8f565..785c012 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -1,3 +1,4 @@ +import glob import json import os import shutil @@ -175,7 +176,7 @@ class PappyConfig(object): self.crypt_dir = os.path.join(os.getcwd(), 'crypt') self.crypt_file = 'project.crypt' self.crypt_mode = None - self.salt = os.urandom(16) + self.salt = os.urandom(16) self.salt_file = 'project.salt' def get_default_config(self): @@ -186,12 +187,14 @@ class PappyConfig(object): return settings def get_project_files(self): - file_glob = glob.glob('*') - pp = os.path.join(os.getcwd()) - project_files = [pp+f for f in file_glob if os.path.isfile(pp+f)] - project_files.remove(self.salt_file) - project_files.remove(self.crypt_file) - return project_files + file_glob = glob.glob('*') + pp = os.path.join(os.getcwd()) + project_files = [pp+f for f in file_glob if os.path.isfile(pp+f)] + if self.salt_file in project_files: + project_files.remove(self.salt_file) + if self.crypt_file in project_files: + project_files.remove(self.crypt_file) + return project_files @staticmethod diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 70a914b..ba9cb49 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -10,7 +10,7 @@ import twisted from . import compress from base64 import b64encode, b64decode -from cryptography import Fernet +from cryptography.fernet import Fernet from twisted.internet import reactor, defer class Crypto(object): @@ -18,16 +18,20 @@ class Crypto(object): self.config = sessconfig self.archive = self.config.archive self.compressor = compress.Compress(sessconfig) + self.key = None + self.password = None + self.salt = None - def encrypt_project(passwd): + def encrypt_project(self): """ Compress and encrypt the project files, deleting clear-text files afterwards """ - # Derive the key - key = crypto_ramp_up(passwd) + + # Get the password and salt, then derive the key + self.crypto_ramp_up() # Instantiate the crypto module - fern = Fernet(key) + fern = Fernet(self.key) # Create project archive and crypto archive self.compressor.compress_project() @@ -39,24 +43,26 @@ class Crypto(object): archive_crypt.write(crypt_token) # Delete clear-text files - delete_clear_files() + # delete_clear_files() # Leave crypto working directory os.chdir('../') - @defer.inlineCallbacks - def decrypt_project(passwd): + def decrypt_project(self): """ Decrypt and decompress the project files """ + # Get the password and salt, then derive the key + self.crypto_ramp_up() + # Create crypto working directory - crypto_path = os.path.join(os.getcwd(), pappy_config.crypt_dir) + crypto_path = os.path.join(os.getcwd(), self.config.crypt_dir) os.mkdir(crypto_path) if os.path.isfile(self.config.crypt_file): # Derive the key - key = crypto_ramp_up(passwd) + key = self.crypto_ramp_up() fern = Fernet(key) # Decrypt the project archive @@ -71,18 +77,14 @@ class Crypto(object): for pf in project_files: shutil.copy2(pf, crypto_path) os.chdir(crypto_path) - - - def crypto_ramp_up(passwd): - salt = "" - if os.path.isfile(self.config.salt_file): - salt = get_salt() - else: - salt = create_salt_file() - key = derive_key(passwd, salt) - return key - - def delete_clear_files(): + + def crypto_ramp_up(self): + if not self.password: + self.get_password() + self.set_salt() + self.derive_key() + + def delete_clear_files(self): """ Deletes all clear-text files left in the project directory. """ @@ -90,7 +92,7 @@ class Crypto(object): for pf in project_files: os.remove(pf) - def delete_crypt_files(): + def delete_crypt_files(self): """ Deletes all encrypted-text files in the project directory. Forces generation of new salt after opening and closing the project. @@ -99,22 +101,29 @@ class Crypto(object): os.remove(self.config.salt_file) os.remove(self.config.crypt_file) - def create_salt_file(): - self.config.salt = urandom(16) + def create_salt_file(self): salt_file = open(self.config.salt_file, 'wb') + + if not self.config.salt: + self.set_salt() + salt_file.write(self.config.salt) salt_file.close() - return salt - def get_salt(): + def set_salt_from_file(self): try: salt_file = open(self.config.salt_file, 'rb') - salt = salt_file.readline() + self.config.salt = salt_file.readline().strip() except: - raise PappyException("Unable to read pappy.salt") - return salt + raise PappyException("Unable to read project.salt") + + def set_salt(self): + if os.path.isfile(self.config.salt_file): + self.set_salt_from_file() + else: + self.config.salt = os.urandom(16) - def get_password(): + def get_password(self): """ Retrieve password from the user. Raise an exception if the password is not capable of utf-8 encoding. @@ -122,12 +131,11 @@ class Crypto(object): encoded_passwd = "" try: passwd = raw_input("Enter a password: ") - encode_passwd = passwd.encode("utf-8") + self.password = passwd.encode("utf-8") except: raise PappyException("Invalid password, try again") - return encoded_passwd - def derive_key(passwd, salt): + def derive_key(self): """ Derive a key sufficient for use as a cryptographic key used to encrypt the project (currently: cryptography.Fernet). @@ -156,9 +164,8 @@ class Crypto(object): - scrypt.error if scrypt failed """ - derived_key = "" try: - dkey = scrypt.hash(passwd, salt, bufflen=32) + if not self.key: + self.key = scrypt.hash(self.password, self.salt, bufflen=32) except e: raise PappyException("Error deriving the key: ", e) - return derived_key diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index ec6fb85..fa03160 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -64,7 +64,7 @@ class PappySession(object): self.dbpool = None self.delete_data_on_quit = False self.ports = None - self.crypto = Crypto(sessconfig) + self.crypto = crypto.Crypto(sessconfig) self.password = None @defer.inlineCallbacks @@ -145,16 +145,11 @@ class PappySession(object): @defer.inlineCallbacks def encrypt(self): - if self.password: - self.crypto.encrypt_project(self.password) - else: - self.password = self.crypto.get_password() - self.crypto.encrypt_project(self.password) + self.crypto.encrypt_project(self.password) @defer.inlineCallbacks def decrypt(self): - self.password = self.crypto.get_password() - self.crypto.decrypt_project(self.password) + self.crypto.decrypt_project() @defer.inlineCallbacks def cleanup(self, ignored=None): diff --git a/pappyproxy/tests/test_crypto.py b/pappyproxy/tests/test_crypto.py new file mode 100644 index 0000000..f707f36 --- /dev/null +++ b/pappyproxy/tests/test_crypto.py @@ -0,0 +1,51 @@ +import os +import pytest +import random +import string +from pappyproxy.session import Session +from pappyproxy.crypto import Crypto +from pappyproxy.config import PappyConfig + +@pytest.fixture +def conf(): + c = PappyConfig() + return c + +@pytest.fixture +def crypt(): + c = Crypto(conf()) + return c + +@pytest.fixture +def tmpname(): + cns = string.ascii_lowercase + string.ascii_uppercase + string.digits + tn = '' + for i in xrange(8): + tn += cns[random.randint(0,len(cns)-1)] + return tn + +tmpdir = '/tmp/test_crypto'+tmpname() +tmpfiles = ['cmdhistory', 'config.json', 'data.db'] + +def stub_files(): + enter_tmpdir() + for sf in tmpfiles: + with os.fdopen(os.open(sf, os.O_CREAT, 0o0600), 'r'): + pass + +def enter_tmpdir(): + if not os.path.isdir(tmpdir): + os.mkdir(tmpdir) + os.chdir(tmpdir) + +def test_decrypt_tmpdir(): + enter_tmpdir() + crypt().decrypt_project() + assert os.path.isdir(os.path.join(os.getcwd(), '../crypt')) + +def test_decrypt_copy_files(): + enter_tmpdir() + stub_files() + crypt().decrypt_project() + for tf in tmpfiles: + assert os.path.isfile(tf) From c32201fd05d3d35f737b06351ca0d0c3e7651763 Mon Sep 17 00:00:00 2001 From: onizenso Date: Fri, 25 Mar 2016 20:28:22 +0000 Subject: [PATCH 07/21] Tested and fixed file copying from `Crypto.decrypt_project` In the function for grabbing project files (`Config.get_project_files`) I was overcomplicating getting the current working directory. Simplified the process and removed the bug! --- pappyproxy/config.py | 18 +++--------------- pappyproxy/crypto.py | 18 +++++++++++------- pappyproxy/tests/test_crypto.py | 17 ++++++++++++++--- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/pappyproxy/config.py b/pappyproxy/config.py index 785c012..3d82fba 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -124,18 +124,6 @@ class PappyConfig(object): :Default: 'project.crypt' - .. data: crypt_mode - - Boolean value to determine whether project is being decrypted or encrypted, during - start-up and tear-down respectively. - - .. data: salt - - Nonce value used for key derivation. Generated by reading 16 bytes - from /dev/urandom. - - :Default: ``os.urandom(16)`` - .. data: salt_file Clear-text file containing the salt generated for key derivation. A new salt @@ -175,8 +163,6 @@ class PappyConfig(object): self.archive = 'project.archive' self.crypt_dir = os.path.join(os.getcwd(), 'crypt') self.crypt_file = 'project.crypt' - self.crypt_mode = None - self.salt = os.urandom(16) self.salt_file = 'project.salt' def get_default_config(self): @@ -188,12 +174,14 @@ class PappyConfig(object): def get_project_files(self): file_glob = glob.glob('*') - pp = os.path.join(os.getcwd()) + pp = os.getcwd() + os.sep project_files = [pp+f for f in file_glob if os.path.isfile(pp+f)] + if self.salt_file in project_files: project_files.remove(self.salt_file) if self.crypt_file in project_files: project_files.remove(self.crypt_file) + return project_files diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index ba9cb49..9194e1b 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -9,6 +9,7 @@ import shutil import twisted from . import compress +from .util import PappyException from base64 import b64encode, b64decode from cryptography.fernet import Fernet from twisted.internet import reactor, defer @@ -56,9 +57,10 @@ class Crypto(object): # Get the password and salt, then derive the key self.crypto_ramp_up() - # Create crypto working directory crypto_path = os.path.join(os.getcwd(), self.config.crypt_dir) - os.mkdir(crypto_path) + + if not os.path.isdir(crypto_path): + os.mkdir(crypto_path) if os.path.isfile(self.config.crypt_file): # Derive the key @@ -113,7 +115,7 @@ class Crypto(object): def set_salt_from_file(self): try: salt_file = open(self.config.salt_file, 'rb') - self.config.salt = salt_file.readline().strip() + self.salt = salt_file.readline().strip() except: raise PappyException("Unable to read project.salt") @@ -121,7 +123,7 @@ class Crypto(object): if os.path.isfile(self.config.salt_file): self.set_salt_from_file() else: - self.config.salt = os.urandom(16) + self.salt = os.urandom(16) def get_password(self): """ @@ -166,6 +168,8 @@ class Crypto(object): try: if not self.key: - self.key = scrypt.hash(self.password, self.salt, bufflen=32) - except e: - raise PappyException("Error deriving the key: ", e) + self.key = scrypt.hash(self.password, self.salt, buflen=32) + except TypeError, e: + raise PappyException("Scrypt failed with type error: ", e) + except scrypt.error, e: + raise PappyException("Scrypt failed with internal error: ", e) diff --git a/pappyproxy/tests/test_crypto.py b/pappyproxy/tests/test_crypto.py index f707f36..9939cd0 100644 --- a/pappyproxy/tests/test_crypto.py +++ b/pappyproxy/tests/test_crypto.py @@ -26,6 +26,7 @@ def tmpname(): tmpdir = '/tmp/test_crypto'+tmpname() tmpfiles = ['cmdhistory', 'config.json', 'data.db'] +tmp_pass = 'fugyeahbruh' def stub_files(): enter_tmpdir() @@ -40,12 +41,22 @@ def enter_tmpdir(): def test_decrypt_tmpdir(): enter_tmpdir() - crypt().decrypt_project() + c = crypt() + + # Stub out the password, working with stdout is a pain with pytest + c.password = tmp_pass + + c.decrypt_project() assert os.path.isdir(os.path.join(os.getcwd(), '../crypt')) def test_decrypt_copy_files(): enter_tmpdir() stub_files() - crypt().decrypt_project() + c = crypt() + + # Stub out the password, working with stdout is a pain with pytest + c.password = tmp_pass + + c.decrypt_project() for tf in tmpfiles: - assert os.path.isfile(tf) + assert os.path.isfile(os.path.join(os.getcwd(),tf)) From a3cb5f13edf5fa0bf69689e8d220831bdea49531 Mon Sep 17 00:00:00 2001 From: Nich Date: Mon, 28 Mar 2016 06:04:27 +0000 Subject: [PATCH 08/21] Fixed bugs with crypto.py, need to work on cleanup Project is now properly encrypting the archive, and now I just need to ensure proper decryption is happening. Also need to work on cleaning up clear text versions of the crypt project files. Need to write tests for flushing out edge cases. --- pappyproxy/compress.py | 38 +++++++++++++++++--------------------- pappyproxy/crypto.py | 6 +++--- pappyproxy/pappy.py | 9 +++++++-- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/pappyproxy/compress.py b/pappyproxy/compress.py index 662f30b..28d1e5d 100644 --- a/pappyproxy/compress.py +++ b/pappyproxy/compress.py @@ -23,19 +23,19 @@ class Compress(object): self.zip_archive = sessconfig.archive self.bz2_archive = sessconfig.archive - def compress_project(): + def compress_project(self): if bz2: - tar_project() + self.tar_project() else: - zip_project() + self.zip_project() - def decompress_project(): + def decompress_project(self): if bz2: - untar_project() + self.untar_project() else: - unzip_project() + self.unzip_project() - def zip_project(): + def zip_project(self): """ Zip project files @@ -52,7 +52,7 @@ class Compress(object): raise PappyException("Error creating the zipfile", e) pass - def unzip_project(): + def unzip_project(self): """ Extract project files from decrypted zip archive. Initially checks the zip archive's magic number and @@ -71,24 +71,20 @@ class Compress(object): zf.extractall() - def tar_project(): - if tarfile.is_tarfile(self.bz2_archive): - archive = tarfile.open(self.bz2_archive, 'w:bz2') - project_files = self.config.get_project_files() - - # Read files line by line to accomodate larger files, e.g. the project database - for pf in project_files: - archive.add(pf) + def tar_project(self): + archive = tarfile.open(self.bz2_archive, 'w:bz2') + project_files = self.config.get_project_files() + + # Read files line by line to accomodate larger files, e.g. the project database + for pf in project_files: + archive.add(pf) archive.close() - def untar_project(): + def untar_project(self): if tarfile.is_tarfile(self.bz2_archive): - # Attempt to read the first 16 bytes of the archive # Raise exception if there is a failure - project_files = self.config.get_project_files() try: with tarfile.open(self.bz2_archive, "r:bz2") as archive: - for pf in project_files: - archive.add(pf) + archive.extractall() except e: raise PappyException("Project archive contents corrupted. Error: ", e) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 9194e1b..d8fa3d2 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -36,7 +36,7 @@ class Crypto(object): # Create project archive and crypto archive self.compressor.compress_project() - archive_file = open(self.archive, 'rb') + archive_file = open(self.archive, 'rb').read() archive_crypt = open(self.config.crypt_file, 'wb') # Encrypt the archive read as a bytestring @@ -57,7 +57,7 @@ class Crypto(object): # Get the password and salt, then derive the key self.crypto_ramp_up() - crypto_path = os.path.join(os.getcwd(), self.config.crypt_dir) + crypto_path = self.config.crypt_dir if not os.path.isdir(crypto_path): os.mkdir(crypto_path) @@ -168,7 +168,7 @@ class Crypto(object): try: if not self.key: - self.key = scrypt.hash(self.password, self.salt, buflen=32) + self.key = b64encode(scrypt.hash(self.password, self.salt, buflen=32)) except TypeError, e: raise PappyException("Scrypt failed with type error: ", e) except scrypt.error, e: diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index fa03160..5f92190 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -145,11 +145,11 @@ class PappySession(object): @defer.inlineCallbacks def encrypt(self): - self.crypto.encrypt_project(self.password) + yield self.crypto.encrypt_project() @defer.inlineCallbacks def decrypt(self): - self.crypto.decrypt_project() + yield self.crypto.decrypt_project() @defer.inlineCallbacks def cleanup(self, ignored=None): @@ -159,6 +159,11 @@ class PappySession(object): if self.delete_data_on_quit: print 'Deleting temporary datafile' os.remove(self.config.datafile) + + # If currently in the crypt directory, + # encrypt the project, delete clear files + if os.getcwd() == self.config.crypt_dir: + self.encrypt() def parse_args(): # parses sys.argv and returns a settings dictionary From bf914e6f86155a36887f2180a4047c3a5646639f Mon Sep 17 00:00:00 2001 From: Nich Date: Mon, 28 Mar 2016 21:12:19 +0000 Subject: [PATCH 09/21] Worked out bugs for crypto mode, it works now! Crypto mode for pappy now works. Still need to work out the kinks for not supplying a project file to `pappy -c`, but when supplied encryption and decryption work. --- pappyproxy/compress.py | 18 ++---- pappyproxy/config.py | 17 +++-- pappyproxy/console.py | 26 ++++---- pappyproxy/crypto.py | 138 +++++++++++++++++++++-------------------- pappyproxy/pappy.py | 7 ++- 5 files changed, 109 insertions(+), 97 deletions(-) diff --git a/pappyproxy/compress.py b/pappyproxy/compress.py index 28d1e5d..427497c 100644 --- a/pappyproxy/compress.py +++ b/pappyproxy/compress.py @@ -7,11 +7,10 @@ import pappyproxy import zipfile import tarfile -# This is a gross hack, please help -bz2 = None try: import bz2 -except: +except ImportError: + bz2 = None print "BZ2 not installed on your system" from base64 import b64encode, b64decode @@ -44,12 +43,10 @@ class Compress(object): """ try: zf = zipfile.ZipFile(self.zip_archive, mode="a") - project_files = self.config.get_project_files() - for pf in project_files: - zf.write(pf) + zf.write(self.config.crypt_dir) zf.close() except e: - raise PappyException("Error creating the zipfile", e) + raise PappyException("Error creating the zipfile. Error: ", e) pass def unzip_project(self): @@ -73,11 +70,8 @@ class Compress(object): def tar_project(self): archive = tarfile.open(self.bz2_archive, 'w:bz2') - project_files = self.config.get_project_files() - # Read files line by line to accomodate larger files, e.g. the project database - for pf in project_files: - archive.add(pf) + archive.add(self.config.crypt_dir) archive.close() def untar_project(self): @@ -86,5 +80,5 @@ class Compress(object): try: with tarfile.open(self.bz2_archive, "r:bz2") as archive: archive.extractall() - except e: + except tarfile.ExtractError, e: raise PappyException("Project archive contents corrupted. Error: ", e) diff --git a/pappyproxy/config.py b/pappyproxy/config.py index 3d82fba..f2b71a6 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -99,13 +99,13 @@ class PappyConfig(object): The dictionary from ~/.pappy/global_config.json. It contains settings for Pappy that are specific to the current computer. Avoid putting settings here, especially if it involves specific projects. - + .. data: archive Project archive compressed as a ``tar.bz2`` archive if libraries available on the system, otherwise falls back to zip archive. - :Default: 'project.archive' + :Default: ``project.archive`` .. data: crypt_dir @@ -115,14 +115,20 @@ class PappyConfig(object): Compressed as a tar.bz2 archive if libraries available on the system, otherwise falls back to zip. - :Default: 'crypt' + :Default: ``crypt`` .. data: crypt_file Encrypted archive of the temporary working directory ``crypt_dir``. Compressed as a tar.bz2 archive if libraries available on the system, otherwise falls back to zip. - :Default: 'project.crypt' + :Default: ``project.crypt`` + + .. data: crypt_session + + Boolean variable to determine whether pappy started in crypto mode + + :Default: False .. data: salt_file @@ -161,8 +167,9 @@ class PappyConfig(object): self.global_config_dict = {} self.archive = 'project.archive' - self.crypt_dir = os.path.join(os.getcwd(), 'crypt') + self.crypt_dir = 'crypt' self.crypt_file = 'project.crypt' + self.crypt_session = False self.salt_file = 'project.salt' def get_default_config(self): diff --git a/pappyproxy/console.py b/pappyproxy/console.py index 7701da2..b670ba2 100644 --- a/pappyproxy/console.py +++ b/pappyproxy/console.py @@ -42,13 +42,15 @@ class ProxyCmd(cmd2.Cmd): self._cmds = {} self._aliases = {} - atexit.register(self.save_histfile) - readline.set_history_length(self.session.config.histsize) - if os.path.exists('cmdhistory'): - if self.session.config.histsize != 0: - readline.read_history_file('cmdhistory') - else: - os.remove('cmdhistory') + # Only read and save history when not in crypto mode + if not self.session.config.crypt_session: + atexit.register(self.save_histfile) + readline.set_history_length(self.session.config.histsize) + if os.path.exists('cmdhistory'): + if self.session.config.histsize != 0: + readline.read_history_file('cmdhistory') + else: + os.remove('cmdhistory') cmd2.Cmd.__init__(self, *args, **kwargs) @@ -110,10 +112,12 @@ class ProxyCmd(cmd2.Cmd): raise AttributeError(attr) def save_histfile(self): - # Write the command to the history file - if self.session.config.histsize != 0: - readline.set_history_length(self.session.config.histsize) - readline.write_history_file('cmdhistory') + # Only write to file if not in crypto mode + if not self.session.config.crypt_session: + # Write the command to the history file + if self.session.config.histsize != 0: + readline.set_history_length(self.session.config.histsize) + readline.write_history_file('cmdhistory') def get_names(self): # Hack to get cmd to recognize do_/etc functions as functions for things diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index d8fa3d2..14a96f1 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -27,58 +27,68 @@ class Crypto(object): """ Compress and encrypt the project files, deleting clear-text files afterwards """ + + # Leave the crypto working directory + os.chdir('../') # Get the password and salt, then derive the key self.crypto_ramp_up() - - # Instantiate the crypto module - fern = Fernet(self.key) - - # Create project archive and crypto archive + self.compressor.compress_project() - archive_file = open(self.archive, 'rb').read() + + # Create project and crypto archive + archive_file = open(self.archive, 'rb') archive_crypt = open(self.config.crypt_file, 'wb') # Encrypt the archive read as a bytestring - crypt_token = fern.encrypt(archive_file) + fern = Fernet(self.key) + crypt_token = fern.encrypt(archive_file.read()) archive_crypt.write(crypt_token) + + # Store the salt for the next decryption + self.create_salt_file() + + archive_file.close() + archive_crypt.close() # Delete clear-text files - # delete_clear_files() + self.delete_clear_files() - # Leave crypto working directory - os.chdir('../') def decrypt_project(self): """ Decrypt and decompress the project files """ - # Get the password and salt, then derive the key - self.crypto_ramp_up() + # If project hasn't been encrypted before, setup crypt working directory + crypt_fp = os.path.join(os.getcwd(), self.config.crypt_file) + if not os.path.isfile(crypt_fp): + os.mkdir(self.config.crypt_dir) - crypto_path = self.config.crypt_dir + project_files = self.config.get_project_files() + for pf in project_files: + shutil.copy2(pf, self.config.crypt_dir) + os.chdir(self.config.crypt_dir) - if not os.path.isdir(crypto_path): - os.mkdir(crypto_path) - - if os.path.isfile(self.config.crypt_file): - # Derive the key - key = self.crypto_ramp_up() - fern = Fernet(key) + # Otherwise, decrypt and decompress the project + else: + self.crypto_ramp_up() + fern = Fernet(self.key) # Decrypt the project archive - archive_crypt = open(self.config.crypt_file, 'rb') + archive_crypt = open(self.config.crypt_file, 'rb').read() + archive_file = open(self.config.archive, 'wb') archive = fern.decrypt(archive_crypt) + + archive_file.write(archive) + archive_file.close() - shutil.move(archive, crypto_path) - os.chdir(crypto_path) self.compressor.decompress_project() - else: - project_files = self.config.get_project_files() - for pf in project_files: - shutil.copy2(pf, crypto_path) - os.chdir(crypto_path) + + # Force generation of new salt and crypt archive + self.delete_crypt_files() + + os.chdir(self.config.crypt_dir) def crypto_ramp_up(self): if not self.password: @@ -86,56 +96,36 @@ class Crypto(object): self.set_salt() self.derive_key() - def delete_clear_files(self): - """ - Deletes all clear-text files left in the project directory. - """ - project_files = self.config.get_project_files() - for pf in project_files: - os.remove(pf) - - def delete_crypt_files(self): + def get_password(self): """ - Deletes all encrypted-text files in the project directory. - Forces generation of new salt after opening and closing the project. - Adds security in the case of a one-time compromise of the system. + Retrieve password from the user. Raise an exception if the + password is not capable of utf-8 encoding. """ - os.remove(self.config.salt_file) - os.remove(self.config.crypt_file) - - def create_salt_file(self): - salt_file = open(self.config.salt_file, 'wb') - - if not self.config.salt: - self.set_salt() - - salt_file.write(self.config.salt) - salt_file.close() - - def set_salt_from_file(self): + encoded_passwd = "" try: - salt_file = open(self.config.salt_file, 'rb') - self.salt = salt_file.readline().strip() + passwd = raw_input("Enter a password: ") + self.password = passwd.encode("utf-8") except: - raise PappyException("Unable to read project.salt") - + raise PappyException("Invalid password, try again") + def set_salt(self): if os.path.isfile(self.config.salt_file): self.set_salt_from_file() else: self.salt = os.urandom(16) - def get_password(self): - """ - Retrieve password from the user. Raise an exception if the - password is not capable of utf-8 encoding. - """ - encoded_passwd = "" + def set_salt_from_file(self): try: - passwd = raw_input("Enter a password: ") - self.password = passwd.encode("utf-8") + salt_file = open(self.config.salt_file, 'rb') + self.salt = salt_file.readline().strip() except: - raise PappyException("Invalid password, try again") + raise PappyException("Unable to read project.salt") + + def create_salt_file(self): + salt_file = open(self.config.salt_file, 'wb') + + salt_file.write(self.salt) + salt_file.close() def derive_key(self): """ @@ -173,3 +163,19 @@ class Crypto(object): raise PappyException("Scrypt failed with type error: ", e) except scrypt.error, e: raise PappyException("Scrypt failed with internal error: ", e) + + def delete_clear_files(self): + """ + Deletes all clear-text files left in the project directory. + """ + shutil.rmtree(self.config.crypt_dir) + os.remove(self.config.archive) + + def delete_crypt_files(self): + """ + Deletes all encrypted-text files in the project directory. + Forces generation of new salt after opening and closing the project. + Adds security in the case of a one-time compromise of the system. + """ + #os.remove(self.config.salt_file) + os.remove(self.config.crypt_file) diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index 5f92190..011f685 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -160,9 +160,8 @@ class PappySession(object): print 'Deleting temporary datafile' os.remove(self.config.datafile) - # If currently in the crypt directory, - # encrypt the project, delete clear files - if os.getcwd() == self.config.crypt_dir: + # Encrypt the project when in crypto mode + if self.config.crypt_session: self.encrypt() def parse_args(): @@ -215,6 +214,8 @@ def main(): signal.signal(signal.SIGINT, inturrupt_handler) if settings['crypt']: + pappy_config.crypt_file = settings['crypt'] + pappy_config.crypt_session = True session.decrypt() conf_settings = pappy_config.load_from_file('./config.json') pappy_config.global_load_from_file() From e7d1f754358b29d9bb71791a3472b173a24d0f86 Mon Sep 17 00:00:00 2001 From: Nich Date: Mon, 28 Mar 2016 21:24:59 +0000 Subject: [PATCH 10/21] Added exception handling for failure to decrypt When entering the wrong password, fernet throws an 'InvalidToken' exception. This is now handled, but does not fully shutdown pappy. Working on asking for password multiple times, then shutting down completely after reasonable amount of total tries, e.g. 3. --- pappyproxy/crypto.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 14a96f1..289de72 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -11,7 +11,7 @@ import twisted from . import compress from .util import PappyException from base64 import b64encode, b64decode -from cryptography.fernet import Fernet +from cryptography.fernet import Fernet, InvalidToken from twisted.internet import reactor, defer class Crypto(object): @@ -78,7 +78,10 @@ class Crypto(object): # Decrypt the project archive archive_crypt = open(self.config.crypt_file, 'rb').read() archive_file = open(self.config.archive, 'wb') - archive = fern.decrypt(archive_crypt) + try: + archive = fern.decrypt(archive_crypt) + except InvalidToken: + raise PappyException("Problem decrypting the file, restart pappy to try again") archive_file.write(archive) archive_file.close() From ff8595e8f40f6177a69b732c92d67c9df5a15d0d Mon Sep 17 00:00:00 2001 From: Nich Date: Mon, 28 Mar 2016 21:51:56 +0000 Subject: [PATCH 11/21] Minor changes, decrypt project within PappySession Changed to decrypting project to within the PappySession object, instead of in main. More maintainable, and makes more sense. --- pappyproxy/compress.py | 9 ++++----- pappyproxy/crypto.py | 4 +++- pappyproxy/pappy.py | 10 ++++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pappyproxy/compress.py b/pappyproxy/compress.py index 427497c..c19761b 100644 --- a/pappyproxy/compress.py +++ b/pappyproxy/compress.py @@ -45,9 +45,8 @@ class Compress(object): zf = zipfile.ZipFile(self.zip_archive, mode="a") zf.write(self.config.crypt_dir) zf.close() - except e: - raise PappyException("Error creating the zipfile. Error: ", e) - pass + except zipfile.LargeZipFile as e: + raise PappyException("Project zipfile too large. Error: ", e) def unzip_project(self): """ @@ -63,7 +62,7 @@ class Compress(object): try: zf.extract("config.json") - except e: + except zipfile.BadZipfile as e: raise PappyException("Project archive contents corrupted. Error: ", e) zf.extractall() @@ -80,5 +79,5 @@ class Compress(object): try: with tarfile.open(self.bz2_archive, "r:bz2") as archive: archive.extractall() - except tarfile.ExtractError, e: + except tarfile.ExtractError as e: raise PappyException("Project archive contents corrupted. Error: ", e) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 289de72..9912880 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -82,6 +82,8 @@ class Crypto(object): archive = fern.decrypt(archive_crypt) except InvalidToken: raise PappyException("Problem decrypting the file, restart pappy to try again") + reactor.stop() + defer.returnValue(None) archive_file.write(archive) archive_file.close() @@ -162,7 +164,7 @@ class Crypto(object): try: if not self.key: self.key = b64encode(scrypt.hash(self.password, self.salt, buflen=32)) - except TypeError, e: + except TypeError as e: raise PappyException("Scrypt failed with type error: ", e) except scrypt.error, e: raise PappyException("Scrypt failed with internal error: ", e) diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index 011f685..cedf407 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -70,6 +70,12 @@ class PappySession(object): @defer.inlineCallbacks def start(self): from . import proxy, plugin + + if self.config.crypt_session: + self.decrypt() + self.config.load_from_file('./config.json') + self.config.global_load_from_file() + self.delete_data_on_quit = False # If the data file doesn't exist, create it with restricted permissions if not os.path.isfile(self.config.datafile): @@ -216,10 +222,6 @@ def main(): if settings['crypt']: pappy_config.crypt_file = settings['crypt'] pappy_config.crypt_session = True - session.decrypt() - conf_settings = pappy_config.load_from_file('./config.json') - pappy_config.global_load_from_file() - session.delete_data_on_quit = False elif settings['lite']: conf_settings = pappy_config.get_default_config() conf_settings['debug_dir'] = None From 5ceedddd1a7ddc797fc3f13514e68c629989ad6f Mon Sep 17 00:00:00 2001 From: Onics Date: Mon, 4 Apr 2016 19:33:21 -0400 Subject: [PATCH 12/21] Fixed minor bugs in decrypting project Project now decrypts properly and fails out loudly when incorrect password is supplied. Must supply project name via command line now. --- pappyproxy/crypto.py | 70 ++++++++++++++++++++++++++++---------------- pappyproxy/pappy.py | 23 ++++++++++----- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 9912880..90fdbd6 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -17,7 +17,7 @@ from twisted.internet import reactor, defer class Crypto(object): def __init__(self, sessconfig): self.config = sessconfig - self.archive = self.config.archive + self.archive = sessconfig.archive self.compressor = compress.Compress(sessconfig) self.key = None self.password = None @@ -31,15 +31,15 @@ class Crypto(object): # Leave the crypto working directory os.chdir('../') - # Get the password and salt, then derive the key - self.crypto_ramp_up() - self.compressor.compress_project() # Create project and crypto archive archive_file = open(self.archive, 'rb') archive_crypt = open(self.config.crypt_file, 'wb') - + + # Get the password and salt, then derive the key + self.crypto_ramp_up() + # Encrypt the archive read as a bytestring fern = Fernet(self.key) crypt_token = fern.encrypt(archive_file.read()) @@ -61,46 +61,64 @@ class Crypto(object): """ # If project hasn't been encrypted before, setup crypt working directory - crypt_fp = os.path.join(os.getcwd(), self.config.crypt_file) - if not os.path.isfile(crypt_fp): + if not os.path.isfile(self.config.crypt_file): os.mkdir(self.config.crypt_dir) project_files = self.config.get_project_files() for pf in project_files: shutil.copy2(pf, self.config.crypt_dir) os.chdir(self.config.crypt_dir) + return True # Otherwise, decrypt and decompress the project else: - self.crypto_ramp_up() - fern = Fernet(self.key) - - # Decrypt the project archive archive_crypt = open(self.config.crypt_file, 'rb').read() archive_file = open(self.config.archive, 'wb') - try: - archive = fern.decrypt(archive_crypt) - except InvalidToken: - raise PappyException("Problem decrypting the file, restart pappy to try again") - reactor.stop() - defer.returnValue(None) - + + retries = 3 + while True: + try: + self.crypto_ramp_up() + fern = Fernet(self.key) + archive = fern.decrypt(archive_crypt) + break + except InvalidToken: + print "Invalid password" + retries -= 1 + # Quit pappy if user doesn't retry + # or if all retries exhuasted + if not self.confirm_password_retry() or retries <= 0: + return False + else: + self.password = None + self.key = None + self.salt = None + pass + archive_file.write(archive) archive_file.close() - + self.compressor.decompress_project() - + # Force generation of new salt and crypt archive self.delete_crypt_files() os.chdir(self.config.crypt_dir) - + return True + + def confirm_password_retry(self): + answer = raw_input("Would you like to re-enter your password? (y/n)").strip() + if answer[0] == "y" or answer[0] == "Y": + return True + else: + return False + def crypto_ramp_up(self): if not self.password: self.get_password() self.set_salt() self.derive_key() - + def get_password(self): """ Retrieve password from the user. Raise an exception if the @@ -108,24 +126,24 @@ class Crypto(object): """ encoded_passwd = "" try: - passwd = raw_input("Enter a password: ") + passwd = raw_input("Enter a password: ").strip() self.password = passwd.encode("utf-8") except: raise PappyException("Invalid password, try again") - + def set_salt(self): if os.path.isfile(self.config.salt_file): self.set_salt_from_file() else: self.salt = os.urandom(16) - + def set_salt_from_file(self): try: salt_file = open(self.config.salt_file, 'rb') self.salt = salt_file.readline().strip() except: raise PappyException("Unable to read project.salt") - + def create_salt_file(self): salt_file = open(self.config.salt_file, 'wb') diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index cedf407..024709f 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -65,7 +65,6 @@ class PappySession(object): self.delete_data_on_quit = False self.ports = None self.crypto = crypto.Crypto(sessconfig) - self.password = None @defer.inlineCallbacks def start(self): @@ -155,8 +154,14 @@ class PappySession(object): @defer.inlineCallbacks def decrypt(self): - yield self.crypto.decrypt_project() - + # Attempt to decrypt project archive + if self.crypto.decrypt_project(): + yield True + # Quit pappy on failure + else: + reactor.stop() + defer.returnValue(None) + @defer.inlineCallbacks def cleanup(self, ignored=None): for port in self.ports: @@ -175,7 +180,12 @@ def parse_args(): parser = argparse.ArgumentParser(description='An intercepting proxy for testing web applications.') parser.add_argument('-l', '--lite', help='Run the proxy in "lite" mode', action='store_true') - parser.add_argument('-c', '--crypt', type=str, nargs='?', help='Start pappy in "crypto" mode, optionally supply a name for the encrypted project archive [CRYPT]') + try: + parser.add_argument('-c', '--crypt', type=str, nargs=1, help='Start pappy in "crypto" mode, must supply a name for the encrypted project archive [CRYPT]') + except: + print 'Must supply a project name: pappy -c ' + reactor.stop() + defer.returnValue(None) args = parser.parse_args(sys.argv[1:]) settings = {} @@ -186,9 +196,8 @@ def parse_args(): settings['lite'] = False if args.crypt: - settings['crypt'] = args.crypt - elif args.crypt == "": - settings['crypt'] = 'project.crypt' + # Convert from single-item list produced by argparse `nargs=1` + settings['crypt'] = args.crypt[0].encode('utf-8') else: settings['crypt'] = None From f5c53add9c13d6b01057ff3dde742e7d0098d4d9 Mon Sep 17 00:00:00 2001 From: Nich Date: Tue, 5 Apr 2016 18:10:04 -0400 Subject: [PATCH 13/21] Minor typography fixes --- pappyproxy/crypto.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 90fdbd6..56fdf7d 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -22,7 +22,7 @@ class Crypto(object): self.key = None self.password = None self.salt = None - + def encrypt_project(self): """ Compress and encrypt the project files, deleting clear-text files afterwards @@ -30,7 +30,7 @@ class Crypto(object): # Leave the crypto working directory os.chdir('../') - + self.compressor.compress_project() # Create project and crypto archive @@ -44,7 +44,7 @@ class Crypto(object): fern = Fernet(self.key) crypt_token = fern.encrypt(archive_file.read()) archive_crypt.write(crypt_token) - + # Store the salt for the next decryption self.create_salt_file() @@ -59,7 +59,7 @@ class Crypto(object): """ Decrypt and decompress the project files """ - + # If project hasn't been encrypted before, setup crypt working directory if not os.path.isfile(self.config.crypt_file): os.mkdir(self.config.crypt_dir) @@ -74,7 +74,7 @@ class Crypto(object): else: archive_crypt = open(self.config.crypt_file, 'rb').read() archive_file = open(self.config.archive, 'wb') - + retries = 3 while True: try: @@ -146,39 +146,39 @@ class Crypto(object): def create_salt_file(self): salt_file = open(self.config.salt_file, 'wb') - + salt_file.write(self.salt) salt_file.close() - + def derive_key(self): """ Derive a key sufficient for use as a cryptographic key used to encrypt the project (currently: cryptography.Fernet). - + cryptography.Fernet utilizes AES-CBC-128, requiring a 32-byte key. Parameter notes from the py-scrypt source-code: https://bitbucket.org/mhallin/py-scrypt/ - + Compute scrypt(password, salt, N, r, p, buflen). - + The parameters r, p, and buflen must satisfy r * p < 2^30 and buflen <= (2^32 - 1) * 32. The parameter N must be a power of 2 greater than 1. N, r and p must all be positive. - + Notes for Python 2: - `password` and `salt` must be str instances - The result will be a str instance - + Notes for Python 3: - `password` and `salt` can be both str and bytes. If they are str instances, they wil be encoded with utf-8. - The result will be a bytes instance - + Exceptions raised: - TypeError on invalid input - scrypt.error if scrypt failed """ - + try: if not self.key: self.key = b64encode(scrypt.hash(self.password, self.salt, buflen=32)) @@ -186,7 +186,7 @@ class Crypto(object): raise PappyException("Scrypt failed with type error: ", e) except scrypt.error, e: raise PappyException("Scrypt failed with internal error: ", e) - + def delete_clear_files(self): """ Deletes all clear-text files left in the project directory. From f7d8df69cc43e73930aa7a35ac66fd3734fcdb54 Mon Sep 17 00:00:00 2001 From: Nich Date: Tue, 5 Apr 2016 18:30:49 -0400 Subject: [PATCH 14/21] Pep8 corrections --- pappyproxy/compress.py | 31 ++++++------- pappyproxy/crypto.py | 100 +++++++++++++++++++++-------------------- pappyproxy/pappy.py | 13 +++--- 3 files changed, 75 insertions(+), 69 deletions(-) diff --git a/pappyproxy/compress.py b/pappyproxy/compress.py index c19761b..2204088 100644 --- a/pappyproxy/compress.py +++ b/pappyproxy/compress.py @@ -14,7 +14,8 @@ except ImportError: print "BZ2 not installed on your system" from base64 import b64encode, b64decode -from os import getcwd, sep, path, urandom +from os import getcwd, sep, path, urandom + class Compress(object): def __init__(self, sessconfig): @@ -27,17 +28,17 @@ class Compress(object): self.tar_project() else: self.zip_project() - + def decompress_project(self): if bz2: self.untar_project() else: self.unzip_project() - + def zip_project(self): """ Zip project files - + Using append mode (mode='a') will create a zip archive if none exists in the project. """ @@ -47,32 +48,32 @@ class Compress(object): zf.close() except zipfile.LargeZipFile as e: raise PappyException("Project zipfile too large. Error: ", e) - + def unzip_project(self): """ Extract project files from decrypted zip archive. Initially checks the zip archive's magic number and - attempts to extract pappy.json to validate integrity + attempts to extract pappy.json to validate integrity of the zipfile. """ if not zipfile.is_zipfile(self.zip_archive): raise PappyException("Project archive corrupted.") - + zf = zipfile.ZipFile(self.zip_archive) - + try: zf.extract("config.json") except zipfile.BadZipfile as e: - raise PappyException("Project archive contents corrupted. Error: ", e) - + raise PappyException("Zip archive corrupted. Error: ", e) + zf.extractall() - + def tar_project(self): archive = tarfile.open(self.bz2_archive, 'w:bz2') - + archive.add(self.config.crypt_dir) - archive.close() - + archive.close() + def untar_project(self): if tarfile.is_tarfile(self.bz2_archive): # Raise exception if there is a failure @@ -80,4 +81,4 @@ class Compress(object): with tarfile.open(self.bz2_archive, "r:bz2") as archive: archive.extractall() except tarfile.ExtractError as e: - raise PappyException("Project archive contents corrupted. Error: ", e) + raise PappyException("Tar archive corrupted. Error: ", e) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 56fdf7d..40359b2 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -14,53 +14,55 @@ from base64 import b64encode, b64decode from cryptography.fernet import Fernet, InvalidToken from twisted.internet import reactor, defer + class Crypto(object): def __init__(self, sessconfig): self.config = sessconfig - self.archive = sessconfig.archive + self.archive = sessconfig.archive self.compressor = compress.Compress(sessconfig) self.key = None self.password = None self.salt = None - + def encrypt_project(self): """ - Compress and encrypt the project files, deleting clear-text files afterwards + Compress and encrypt the project files, + deleting clear-text files afterwards """ - + # Leave the crypto working directory os.chdir('../') - + self.compressor.compress_project() - - # Create project and crypto archive - archive_file = open(self.archive, 'rb') + + # Create project and crypto archive + archive_file = open(self.archive, 'rb') archive_crypt = open(self.config.crypt_file, 'wb') - + # Get the password and salt, then derive the key self.crypto_ramp_up() - + # Encrypt the archive read as a bytestring fern = Fernet(self.key) crypt_token = fern.encrypt(archive_file.read()) archive_crypt.write(crypt_token) - + # Store the salt for the next decryption self.create_salt_file() archive_file.close() archive_crypt.close() - + # Delete clear-text files self.delete_clear_files() - - + def decrypt_project(self): """ Decrypt and decompress the project files """ - - # If project hasn't been encrypted before, setup crypt working directory + + # If project hasn't been encrypted before, + # setup crypt working directory if not os.path.isfile(self.config.crypt_file): os.mkdir(self.config.crypt_dir) @@ -69,17 +71,17 @@ class Crypto(object): shutil.copy2(pf, self.config.crypt_dir) os.chdir(self.config.crypt_dir) return True - - # Otherwise, decrypt and decompress the project - else: + + # Otherwise, decrypt and decompress the project + else: archive_crypt = open(self.config.crypt_file, 'rb').read() archive_file = open(self.config.archive, 'wb') - + retries = 3 while True: try: - self.crypto_ramp_up() - fern = Fernet(self.key) + self.crypto_ramp_up() + fern = Fernet(self.key) archive = fern.decrypt(archive_crypt) break except InvalidToken: @@ -93,35 +95,35 @@ class Crypto(object): self.password = None self.key = None self.salt = None - pass - + pass + archive_file.write(archive) archive_file.close() - + self.compressor.decompress_project() - + # Force generation of new salt and crypt archive self.delete_crypt_files() - + os.chdir(self.config.crypt_dir) return True - + def confirm_password_retry(self): - answer = raw_input("Would you like to re-enter your password? (y/n)").strip() + answer = raw_input("Re-enter your password? (y/n)").strip() if answer[0] == "y" or answer[0] == "Y": return True else: return False - + def crypto_ramp_up(self): if not self.password: self.get_password() self.set_salt() self.derive_key() - + def get_password(self): """ - Retrieve password from the user. Raise an exception if the + Retrieve password from the user. Raise an exception if the password is not capable of utf-8 encoding. """ encoded_passwd = "" @@ -130,75 +132,75 @@ class Crypto(object): self.password = passwd.encode("utf-8") except: raise PappyException("Invalid password, try again") - + def set_salt(self): if os.path.isfile(self.config.salt_file): self.set_salt_from_file() else: - self.salt = os.urandom(16) - + self.salt = os.urandom(16) + def set_salt_from_file(self): try: salt_file = open(self.config.salt_file, 'rb') self.salt = salt_file.readline().strip() except: raise PappyException("Unable to read project.salt") - + def create_salt_file(self): salt_file = open(self.config.salt_file, 'wb') - + salt_file.write(self.salt) salt_file.close() - + def derive_key(self): """ Derive a key sufficient for use as a cryptographic key used to encrypt the project (currently: cryptography.Fernet). - + cryptography.Fernet utilizes AES-CBC-128, requiring a 32-byte key. Parameter notes from the py-scrypt source-code: https://bitbucket.org/mhallin/py-scrypt/ - + Compute scrypt(password, salt, N, r, p, buflen). - + The parameters r, p, and buflen must satisfy r * p < 2^30 and buflen <= (2^32 - 1) * 32. The parameter N must be a power of 2 greater than 1. N, r and p must all be positive. - + Notes for Python 2: - `password` and `salt` must be str instances - The result will be a str instance - + Notes for Python 3: - `password` and `salt` can be both str and bytes. If they are str instances, they wil be encoded with utf-8. - The result will be a bytes instance - + Exceptions raised: - TypeError on invalid input - scrypt.error if scrypt failed """ - + try: if not self.key: - self.key = b64encode(scrypt.hash(self.password, self.salt, buflen=32)) + shash = scrypt.hash(self.password, self.salt, buflen=32) + self.key = b64encode(shash) except TypeError as e: raise PappyException("Scrypt failed with type error: ", e) except scrypt.error, e: raise PappyException("Scrypt failed with internal error: ", e) - + def delete_clear_files(self): """ Deletes all clear-text files left in the project directory. """ shutil.rmtree(self.config.crypt_dir) os.remove(self.config.archive) - + def delete_crypt_files(self): """ Deletes all encrypted-text files in the project directory. Forces generation of new salt after opening and closing the project. Adds security in the case of a one-time compromise of the system. """ - #os.remove(self.config.salt_file) os.remove(self.config.crypt_file) diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index 024709f..364fa07 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -159,9 +159,9 @@ class PappySession(object): yield True # Quit pappy on failure else: - reactor.stop() + reactor.stop() defer.returnValue(None) - + @defer.inlineCallbacks def cleanup(self, ignored=None): for port in self.ports: @@ -181,10 +181,13 @@ def parse_args(): parser = argparse.ArgumentParser(description='An intercepting proxy for testing web applications.') parser.add_argument('-l', '--lite', help='Run the proxy in "lite" mode', action='store_true') try: - parser.add_argument('-c', '--crypt', type=str, nargs=1, help='Start pappy in "crypto" mode, must supply a name for the encrypted project archive [CRYPT]') + hlpmsg = 'Start pappy in "crypto" mode,'+ + 'must supply a name for the encrypted'+ + 'project archive [CRYPT]' + parser.add_argument('-c', '--crypt', type=str, nargs=1, help=hlpmsg) except: print 'Must supply a project name: pappy -c ' - reactor.stop() + reactor.stop() defer.returnValue(None) args = parser.parse_args(sys.argv[1:]) @@ -197,7 +200,7 @@ def parse_args(): if args.crypt: # Convert from single-item list produced by argparse `nargs=1` - settings['crypt'] = args.crypt[0].encode('utf-8') + settings['crypt'] = args.crypt[0].encode('utf-8') else: settings['crypt'] = None From 6b8861058ecba0e66e6aef5034e8f54e2ed0c379 Mon Sep 17 00:00:00 2001 From: Nich Date: Tue, 5 Apr 2016 18:45:38 -0400 Subject: [PATCH 15/21] Minor syntax error fix --- pappyproxy/pappy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index 364fa07..0fb3678 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -181,9 +181,9 @@ def parse_args(): parser = argparse.ArgumentParser(description='An intercepting proxy for testing web applications.') parser.add_argument('-l', '--lite', help='Run the proxy in "lite" mode', action='store_true') try: - hlpmsg = 'Start pappy in "crypto" mode,'+ - 'must supply a name for the encrypted'+ - 'project archive [CRYPT]' + hlpmsg = ''.join(['Start pappy in "crypto" mode,', + 'must supply a name for the encrypted', + 'project archive [CRYPT]']) parser.add_argument('-c', '--crypt', type=str, nargs=1, help=hlpmsg) except: print 'Must supply a project name: pappy -c ' From 772e7ee50725b2d00bdacc141c5b4b606c053138 Mon Sep 17 00:00:00 2001 From: Nich Date: Tue, 5 Apr 2016 19:04:38 -0400 Subject: [PATCH 16/21] Fixed error when user fails to enter correct password Pappy would raise exceptions and continued project in clear-text when the user failed to enter the correct decryption password. Added a boolean status variable to config `crypt_success` that gets set to true when project decrypts correctly, otherwise it is set to false. --- pappyproxy/config.py | 1 + pappyproxy/crypto.py | 2 ++ pappyproxy/pappy.py | 8 +++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pappyproxy/config.py b/pappyproxy/config.py index f2b71a6..ed29e8a 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -170,6 +170,7 @@ class PappyConfig(object): self.crypt_dir = 'crypt' self.crypt_file = 'project.crypt' self.crypt_session = False + self.crypt_success = False self.salt_file = 'project.salt' def get_default_config(self): diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 40359b2..a8c16cd 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -90,6 +90,7 @@ class Crypto(object): # Quit pappy if user doesn't retry # or if all retries exhuasted if not self.confirm_password_retry() or retries <= 0: + self.config.crypt_success = False return False else: self.password = None @@ -106,6 +107,7 @@ class Crypto(object): self.delete_crypt_files() os.chdir(self.config.crypt_dir) + self.config.crypt_success = True return True def confirm_password_retry(self): diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index 0fb3678..0f248c9 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -72,9 +72,11 @@ class PappySession(object): if self.config.crypt_session: self.decrypt() - self.config.load_from_file('./config.json') - self.config.global_load_from_file() - self.delete_data_on_quit = False + + if self.config.crypt_success: + self.config.load_from_file('./config.json') + self.config.global_load_from_file() + self.delete_data_on_quit = False # If the data file doesn't exist, create it with restricted permissions if not os.path.isfile(self.config.datafile): From 587cf75058a1e1e4ce6b52449c8050defaa8a3f1 Mon Sep 17 00:00:00 2001 From: Nich Date: Wed, 6 Apr 2016 17:11:44 -0400 Subject: [PATCH 17/21] Fixed issue of system exiting during active session When the entire OS halts during the middle of a pappy crypto session, the `crypt` working directory is left unencrypted. To fix this, I added another conditional into `crypto.py`'s decryption function to enter an existing crypto working directory, so that when the project finishes it will encrypt/exit properly. --- pappyproxy/crypto.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index a8c16cd..f2aa2be 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -61,19 +61,8 @@ class Crypto(object): Decrypt and decompress the project files """ - # If project hasn't been encrypted before, - # setup crypt working directory - if not os.path.isfile(self.config.crypt_file): - os.mkdir(self.config.crypt_dir) - - project_files = self.config.get_project_files() - for pf in project_files: - shutil.copy2(pf, self.config.crypt_dir) - os.chdir(self.config.crypt_dir) - return True - - # Otherwise, decrypt and decompress the project - else: + # Decrypt and decompress the project if crypt_file exists + if os.path.isfile(self.config.crypt_file): archive_crypt = open(self.config.crypt_file, 'rb').read() archive_file = open(self.config.archive, 'wb') @@ -109,6 +98,22 @@ class Crypto(object): os.chdir(self.config.crypt_dir) self.config.crypt_success = True return True + # If project exited before encrypting the working directory + # change to the working directory to resume the session + elif os.path.isdir(self.config.crypt_dir): + os.chdir(self.config.crypt_dir) + return True + # If project hasn't been encrypted before, + # setup crypt working directory + else: + os.mkdir(self.config.crypt_dir) + + project_files = self.config.get_project_files() + for pf in project_files: + shutil.copy2(pf, self.config.crypt_dir) + os.chdir(self.config.crypt_dir) + return True + def confirm_password_retry(self): answer = raw_input("Re-enter your password? (y/n)").strip() From 2df463fc79f8a0cd4f09b163620c9e6f9cc10693 Mon Sep 17 00:00:00 2001 From: Nich Date: Mon, 11 Apr 2016 18:36:47 -0400 Subject: [PATCH 18/21] Issues with decrypting project Pappy is no longer reading the salt correctly, and now fails out when raising an exception. Need to fix exception handling. --- pappyproxy/config.py | 14 ++++---------- pappyproxy/crypto.py | 38 ++++++++++++++++++++++++++------------ pappyproxy/pappy.py | 19 ++++++++++++------- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/pappyproxy/config.py b/pappyproxy/config.py index ed29e8a..04abe1d 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -130,14 +130,11 @@ class PappyConfig(object): :Default: False - .. data: salt_file + .. data: salt_len - Clear-text file containing the salt generated for key derivation. A new salt - will be generated each time the project is encrypted. After successfully - decrypting the project file (``project.crypt``), the salt file (``project.salt``) - will be deleted. + Length of the nonce-salt value appended to the end of `crypt_file` - :Default: ``project.salt`` + :Default: 16 """ def __init__(self): @@ -170,8 +167,7 @@ class PappyConfig(object): self.crypt_dir = 'crypt' self.crypt_file = 'project.crypt' self.crypt_session = False - self.crypt_success = False - self.salt_file = 'project.salt' + self.salt_len = 16 def get_default_config(self): default_config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), @@ -185,8 +181,6 @@ class PappyConfig(object): pp = os.getcwd() + os.sep project_files = [pp+f for f in file_glob if os.path.isfile(pp+f)] - if self.salt_file in project_files: - project_files.remove(self.salt_file) if self.crypt_file in project_files: project_files.remove(self.crypt_file) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index a8c16cd..79aed48 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import crochet +import getpass import glob import os import pappyproxy @@ -55,6 +56,7 @@ class Crypto(object): # Delete clear-text files self.delete_clear_files() + return True def decrypt_project(self): """ @@ -74,7 +76,10 @@ class Crypto(object): # Otherwise, decrypt and decompress the project else: - archive_crypt = open(self.config.crypt_file, 'rb').read() + cf = self.config.crypt_file + sl = self.config.salt_len + crl = os.path.getsize(cf) - sl + archive_crypt = open(cf, 'rb').read(crl) archive_file = open(self.config.archive, 'wb') retries = 3 @@ -103,11 +108,7 @@ class Crypto(object): self.compressor.decompress_project() - # Force generation of new salt and crypt archive - self.delete_crypt_files() - os.chdir(self.config.crypt_dir) - self.config.crypt_success = True return True def confirm_password_retry(self): @@ -130,27 +131,40 @@ class Crypto(object): """ encoded_passwd = "" try: - passwd = raw_input("Enter a password: ").strip() + passwd = getpass.getpass("Enter a password: ").strip() self.password = passwd.encode("utf-8") except: raise PappyException("Invalid password, try again") def set_salt(self): - if os.path.isfile(self.config.salt_file): + if os.path.isfile(self.config.crypt_file): self.set_salt_from_file() else: self.salt = os.urandom(16) def set_salt_from_file(self): try: - salt_file = open(self.config.salt_file, 'rb') - self.salt = salt_file.readline().strip() + # Seek to `salt_len` bytes before the EOF + # then read `salt_len` bytes to retrieve the salt + + # WARNING: must open `crypt_file` in `rb` mode + # or `salt_file.seek()` will result in undefined + # behavior. + salt_file = open(self.config.crypt_file, 'rb') + sl = self.config.salt_len + salt_file.seek(sl, 2) + self.salt = salt_file.read(sl) except: - raise PappyException("Unable to read project.salt") + cf = self.config.crypt_file + raise PappyException("Unable to read %s" % cf) def create_salt_file(self): - salt_file = open(self.config.salt_file, 'wb') - + # WARNING: must open `crypt_file` in `wb` mode + # or `salt_file.seek()` will result in undefined + # behavior. + salt_file = open(self.config.crypt_file, 'wb') + # Seek to the end of the encrypted archive + salt_file.seek(0,2) salt_file.write(self.salt) salt_file.close() diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index 0f248c9..83882d6 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -26,6 +26,7 @@ from . import context from . import crypto from . import http from .console import ProxyCmd +from .util import PappyException from twisted.enterprise import adbapi from twisted.internet import reactor, defer from twisted.internet.error import CannotListenError @@ -71,9 +72,7 @@ class PappySession(object): from . import proxy, plugin if self.config.crypt_session: - self.decrypt() - - if self.config.crypt_success: + if self.decrypt(): self.config.load_from_file('./config.json') self.config.global_load_from_file() self.delete_data_on_quit = False @@ -150,17 +149,23 @@ class PappySession(object): self.complete_defer = deferToThread(self.cons.cmdloop) self.complete_defer.addCallback(self.cleanup) - @defer.inlineCallbacks def encrypt(self): - yield self.crypto.encrypt_project() + if self.crypto.encrypt_project(): + return True + else: + errmsg = "There was an issue encrypting the project." + raise PappyException(errmsg) + reactor.stop() + defer.returnValue(None) - @defer.inlineCallbacks def decrypt(self): # Attempt to decrypt project archive if self.crypto.decrypt_project(): - yield True + return True # Quit pappy on failure else: + errmsg = "There was an issue encrypting the project." + raise PappyException(errmsg) reactor.stop() defer.returnValue(None) From 976287a67b51dc3eb9947da549749699d56c7189 Mon Sep 17 00:00:00 2001 From: Nich Date: Tue, 12 Apr 2016 18:33:22 -0400 Subject: [PATCH 19/21] Fixed Crypto errors and uncaught exceptions The crypto salt file is no longer a thing. Now the salt is merely appended to the `crypt_file` as the last sixteen bytes. The salt is read from the end of the `crypt_file` on subsequent decryptions. Added a setting for enabling 'debug' mode, using `pdb` for dynamic debugging in pappy sessions. When needing to debug a function, or set of functionality add the conditional `if config.debug`, then set an entry point for `pdb` using `import pdb; pdb.set_trace()` May remove this functionality if not desired in the main project. --- pappyproxy/config.py | 1 + pappyproxy/crypto.py | 55 +++++++++++++++++++++++++------------------- pappyproxy/pappy.py | 21 ++++++++++------- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/pappyproxy/config.py b/pappyproxy/config.py index 04abe1d..32dfeae 100644 --- a/pappyproxy/config.py +++ b/pappyproxy/config.py @@ -164,6 +164,7 @@ class PappyConfig(object): self.global_config_dict = {} self.archive = 'project.archive' + self.debug = False self.crypt_dir = 'crypt' self.crypt_file = 'project.crypt' self.crypt_session = False diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 79aed48..17cece2 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -32,28 +32,33 @@ class Crypto(object): """ # Leave the crypto working directory - os.chdir('../') + if self.config.crypt_dir in os.getcwd(): + os.chdir('../') self.compressor.compress_project() + # Get the password and salt, then derive the key + self.crypto_ramp_up() + # Create project and crypto archive archive_file = open(self.archive, 'rb') archive_crypt = open(self.config.crypt_file, 'wb') - # Get the password and salt, then derive the key - self.crypto_ramp_up() + try: + # Encrypt the archive read as a bytestring + fern = Fernet(self.key) + crypt_token = fern.encrypt(archive_file.read()) + archive_crypt.write(crypt_token) + except InvalidToken as e: + raise PappyException("Error encrypting project: ", e) + return False - # Encrypt the archive read as a bytestring - fern = Fernet(self.key) - crypt_token = fern.encrypt(archive_file.read()) - archive_crypt.write(crypt_token) + archive_file.close() + archive_crypt.close() # Store the salt for the next decryption self.create_salt_file() - archive_file.close() - archive_crypt.close() - # Delete clear-text files self.delete_clear_files() return True @@ -79,6 +84,7 @@ class Crypto(object): cf = self.config.crypt_file sl = self.config.salt_len crl = os.path.getsize(cf) - sl + archive_crypt = open(cf, 'rb').read(crl) archive_file = open(self.config.archive, 'wb') @@ -89,18 +95,16 @@ class Crypto(object): fern = Fernet(self.key) archive = fern.decrypt(archive_crypt) break - except InvalidToken: - print "Invalid password" + except InvalidToken as e: + print "Invalid decryption: ", e retries -= 1 # Quit pappy if user doesn't retry # or if all retries exhuasted if not self.confirm_password_retry() or retries <= 0: - self.config.crypt_success = False return False else: self.password = None self.key = None - self.salt = None pass archive_file.write(archive) @@ -112,7 +116,7 @@ class Crypto(object): return True def confirm_password_retry(self): - answer = raw_input("Re-enter your password? (y/n)").strip() + answer = raw_input("Re-enter your password? (y/n): ").strip() if answer[0] == "y" or answer[0] == "Y": return True else: @@ -121,7 +125,8 @@ class Crypto(object): def crypto_ramp_up(self): if not self.password: self.get_password() - self.set_salt() + if not self.salt: + self.set_salt() self.derive_key() def get_password(self): @@ -137,7 +142,11 @@ class Crypto(object): raise PappyException("Invalid password, try again") def set_salt(self): - if os.path.isfile(self.config.crypt_file): + if self.config.crypt_dir in os.getcwd(): + os.chdir('../') + self.set_salt_from_file() + os.chdir(self.config.crypt_dir) + elif os.path.isfile(self.config.crypt_file): self.set_salt_from_file() else: self.salt = os.urandom(16) @@ -152,19 +161,17 @@ class Crypto(object): # behavior. salt_file = open(self.config.crypt_file, 'rb') sl = self.config.salt_len - salt_file.seek(sl, 2) + # Negate the salt length to seek to the + # correct position in the buffer + salt_file.seek(-sl, 2) self.salt = salt_file.read(sl) + salt_file.close() except: cf = self.config.crypt_file raise PappyException("Unable to read %s" % cf) def create_salt_file(self): - # WARNING: must open `crypt_file` in `wb` mode - # or `salt_file.seek()` will result in undefined - # behavior. - salt_file = open(self.config.crypt_file, 'wb') - # Seek to the end of the encrypted archive - salt_file.seek(0,2) + salt_file = open(self.config.crypt_file, 'a') salt_file.write(self.salt) salt_file.close() diff --git a/pappyproxy/pappy.py b/pappyproxy/pappy.py index 83882d6..fd20b21 100755 --- a/pappyproxy/pappy.py +++ b/pappyproxy/pappy.py @@ -76,6 +76,9 @@ class PappySession(object): self.config.load_from_file('./config.json') self.config.global_load_from_file() self.delete_data_on_quit = False + else: + self.complete_defer.callback(None) + return # If the data file doesn't exist, create it with restricted permissions if not os.path.isfile(self.config.datafile): @@ -153,10 +156,7 @@ class PappySession(object): if self.crypto.encrypt_project(): return True else: - errmsg = "There was an issue encrypting the project." - raise PappyException(errmsg) - reactor.stop() - defer.returnValue(None) + return False def decrypt(self): # Attempt to decrypt project archive @@ -164,10 +164,7 @@ class PappySession(object): return True # Quit pappy on failure else: - errmsg = "There was an issue encrypting the project." - raise PappyException(errmsg) - reactor.stop() - defer.returnValue(None) + return False @defer.inlineCallbacks def cleanup(self, ignored=None): @@ -187,6 +184,7 @@ def parse_args(): parser = argparse.ArgumentParser(description='An intercepting proxy for testing web applications.') parser.add_argument('-l', '--lite', help='Run the proxy in "lite" mode', action='store_true') + parser.add_argument('-d', '--debug', help='Run the proxy in "debug" mode', action='store_true') try: hlpmsg = ''.join(['Start pappy in "crypto" mode,', 'must supply a name for the encrypted', @@ -211,6 +209,10 @@ def parse_args(): else: settings['crypt'] = None + if args.debug: + settings['debug'] = True + else: + settings['debug'] = False return settings def set_text_factory(conn): @@ -257,6 +259,9 @@ def main(): pappy_config.global_load_from_file() session.delete_data_on_quit = False + if settings['debug']: + pappy_config.debug = True + yield session.start() session.complete_defer.addCallback(lambda ignored: reactor.stop()) From 04f3ac3199d8ccd3f563605d4a3784bf981a9e03 Mon Sep 17 00:00:00 2001 From: Nich Date: Tue, 12 Apr 2016 18:54:38 -0400 Subject: [PATCH 20/21] Added deletion of `crypt_file` to generate new salt every session After successfully decrypting the project, the `crypt_file` is deleted to generate a new salt for each session. --- pappyproxy/crypto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 17cece2..7540090 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -111,6 +111,7 @@ class Crypto(object): archive_file.close() self.compressor.decompress_project() + self.delete_crypt_files() os.chdir(self.config.crypt_dir) return True From c5fe21719af60a6544b40f3b6e7e5dd1e252287e Mon Sep 17 00:00:00 2001 From: Nich Date: Tue, 12 Apr 2016 19:05:53 -0400 Subject: [PATCH 21/21] Salt remains the same across sessions For Fernet to work correctly, the salt must be the same when re-encrypting the project after a successful decryption. This is because the key for decryption and encryption must be the same during a single session. So if the project was decrypted with one salt, the we want to generate a new salt to re-encrypt the file with a new key, Fernet will cough up an exception. Presumably this problem won't exist with other crypto-systems (e.g. AES-GCM). --- pappyproxy/crypto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pappyproxy/crypto.py b/pappyproxy/crypto.py index 7540090..725606f 100644 --- a/pappyproxy/crypto.py +++ b/pappyproxy/crypto.py @@ -101,6 +101,7 @@ class Crypto(object): # Quit pappy if user doesn't retry # or if all retries exhuasted if not self.confirm_password_retry() or retries <= 0: + os.remove(self.config.archive) return False else: self.password = None