import base64 import collections import console import context import crochet import datetime import gzip import json import proxy import re import StringIO import urlparse import zlib from twisted.internet import defer, reactor from util import PappyException ENCODE_NONE = 0 ENCODE_DEFLATE = 1 ENCODE_GZIP = 2 dbpool = None class DataAlreadyComplete(Exception): pass def init(pool): global dbpool if dbpool is None: dbpool = pool assert(dbpool) def destruct(): assert(dbpool) dbpool.close() def decode_encoded(data, encoding): if encoding == ENCODE_NONE: return data if encoding == ENCODE_DEFLATE: dec_data = StringIO.StringIO(zlib.decompress(data)) else: dec_data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(data)) return dec_data.read() def repeatable_parse_qs(s): pairs = s.split('&') ret_dict = RepeatableDict() for pair in pairs: if '=' in pair: t = tuple(pair.split('=', 1)) ret_dict.append(t[0], t[1]) else: ret_dict.append(pair, None) return ret_dict class RepeatableDict: """ A dict that retains the order of items inserted and keeps track of duplicate values. Can optionally treat keys as case insensitive. Custom made for the proxy, so it has strange features """ def __init__(self, from_pairs=None, case_insensitive=False): # If efficiency becomes a problem, add a dict that keeps a list by key # and use that for getting data. But until then, this stays. self._pairs = [] self._keys = set() self._modify_callback = None self.case_insensitive = case_insensitive if from_pairs: for k, v in from_pairs: self.append(k, v) def _ef_key(self, key): # "effective key", returns key.lower() if we're case insensitive, # otherwise it returns the same key if self.case_insensitive: return key.lower() return key def _mod_callback(self): # Calls the modify callback if we have one if self._modify_callback: self._modify_callback() def __contains__(self, val): return self._ef_key(val) in self._keys def __getitem__(self, key): for p in reversed(self._pairs): if self._ef_key(p[0]) == self._ef_key(key): return p[1] raise KeyError def __setitem__(self, key, val): # Replaces first instance of `key` and deletes the rest self.set_val(key, val) def __delitem__(self, key): self._keys.remove(key) self._pairs = [p for p in self._pairs if self._ef_key(p[0]) != self._ef_key(key)] self._mod_callback() def __nonzero__(self): if self._pairs: return True else: return False def _add_key(self, key): self._keys.add(self._ef_key(key)) def _remove_key(self, key): self._keys.remove(self._ef_key(key)) def all_pairs(self): return self._pairs[:] def append(self, key, val, do_callback=True): # Add a duplicate entry for key self._add_key(key) self._pairs.append((key, val)) if do_callback: self._mod_callback() def set_val(self, key, val, do_callback=True): new_pairs = [] added = False self._add_key(key) for p in self._pairs: if self._ef_key(p[0]) == self._ef_key(key): if not added: # only add the first instance new_pairs.append((key, val)) added = True else: new_pairs.append(p) if not added: new_pairs.append((key, val)) self._pairs = new_pairs if do_callback: self._mod_callback() def update(self, key, val, do_callback=True): # If key is already in the dict, replace that value with the new value if key in self: for k, v in self.all_pairs(): if self._ef_key(k) == self._ef_key(key): self.set_val(k, val, do_callback=do_callback) break else: self.set_val(key, val, do_callback=do_callback) def clear(self, do_callback=True): self._pairs = [] if do_callback: self._mod_callback() def all_vals(self, key): return [p[1] for p in self._pairs if self._ef_key(p[0]) == self._ef_key(key)] def add_pairs(self, pairs, do_callback=True): for pair in pairs: self._add_key(pair[0]) self._pairs += pairs if do_callback: self._mod_callback() def sort(self): # Sorts pairs by key alphabetaclly pairs = sorted(pairs, key=lambda x: x[0]) def set_modify_callback(self, callback): # Add a function to be called whenever an element is added, changed, or # deleted. Set to None to remove self._modify_callback = callback class LengthData: def __init__(self, length=None): self.raw_data = '' self.complete = False self.length = length or 0 if self.length == 0: self.complete = True def add_data(self, data): if self.complete: raise DataAlreadyComplete() remaining_length = self.length-len(self.raw_data) if len(data) >= remaining_length: self.raw_data += data[:remaining_length] assert(len(self.raw_data) == self.length) self.complete = True else: self.raw_data += data class ChunkedData: def __init__(self): self.raw_data = '' self._pos = 0 self._state = 0 # 0=reading length, 1=reading data, 2=going over known string self._len_str = '' self._chunk_remaining = 0 self._known_str = '' self._known_str_pos = 0 self._next_state = 0 self._raw_data = '' self.complete = False self.unchunked_data = '' def add_data(self, data): self._raw_data += data self.scan_forward() def scan_forward(self): # Don't add more data if we're already done if self.complete: return while self._pos < len(self._raw_data): curchar = self._raw_data[self._pos] if self._state == 0: if curchar.lower() in '0123456789abcdef': # Read the next char of the length self._len_str += curchar # Move to the next char self._pos += 1 elif curchar == '\r': # Save how much chunk to read self._chunk_remaining = int(self._len_str, 16) # If the length is 0, chunked encoding is done! if self._chunk_remaining == 0: self.complete = True # I should probably just rename raw_data since it's what # you use to look at unchunked data, but you're not # supposed to look at it until after it's complete # anyways self._raw_data = self.unchunked_data self.raw_data = self._raw_data # Expose raw_data return # There should be a newline after the \r self._known_str = '\n' self._state = 2 self._next_state = 1 # Reset the length str self._len_str = '' # Move to the next char self._pos += 1 else: raise Exception("Malformed chunked encoding!") elif self._state == 1: if self._chunk_remaining > 0: # Read next byte of data self.unchunked_data += curchar self._chunk_remaining -= 1 self._pos += 1 else: # Read newline then read a new chunk self._known_str = '\r\n' self._next_state = 0 # Read len after newlines self._state = 2 # Read newlines # Don't move to the next char because we didn't do anything elif self._state == 2: # Read a char of an expected string # If the expected char doesn't match, throw an error if self._known_str[self._known_str_pos] != curchar: raise Exception("Unexpected data") # Move to the next char in the raw data and in our known string self._known_str_pos += 1 self._pos += 1 # If we've reached the end of the known string, go to the next state if self._known_str_pos == len(self._known_str): self._known_str_pos = 0 self._state = self._next_state class ResponseCookie(object): def __init__(self, set_cookie_string=None): self.key = None self.val = None self.expires = None self.max_age = None self.domain = None self.path = None self.secure = False self.http_only = False if set_cookie_string: self.from_cookie(set_cookie_string) @property def cookie_av(self): av = '%s=%s' % (self.key, self.val) to_add = [av] if self.expires: to_add.append('expires=%s'%self.expires) if self.max_age: to_add.append('Max-Age=%d'%self.max_age) if self.domain: to_add.append('Domain=%s'%self.domain) if self.path: to_add.append('Path=%s'%self.path) if self.secure: to_add.append('secure') if self.http_only: to_add.append('httponly') return '; '.join(to_add) def parse_cookie_av(self, cookie_av): if '=' in cookie_av: key, val = cookie_av.split('=', 1) key = key.lstrip() if key.lower() == 'expires': self.expires = val if key.lower() == 'max-age': self.max_age = int(val) if key.lower() == 'domain': self.domain = val if key.lower() == 'path': self.path = val elif cookie_av.lstrip().lower() == 'secure': self.secure = True elif cookie_av.lstrip().lower() == 'httponly': self.http_only = True def from_cookie(self, set_cookie_string): if ';' in set_cookie_string: cookie_pair, rest = set_cookie_string.split(';', 1) self.key, self.val = cookie_pair.split('=',1) cookie_avs = rest.split(';') for cookie_av in cookie_avs: cookie_av.lstrip() self.parse_cookie_av(cookie_av) else: self.key, self.val = set_cookie_string.split('=',1) class Request(object): def __init__(self, full_request=None, update_content_length=False): self.time_end = None self.time_start = None self.complete = False self.cookies = RepeatableDict() self.fragment = None self.get_params = RepeatableDict() self.header_len = 0 self.headers = RepeatableDict(case_insensitive=True) self.headers_complete = False self.host = None self.is_ssl = False self.path = '' self.port = None self.post_params = RepeatableDict() self._raw_data = '' self.reqid = None self.response = None self.submitted = False self.unmangled = None self.verb = '' self.version = '' self._first_line = True #self._connect_response = False #self._encoding_type = ENCODE_NONE self._data_length = 0 self._partial_data = '' self.set_dict_callbacks() # Get values from the raw request if full_request is not None: self.from_full_request(full_request, update_content_length) @property def rsptime(self): if self.time_start and self.time_end: return self.time_end-self.time_start else: return None @property def status_line(self): path = self.path if self.get_params: path += '?' pairs = [] for pair in self.get_params.all_pairs(): if pair[1] is None: pairs.append(pair[0]) else: pairs.append('='.join(pair)) path += '&'.join(pairs) if self.fragment: path += '#' path += self.fragment return '%s %s %s' % (self.verb, path, self.version) @status_line.setter def status_line(self, val): self.handle_statusline(val) @property def raw_headers(self): ret = self.status_line + '\r\n' for k, v in self.headers.all_pairs(): ret = ret + "%s: %s\r\n" % (k, v) ret = ret + '\r\n' return ret @property def full_request(self): ret = self.raw_headers ret = ret + self.raw_data return ret @property def raw_data(self): return self._raw_data @raw_data.setter def raw_data(self, val): self._raw_data = val self.update_from_data() self.complete = True def set_dict_callbacks(self): # Add callbacks to dicts self.headers.set_modify_callback(self.update_from_text) self.cookies.set_modify_callback(self.update_from_objects) self.post_params.set_modify_callback(self.update_from_data) def from_full_request(self, full_request, update_content_length=False): # Get rid of leading CRLF. Not in spec, should remove eventually # technically doesn't treat \r\n same as \n, but whatever. while full_request[0:2] == '\r\n': full_request = full_request[2:] # We do redundant splits, but whatever lines = full_request.splitlines() for line in lines: if self.headers_complete: break self.add_line(line) if not self.headers_complete: self.add_line('') if not self.complete: data = full_request[self.header_len:] if update_content_length: self.raw_data = data else: self.add_data(data) assert(self.complete) def update_from_data(self): # Updates metadata that's based off of data self.headers.update('Content-Length', str(len(self.raw_data)), do_callback=False) if 'content-type' in self.headers: if self.headers['content-type'] == 'application/x-www-form-urlencoded': self.post_params = repeatable_parse_qs(self.raw_data) self.set_dict_callbacks() def update_from_objects(self): # Updates text values that depend on objects. # DOES NOT MAINTAIN HEADER DUPLICATION, ORDER, OR CAPITALIZATION if self.cookies: assignments = [] for ck, cv in self.cookies.all_pairs(): asn = '%s=%s' % (ck, cv) assignments.append(asn) header_val = '; '.join(assignments) self.headers.update('Cookie', header_val, do_callback=False) if self.post_params: pairs = [] for k, v in self.post_params: pairs.append('%s=%s' % (k, v)) self.raw_data = '&'.join(pairs) def update_from_text(self): # Updates metadata that depends on header/status line values self.cookies = RepeatableDict() self.set_dict_callbacks() for k, v in self.headers.all_pairs(): self.handle_header(k, v) def add_data(self, data): # Add data (headers must be complete) len_remaining = self._data_length - len(self._partial_data) if len(data) >= len_remaining: self._partial_data += data[:len_remaining] self._raw_data = self._partial_data self.complete = True self.handle_data_end() else: self._partial_data += data def _process_host(self, hostline, overwrite=False): # Only overwrite if told to since we may set it during the CONNECT request and we don't want to # overwrite that # Get address and port if ':' in hostline: self.host, self.port = hostline.split(':') self.port = int(self.port) if self.port == 443: self.is_ssl = True else: self.host = hostline if not self.port or overwrite: # could be changed by connect request self.port = 80 self.host.strip() def add_line(self, line): # Add a line (for status line and headers) # Modifies first line if it is in full url form if self._first_line and line == '': # Ignore leading newlines because fuck the spec return if self._first_line: self.handle_statusline(line) self._first_line = False else: # Either header or newline (end of headers) if line == '': self.headers_complete = True if self._data_length == 0: self.complete = True else: key, val = line.split(':', 1) val = val.strip() if self.handle_header(key, val): self.headers.append(key, val, do_callback=False) self.header_len += len(line)+2 def handle_statusline(self, status_line): parts = status_line.split() uri = None if len(parts) == 3: self.verb, uri, self.version = parts elif len(parts) == 2: self.verb, self.version = parts else: raise Exception("Unexpected format of first line of request") # Get path using urlparse if uri is not None: if not re.match('(?:^.+)://', uri): uri = '//' + uri parsed_path = urlparse.urlparse(uri) netloc = parsed_path.netloc self._process_host(netloc) # Check for https if re.match('^https://', uri) or self.port == 443: self.is_ssl = True self.port = 443 reqpath = parsed_path.path self.path = parsed_path.path if parsed_path.query: reqpath += '?' reqpath += parsed_path.query self.get_params = repeatable_parse_qs(parsed_path.query) if parsed_path.fragment: reqpath += '#' reqpath += parsed_path.fragment self.fragment = parsed_path.fragment def handle_header(self, key, val): # We may have duplicate headers stripped = False if key.lower() == 'content-length': self._data_length = int(val) elif key.lower() == 'cookie': # We still want the raw key/val for the cookies header # because it's still a header cookie_strs = val.split('; ') # The only whitespace that matters is the space right after the # semicolon. If actual implementations mess this up, we could # probably strip whitespace around the key/value for cookie_str in cookie_strs: splitted = cookie_str.split('=',1) assert(len(splitted) == 2) (cookie_key, cookie_val) = splitted # we want to parse duplicate cookies self.cookies.append(cookie_key, cookie_val, do_callback=False) elif key.lower() == 'host': self._process_host(val) elif key.lower() == 'connection': #stripped = True pass return (not stripped) def handle_data_end(self): if 'content-type' in self.headers: if self.headers['content-type'] == 'application/x-www-form-urlencoded': self.post_params = repeatable_parse_qs(self.raw_data) self.set_dict_callbacks() @defer.inlineCallbacks def save(self): assert(dbpool) if self.reqid: # If we have reqid, we're updating yield dbpool.runInteraction(self._update) assert(self.reqid is not None) else: yield dbpool.runInteraction(self._insert) assert(self.reqid is not None) @defer.inlineCallbacks def deep_save(self): "Saves self, unmangled, response, and unmangled response" if self.response: if self.response.unmangled: yield self.response.unmangled.save() yield self.response.save() if self.unmangled: yield self.unmangled.save() yield self.save() def _update(self, txn): # If we don't have an reqid, we're creating a new reuqest row setnames = ["full_request=?"] queryargs = [self.full_request] if self.response: setnames.append('response_id=?') assert(self.response.rspid is not None) # should be saved first queryargs.append(self.response.rspid) if self.unmangled: setnames.append('unmangled_id=?') assert(self.unmangled.reqid is not None) # should be saved first queryargs.append(self.unmangled.reqid) if self.time_start: setnames.append('start_datetime=?') queryargs.append(self.time_start.isoformat()) if self.time_end: setnames.append('end_datetime=?') queryargs.append(self.time_end.isoformat()) setnames.append('submitted=?') if self.submitted: queryargs.append('1') else: queryargs.append('0') queryargs.append(self.reqid) txn.execute( """ UPDATE requests SET %s WHERE id=?; """ % ','.join(setnames), tuple(queryargs) ) def _insert(self, txn): # If we don't have an reqid, we're creating a new reuqest row colnames = ["full_request"] colvals = [self.full_request] if self.response: colnames.append('response_id') assert(self.response.rspid is not None) # should be saved first colvals.append(self.response.rspid) if self.unmangled: colnames.append('unmangled_id') assert(self.unmangled.reqid is not None) # should be saved first colvals.append(self.unmangled.reqid) if self.time_start: colnames.append('start_datetime') colvals.append(self.time_start.isoformat()) if self.time_end: colnames.append('end_datetime') colvals.append(self.time_end.isoformat()) colnames.append('submitted') if self.submitted: colvals.append('1') else: colvals.append('0') txn.execute( """ INSERT INTO requests (%s) VALUES (%s); """ % (','.join(colnames), ','.join(['?']*len(colvals))), tuple(colvals) ) self.reqid = txn.lastrowid assert txn.lastrowid is not None assert self.reqid is not None def to_json(self): # We base64 encode the full response because json doesn't paly nice with # binary blobs data = { 'full_request': base64.b64encode(self.full_request), 'reqid': self.reqid, } if self.response: data['response_id'] = self.response.rspid else: data['response_id'] = None if self.unmangled: data['unmangled_id'] = self.unmangled.reqid if self.time_start: data['start'] = self.time_start.isoformat() if self.time_end: data['end'] = self.time_end.isoformat() return json.dumps(data) def from_json(self, json_string): data = json.loads(json_string) self.from_full_request(base64.b64decode(data['full_request'])) self.update_from_text() self.update_from_data() if data['reqid']: self.reqid = int(data['reqid']) def delete(self): assert(self.reqid is not None) row = yield dbpool.runQuery( """ DELETE FROM requests WHERE id=?; """, (self.reqid,) ) def duplicate(self): return Request(self.full_request) @staticmethod @defer.inlineCallbacks def submit(host, port, is_ssl, full_request): new_obj = Request(full_request) factory = proxy.ProxyClientFactory(new_obj) factory.connection_id = proxy.get_next_connection_id() if is_ssl: reactor.connectSSL(host, port, factory, proxy.ClientTLSContext()) else: reactor.connectTCP(host, port, factory) new_req = yield factory.data_defer defer.returnValue(new_req) def submit_self(self): new_req = Request.submit(self.host, self.port, self.is_ssl, self.full_request) return new_req @staticmethod @defer.inlineCallbacks def load_request(reqid): assert(dbpool) rows = yield dbpool.runQuery( """ SELECT full_request, response_id, id, unmangled_id, start_datetime, end_datetime FROM requests WHERE id=?; """, (reqid,) ) if len(rows) != 1: raise PappyException("Request with id %d does not exist" % reqid) full_request = rows[0][0] req = Request(full_request) if rows[0][1]: rsp = yield Response.load_response(int(rows[0][1])) req.response = rsp if rows[0][3]: unmangled_req = yield Request.load_request(int(rows[0][3])) req.unmangled = unmangled_req if rows[0][4]: req.time_start = datetime.datetime.strptime(rows[0][4], "%Y-%m-%dT%H:%M:%S.%f") if rows[0][5]: req.time_end = datetime.datetime.strptime(rows[0][5], "%Y-%m-%dT%H:%M:%S.%f") req.reqid = int(rows[0][2]) defer.returnValue(req) @staticmethod @defer.inlineCallbacks def load_from_filters(filters): # Not efficient in any way # But it stays this way until we hit performance issues assert(dbpool) rows = yield dbpool.runQuery( """ SELECT r1.id FROM requests r1 LEFT JOIN requests r2 ON r1.id=r2.unmangled_id WHERE r2.id is NULL; """, ) reqs = [] for r in rows: newreq = yield Request.load_request(int(r[0])) reqs.append(newreq) reqs = context.filter_reqs(reqs, filters) defer.returnValue(reqs) class Response(object): def __init__(self, full_response=None, update_content_length=False): self.complete = False self.cookies = RepeatableDict() self.header_len = 0 self.headers = RepeatableDict(case_insensitive=True) self.headers_complete = False self.host = None self._raw_data = '' self.response_code = 0 self.response_text = '' self.rspid = None self._status_line = '' self.unmangled = None self.version = '' self._encoding_type = ENCODE_NONE self._first_line = True self._data_obj = None self._end_after_headers = False self.set_dict_callbacks() if full_response is not None: self.from_full_response(full_response, update_content_length) @property def raw_headers(self): ret = self.status_line + '\r\n' for k, v in self.headers.all_pairs(): ret = ret + "%s: %s\r\n" % (k, v) ret = ret + '\r\n' return ret @property def status_line(self): return self._status_line @status_line.setter def status_line(self, val): self._status_line = val self.handle_statusline(val) @property def raw_data(self): return self._raw_data @raw_data.setter def raw_data(self, val): self._raw_data = val self._data_obj = LengthData(len(val)) self._data_obj.add_data(val) self._encoding_type = ENCODE_NONE self.complete = True self.update_from_data() @property def full_response(self): ret = self.raw_headers ret = ret + self.raw_data return ret def set_dict_callbacks(self): # Add callbacks to dicts self.headers.set_modify_callback(self.update_from_text) self.cookies.set_modify_callback(self.update_from_objects) def from_full_response(self, full_response, update_content_length=False): # Get rid of leading CRLF. Not in spec, should remove eventually while full_response[0:2] == '\r\n': full_response = full_response[2:] # We do redundant splits, but whatever lines = full_response.splitlines() for line in lines: if self.headers_complete: break self.add_line(line) if not self.headers_complete: self.add_line('') if not self.complete: data = full_response[self.header_len:] if update_content_length: self.raw_data = data else: self.add_data(data) assert(self.complete) def add_line(self, line): assert(not self.headers_complete) self.header_len += len(line)+2 if not line and self._first_line: return if not line: self.headers_complete = True if self._end_after_headers: self.complete = True return if not self._data_obj: self._data_obj = LengthData(0) self.complete = self._data_obj.complete return if self._first_line: self.handle_statusline(line) self._first_line = False else: key, val = line.split(':', 1) val = val.strip() self.handle_header(key, val) def handle_statusline(self, status_line): self._first_line = False self._status_line = status_line self.version, self.response_code, self.response_text = \ status_line.split(' ', 2) self.response_code = int(self.response_code) if self.response_code == 304 or self.response_code == 204 or \ self.response_code/100 == 1: self._end_after_headers = True def handle_header(self, key, val): stripped = False if key.lower() == 'content-encoding': if val in ('gzip', 'x-gzip'): self._encoding_type = ENCODE_GZIP elif val in ('deflate'): self._encoding_type = ENCODE_DEFLATE # We send our requests already decoded, so we don't want a header # saying it's encoded if self._encoding_type != ENCODE_NONE: stripped = True elif key.lower() == 'transfer-encoding' and val.lower() == 'chunked': self._data_obj = ChunkedData() self.complete = self._data_obj.complete stripped = True elif key.lower() == 'content-length': # We use our own content length self._data_obj = LengthData(int(val)) elif key.lower() == 'set-cookie': cookie = ResponseCookie(val) self.cookies.append(cookie.key, cookie, do_callback=False) elif key.lower() == 'host': self.host = val if stripped: return False else: self.headers.append(key, val, do_callback=False) return True def update_from_data(self): self.headers.update('Content-Length', str(len(self.raw_data)), do_callback=False) def update_from_objects(self): # Updates headers from objects # DOES NOT MAINTAIN HEADER DUPLICATION, ORDER, OR CAPITALIZATION # Cookies new_headers = RepeatableDict() cookies_added = False for pair in self.headers.all_pairs(): if pair[0].lower() == 'set-cookie': # If we haven't added our cookies, add them all. Otherwise # strip the header (do nothing) if not cookies_added: # Add all our cookies here for k, c in self.cookies.all_pairs(): new_headers.append('Set-Cookie', c.cookie_av) cookies_added = True else: new_headers.append(pair[0], pair[1]) if not cookies_added: # Add all our cookies to the end for k, c in self.cookies.all_pairs(): new_headers.append('Set-Cookie', c.cookie_av) self.headers = new_headers self.set_dict_callbacks() def update_from_text(self): self.cookies = RepeatableDict() self.set_dict_callbacks() for k, v in self.headers.all_pairs(): if k.lower() == 'set-cookie': # Parse the cookie cookie = ResponseCookie(v) self.cookies.append(cookie.key, cookie, do_callback=False) def add_data(self, data): assert(self._data_obj) assert(not self._data_obj.complete) assert not self.complete self._data_obj.add_data(data) if self._data_obj.complete: self._raw_data = decode_encoded(self._data_obj.raw_data, self._encoding_type) self.complete = True self.update_from_data() def add_cookie(self, cookie): self.cookies.append(cookie.key, cookie, do_callback=False) def to_json(self): # We base64 encode the full response because json doesn't paly nice with # binary blobs data = { 'rspid': self.rspid, 'full_response': base64.b64encode(self.full_response), } if self.unmangled: data['unmangled_id'] = self.unmangled.rspid return json.dumps(data) def from_json(self, json_string): data = json.loads(json_string) self.from_full_response(base64.b64decode(data['full_response'])) self.update_from_text() self.update_from_data() if data['rspid']: self.rspid = int(data['rspid']) @defer.inlineCallbacks def save(self): assert(dbpool) if self.rspid: # If we have rspid, we're updating yield dbpool.runInteraction(self._update) else: yield dbpool.runInteraction(self._insert) assert(self.rspid is not None) def _update(self, txn): setnames = ["full_response=?"] queryargs = [self.full_response] if self.unmangled: setnames.append('unmangled_id=?') assert(self.unmangled.rspid is not None) # should be saved first queryargs.append(self.unmangled.rspid) queryargs.append(self.rspid) txn.execute( """ UPDATE responses SET %s WHERE id=?; """ % ','.join(setnames), tuple(queryargs) ) assert(self.rspid is not None) def _insert(self, txn): # If we don't have an rspid, we're creating a new one colnames = ["full_response"] colvals = [self.full_response] if self.unmangled is not None: colnames.append('unmangled_id') assert(self.unmangled.rspid is not None) # should be saved first colvals.append(self.unmangled.rspid) txn.execute( """ INSERT INTO responses (%s) VALUES (%s); """ % (','.join(colnames), ','.join(['?']*len(colvals))), tuple(colvals) ) self.rspid = txn.lastrowid assert(self.rspid is not None) def delete(self): assert(self.rspid is not None) row = yield dbpool.runQuery( """ DELETE FROM responses WHERE id=?; """, (self.rspid,) ) @staticmethod @defer.inlineCallbacks def load_response(respid): assert(dbpool) rows = yield dbpool.runQuery( """ SELECT full_response, id, unmangled_id FROM responses WHERE id=?; """, (respid,) ) if len(rows) != 1: raise PappyException("Response with request id %d does not exist" % respid) full_response = rows[0][0] resp = Response(full_response) resp.rspid = int(rows[0][1]) if rows[0][2]: unmangled_response = yield Response.load_response(int(rows[0][2])) resp.unmangled = unmangled_response defer.returnValue(resp)