diff --git a/README.md b/README.md index 9c2bdce..29da205 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,131 @@ -The Puppy Proxy (New Pappy) -=========================== - -For documentation on what the commands are, see the [Pappy README](https://github.com/roglew/pappy-proxy) +The Puppy Proxy +=============== What is this? ------------- -This is a beta version of what I plan on releasing as the next version of Pappy. Technically it should work, but there are a few missing features that I want to finish before replacing Pappy. A huge part of the code has been rewritten in Go and most commands have been reimplemented. +Puppy is a golang library that can be used to create proxies to intercept and modify HTTP and websocket messages that pass through it. Puppy itself does not provide any interactive interface, it provides an API to do proxy things in go. If you want a useful tool that uses Puppy, try [Pappy](https://github.com/roglew/pappy-proxy). + +Puppy was originally aimed to be a starting point to write a tool similar to [Burp Suite](https://portswigger.net/burp/) and to provide a base for writing other HTTP proxy software. + +Features +-------- + +* Intercept and modify any HTTP messages passing through the proxy +* Websocket support +* Use custom CA certificate to strip TLS from HTTPS connections +* Built in IPC API +* Support for transparent request redirection +* Built in support for writing messages to SQLite database +* Flexible history search + +Example +------- + +The following example creates a simple proxy which listens on port 8080. In order to send HTTPS traffic through the proxy, you must add the generated server.pem certificate as a CA to your browser. + +```go +package main + +import ( + "fmt" + "net" + "os" + "path" + "puppy" +) + +func checkerr(err error) { + if err != nil { + panic(err) + } +} + +func main() { + // Create the proxy without a logger + iproxy := puppy.NewInterceptingProxy(nil) + + // Load the CA certs + ex, err := os.Executable() + checkerr(err) + certFile := path.Dir(ex) + "/server.pem" + pkeyFile := path.Dir(ex) + "/server.key" + err = iproxy.LoadCACertificates(certFile, pkeyFile) + if err != nil { + // Try generating the certs in case they're missing + _, err := puppy.GenerateCACertsToDisk(certFile, pkeyFile) + checkerr(err) + err = iproxy.LoadCACertificates(certFile, pkeyFile) + checkerr(err) + } + + // Listen on port 8080 + listener, err := net.Listen("tcp", "127.0.0.1:8080") + checkerr(err) + iproxy.AddListener(listener) -**Back up your data.db files before using this**. The database schema may change and I may or may not correctly upgrade it from the published schema version here. It also breaks backwards compatibility with the last version of Pappy. + // Wait for exit + fmt.Println("Proxy is running on localhost:8080") + select {} +} +``` -Installation ------------- +Next, we will demonstrate editing messages by turning the proxy into a cloud2butt proxy which will replace every instance of the word "cloud" with the word "butt". This is done by writing a function that takes in a request and a response and returns a new response then adding it to the proxy: -1. [Set up go](https://golang.org/doc/install) -1. [Set up pip](https://pip.pypa.io/en/stable/) +```go +package main -Then run: +import ( + "bytes" + "fmt" + "net" + "os" + "path" + "puppy" +) -~~~ -# Get puppy and all its dependencies -go get https://github.com/roglew/puppy -cd ~/$GOPATH/puppy -go get ./... +func checkerr(err error) { + if err != nil { + panic(err) + } +} -# Build the go binary -cd ~/$GOPATH/bin -go build puppy -cd ~/$GOPATH/src/puppy/python/puppy +func main() { + // Create the proxy without a logger + iproxy := puppy.NewInterceptingProxy(nil) -# Optionally set up the virtualenv here + // Load the CA certs + ex, err := os.Executable() + checkerr(err) + certFile := path.Dir(ex) + "/server.pem" + pkeyFile := path.Dir(ex) + "/server.key" + err = iproxy.LoadCACertificates(certFile, pkeyFile) + if err != nil { + // Try generating the certs in case they're missing + _, err := puppy.GenerateCACertsToDisk(certFile, pkeyFile) + checkerr(err) + err = iproxy.LoadCACertificates(certFile, pkeyFile) + checkerr(err) + } -# Set up the python interface -pip install -e . -~~~ + // Cloud2Butt interceptor + var cloud2butt = func(req *puppy.ProxyRequest, rsp *puppy.ProxyResponse) (*puppy.ProxyResponse, error) { + newBody := rsp.BodyBytes() + newBody = bytes.Replace(newBody, []byte("cloud"), []byte("butt"), -1) + newBody = bytes.Replace(newBody, []byte("Cloud"), []byte("Butt"), -1) + rsp.SetBodyBytes(newBody) + return rsp, nil + } + iproxy.AddRspInterceptor(cloud2butt) -Then you can run puppy by running `puppy`. It will use the puppy binary in `~/$GOPATH/bin` so leave the binary there. + // Listen on port 8080 + listener, err := net.Listen("tcp", "127.0.0.1:8080") + checkerr(err) + iproxy.AddListener(listener) -Missing Features From Pappy ---------------------------- -All that's left is updating documentation! + // Wait for exit + fmt.Println("Proxy is running on localhost:8080") + select {} +} +``` -Need more info? ---------------- -Right now I haven't written any documentation, so feel free to contact me for help. +For more information, check out the documentation. \ No newline at end of file diff --git a/certs.go b/certs.go index 2e69fb0..e146e8b 100644 --- a/certs.go +++ b/certs.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "crypto/rand" @@ -9,9 +9,11 @@ import ( "encoding/pem" "fmt" "math/big" + "os" "time" ) +// A certificate/private key pair type CAKeyPair struct { Certificate []byte PrivateKey *rsa.PrivateKey @@ -23,6 +25,7 @@ func bigIntHash(n *big.Int) []byte { return h.Sum(nil) } +// GenerateCACerts generates a random CAKeyPair func GenerateCACerts() (*CAKeyPair, error) { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { @@ -65,6 +68,37 @@ func GenerateCACerts() (*CAKeyPair, error) { }, nil } +// Generate a pair of certificates and write them to the disk. Returns the generated keypair +func GenerateCACertsToDisk(CertificateFile string, PrivateKeyFile string) (*CAKeyPair, error) { + pair, err := GenerateCACerts() + if err != nil { + return nil, err + } + + pkeyFile, err := os.OpenFile(PrivateKeyFile, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return nil, err + } + pkeyFile.Write(pair.PrivateKeyPEM()) + if err := pkeyFile.Close(); err != nil { + return nil, err + } + + certFile, err := os.OpenFile(CertificateFile, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return nil, err + } + + certFile.Write(pair.CACertPEM()) + if err := certFile.Close(); err != nil { + return nil, err + } + + return pair, nil +} + + +// PrivateKeyPEM returns the private key of the CAKeyPair PEM encoded func (pair *CAKeyPair) PrivateKeyPEM() []byte { return pem.EncodeToMemory( &pem.Block{ @@ -74,6 +108,7 @@ func (pair *CAKeyPair) PrivateKeyPEM() []byte { ) } +// PrivateKeyPEM returns the CA cert of the CAKeyPair PEM encoded func (pair *CAKeyPair) CACertPEM() []byte { return pem.EncodeToMemory( &pem.Block{ diff --git a/cmd/main/main.go b/cmd/main/main.go new file mode 100644 index 0000000..88fd82d --- /dev/null +++ b/cmd/main/main.go @@ -0,0 +1,163 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io/ioutil" + "log" + "net" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "puppy" +) + +var logBanner string = ` +======================================== +PUPPYSTARTEDPUPPYSTARTEDPUPPYSTARTEDPUPP + .--. .---. + /:. '. .' .. '._.---. + /:::-. \.-"""-;' .-:::. .::\ + /::'| '\/ _ _ \' '\:' ::::| + __.' | / (o|o) \ ''. ':/ + / .:. / | ___ | '---' +| ::::' /: (._.) .:\ +\ .=' |:' :::| + '""' \ .-. ':/ + '---'|I|'---' +jgs '-' +PUPPYSTARTEDPUPPYSTARTEDPUPPYSTARTEDPUPP +======================================== +` + +type listenArg struct { + Type string + Addr string +} + +func quitErr(msg string) { + os.Stderr.WriteString(msg) + os.Stderr.WriteString("\n") + os.Exit(1) +} + +func checkErr(err error) { + if err != nil { + quitErr(err.Error()) + } +} + +func parseListenString(lstr string) (*listenArg, error) { + args := strings.SplitN(lstr, ":", 2) + if len(args) != 2 { + return nil, errors.New("invalid listener. Must be in the form of \"tye:addr\"") + } + argStruct := &listenArg{ + Type: strings.ToLower(args[0]), + Addr: args[1], + } + if argStruct.Type != "tcp" && argStruct.Type != "unix" { + return nil, fmt.Errorf("invalid listener type: %s", argStruct.Type) + } + return argStruct, nil +} + +func unixAddr() string { + return fmt.Sprintf("%s/proxy.%d.%d.sock", os.TempDir(), os.Getpid(), time.Now().UnixNano()) +} + +var mln net.Listener +var logger *log.Logger + +func cleanup() { + if mln != nil { + mln.Close() + } +} + +var MainLogger *log.Logger + +func main() { + defer cleanup() + // Handle signals + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, os.Interrupt, os.Kill, syscall.SIGTERM) + go func() { + <-sigc + if logger != nil { + logger.Println("Caught signal. Cleaning up.") + } + cleanup() + os.Exit(0) + }() + + msgListenStr := flag.String("msglisten", "", "Listener for the message handler. Examples: \"tcp::8080\", \"tcp:127.0.0.1:8080\", \"unix:/tmp/foobar\"") + autoListen := flag.Bool("msgauto", false, "Automatically pick and open a unix or tcp socket for the message listener") + debugFlag := flag.Bool("dbg", false, "Enable debug logging") + flag.Parse() + + if *debugFlag { + logfile, err := os.OpenFile("log.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + checkErr(err) + logger = log.New(logfile, "[*] ", log.Lshortfile) + } else { + logger = log.New(ioutil.Discard, "[*] ", log.Lshortfile) + log.SetFlags(0) + } + MainLogger = logger + + // Parse arguments to structs + if *msgListenStr == "" && *autoListen == false { + quitErr("message listener address or `--msgauto` required") + } + if *msgListenStr != "" && *autoListen == true { + quitErr("only one of listener address or `--msgauto` can be used") + } + + // Create the message listener + var listenStr string + if *msgListenStr != "" { + msgAddr, err := parseListenString(*msgListenStr) + checkErr(err) + if msgAddr.Type == "tcp" { + var err error + mln, err = net.Listen("tcp", msgAddr.Addr) + checkErr(err) + } else if msgAddr.Type == "unix" { + var err error + mln, err = net.Listen("unix", msgAddr.Addr) + checkErr(err) + } else { + quitErr("unsupported listener type:" + msgAddr.Type) + } + listenStr = fmt.Sprintf("%s:%s", msgAddr.Type, msgAddr.Addr) + } else { + fpath := unixAddr() + ulisten, err := net.Listen("unix", fpath) + if err == nil { + mln = ulisten + listenStr = fmt.Sprintf("unix:%s", fpath) + } else { + tcplisten, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + quitErr("unable to open any messaging ports") + } + mln = tcplisten + listenStr = fmt.Sprintf("tcp:%s", tcplisten.Addr().String()) + } + } + + // Set up the intercepting proxy + iproxy := puppy.NewInterceptingProxy(logger) + iproxy.AddHTTPHandler("puppy", puppy.CreateWebUIHandler()) + + // Create a message server and have it serve for the iproxy + mserv := puppy.NewProxyMessageListener(logger, iproxy) + logger.Print(logBanner) + fmt.Println(listenStr) + mserv.Serve(mln) // serve until killed +} diff --git a/credits.go b/credits.go index ddb540b..5f11823 100644 --- a/credits.go +++ b/credits.go @@ -1,4 +1,4 @@ -package main +package puppy /* List of info that is used to display credits @@ -13,7 +13,7 @@ type creditItem struct { longCopyright string } -var LIB_CREDITS = []creditItem{ +var lib_credits = []creditItem{ creditItem{ "goproxy", "https://github.com/elazarl/goproxy", diff --git a/messageserv.go b/messageserv.go index 5593b47..30df324 100644 --- a/messageserv.go +++ b/messageserv.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "bufio" @@ -14,8 +14,10 @@ import ( Message Server */ -type MessageHandler func([]byte, net.Conn, *log.Logger, *InterceptingProxy) +// A handler to handle a JSON message +type MessageHandler func(message []byte, conn net.Conn, logger *log.Logger, iproxy *InterceptingProxy) +// A listener that handles reading JSON messages and sending them to the correct handler type MessageListener struct { handlers map[string]MessageHandler iproxy *InterceptingProxy @@ -31,6 +33,7 @@ type errorMessage struct { Reason string } +// NewMessageListener creates a new message listener associated with the given intercepting proxy func NewMessageListener(l *log.Logger, iproxy *InterceptingProxy) *MessageListener { m := &MessageListener{ handlers: make(map[string]MessageHandler), @@ -40,6 +43,7 @@ func NewMessageListener(l *log.Logger, iproxy *InterceptingProxy) *MessageListen return m } +// AddHandler will have the listener call the given handler when the "Command" parameter matches the given value func (l *MessageListener) AddHandler(command string, handler MessageHandler) { l.handlers[strings.ToLower(command)] = handler } @@ -60,6 +64,7 @@ func (l *MessageListener) Handle(message []byte, conn net.Conn) error { return nil } +// Serve will have the listener serve messages on the given listener func (l *MessageListener) Serve(nl net.Listener) { for { conn, err := nl.Accept() @@ -88,6 +93,7 @@ func (l *MessageListener) Serve(nl net.Listener) { } } +// Error response writes an error message to the given writer func ErrorResponse(w io.Writer, reason string) { var m errorMessage m.Success = false @@ -95,16 +101,17 @@ func ErrorResponse(w io.Writer, reason string) { MessageResponse(w, m) } +// MessageResponse writes a response to a given writer func MessageResponse(w io.Writer, m interface{}) { b, err := json.Marshal(&m) if err != nil { panic(err) } - MainLogger.Printf("< %s\n", string(b)) w.Write(b) w.Write([]byte("\n")) } +// ReadMessage reads a message from the given reader func ReadMessage(r *bufio.Reader) ([]byte, error) { m, err := r.ReadBytes('\n') if err != nil { diff --git a/proxy.go b/proxy.go index c8d33f3..8203ed7 100644 --- a/proxy.go +++ b/proxy.go @@ -1,9 +1,11 @@ -package main +// Puppy provices an interface to create a proxy to intercept and modify HTTP and websocket messages passing through the proxy +package puppy import ( "crypto/tls" "encoding/base64" "fmt" + "io/ioutil" "log" "net" "net/http" @@ -16,14 +18,15 @@ import ( var getNextSubId = IdCounter() var getNextStorageId = IdCounter() -// Working on using this for webui -type proxyWebUIHandler func(http.ResponseWriter, *http.Request, *InterceptingProxy) +// ProxyWebUIHandler is a function that can be used for handling web requests intended to be handled by the proxy +type ProxyWebUIHandler func(http.ResponseWriter, *http.Request, *InterceptingProxy) type savedStorage struct { storage MessageStorage description string } +// InterceptingProxy is a struct which represents a proxy which can intercept and modify HTTP and websocket messages type InterceptingProxy struct { slistener *ProxyListener server *http.Server @@ -48,48 +51,66 @@ type InterceptingProxy struct { rspSubs []*RspIntSub wsSubs []*WSIntSub - httpHandlers map[string]proxyWebUIHandler + httpHandlers map[string]ProxyWebUIHandler messageStorage map[int]*savedStorage } +// ProxyCredentials are a username/password combination used to represent an HTTP BasicAuth session type ProxyCredentials struct { Username string Password string } +// RequestInterceptor is a function that takes in a ProxyRequest and returns a modified ProxyRequest or nil to represent dropping the request type RequestInterceptor func(req *ProxyRequest) (*ProxyRequest, error) + +// ResponseInterceptor is a function that takes in a ProxyResponse and the original request and returns a modified ProxyResponse or nil to represent dropping the response type ResponseInterceptor func(req *ProxyRequest, rsp *ProxyResponse) (*ProxyResponse, error) + +// WSInterceptor is a function that takes in a ProxyWSMessage and the ProxyRequest/ProxyResponse which made up its handshake and returns and returns a modified ProxyWSMessage or nil to represent dropping the message. A WSInterceptor should be able to modify messages originating from both the client and the remote server. type WSInterceptor func(req *ProxyRequest, rsp *ProxyResponse, msg *ProxyWSMessage) (*ProxyWSMessage, error) +// ReqIntSub represents an active HTTP request interception session in an InterceptingProxy type ReqIntSub struct { id int Interceptor RequestInterceptor } +// RspIntSub represents an active HTTP response interception session in an InterceptingProxy type RspIntSub struct { id int Interceptor ResponseInterceptor } +// WSIntSub represents an active websocket interception session in an InterceptingProxy type WSIntSub struct { id int Interceptor WSInterceptor } +// SerializeHeader serializes the ProxyCredentials into a value that can be included in an Authorization header func (creds *ProxyCredentials) SerializeHeader() string { toEncode := []byte(fmt.Sprintf("%s:%s", creds.Username, creds.Password)) encoded := base64.StdEncoding.EncodeToString(toEncode) return fmt.Sprintf("Basic %s", encoded) } +// NewInterceptingProxy will create a new InterceptingProxy and have it log using the provided logger. If logger is nil, the proxy will log to ioutil.Discard func NewInterceptingProxy(logger *log.Logger) *InterceptingProxy { var iproxy InterceptingProxy + var useLogger *log.Logger + if logger != nil { + useLogger = logger + } else { + useLogger = log.New(ioutil.Discard, "[*] ", log.Lshortfile) + } + iproxy.messageStorage = make(map[int]*savedStorage) - iproxy.slistener = NewProxyListener(logger) - iproxy.server = newProxyServer(logger, &iproxy) - iproxy.logger = logger - iproxy.httpHandlers = make(map[string]proxyWebUIHandler) + iproxy.slistener = NewProxyListener(useLogger) + iproxy.server = newProxyServer(useLogger, &iproxy) + iproxy.logger = useLogger + iproxy.httpHandlers = make(map[string]ProxyWebUIHandler) go func() { iproxy.server.Serve(iproxy.slistener) @@ -97,8 +118,8 @@ func NewInterceptingProxy(logger *log.Logger) *InterceptingProxy { return &iproxy } +// Close closes all listeners being used by the proxy. Does not shut down internal HTTP server because there is no way to gracefully shut down an http server yet. func (iproxy *InterceptingProxy) Close() { - // Closes all associated listeners, but does not shut down the server because there is no way to gracefully shut down an http server yet :| // Will throw errors when the server finally shuts down and tries to call iproxy.slistener.Close a second time iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -106,6 +127,18 @@ func (iproxy *InterceptingProxy) Close() { //iproxy.server.Close() // Coming eventually... I hope } +// LoadCACertificates loads a private/public key pair which should be used when generating self-signed certs for TLS connections +func (iproxy *InterceptingProxy) LoadCACertificates(certFile, keyFile string) error { + caCert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("could not load certificate pair: %s", err.Error()) + } + + iproxy.SetCACertificate(&caCert) + return nil +} + +// SetCACertificate sets certificate which should be used when generating self-signed certs for TLS connections func (iproxy *InterceptingProxy) SetCACertificate(caCert *tls.Certificate) { if iproxy.slistener == nil { panic("intercepting proxy does not have a proxy listener") @@ -113,22 +146,33 @@ func (iproxy *InterceptingProxy) SetCACertificate(caCert *tls.Certificate) { iproxy.slistener.SetCACertificate(caCert) } +// GetCACertificate returns certificate used to self-sign certificates for TLS connections func (iproxy *InterceptingProxy) GetCACertificate() *tls.Certificate { return iproxy.slistener.GetCACertificate() } +// AddListener will have the proxy listen for HTTP connections on a listener. Proxy will attempt to strip TLS from the connection func (iproxy *InterceptingProxy) AddListener(l net.Listener) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() iproxy.slistener.AddListener(l) } +// Have the proxy listen for HTTP connections on a listener and transparently redirect them to the destination. Listeners added this way can only redirect requests to a single destination. However, it does not rely on the client being aware that it is using an HTTP proxy. +func (iproxy *InterceptingProxy) AddTransparentListener(l net.Listener, destHost string, destPort int, useTLS bool) { + iproxy.mtx.Lock() + defer iproxy.mtx.Unlock() + iproxy.slistener.AddTransparentListener(l, destHost, destPort, useTLS) +} + +// RemoveListner will have the proxy stop listening to a listener func (iproxy *InterceptingProxy) RemoveListener(l net.Listener) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() iproxy.slistener.RemoveListener(l) } +// GetMessageStorage takes in a storage ID and returns the storage associated with the ID func (iproxy *InterceptingProxy) GetMessageStorage(id int) (MessageStorage, string) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -139,6 +183,7 @@ func (iproxy *InterceptingProxy) GetMessageStorage(id int) (MessageStorage, stri return savedStorage.storage, savedStorage.description } +// AddMessageStorage associates a MessageStorage with the proxy and returns an ID to be used when referencing the storage in the future func (iproxy *InterceptingProxy) AddMessageStorage(storage MessageStorage, description string) int { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -147,6 +192,7 @@ func (iproxy *InterceptingProxy) AddMessageStorage(storage MessageStorage, descr return id } +// CloseMessageStorage closes a message storage associated with the proxy func (iproxy *InterceptingProxy) CloseMessageStorage(id int) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -158,12 +204,14 @@ func (iproxy *InterceptingProxy) CloseMessageStorage(id int) { savedStorage.storage.Close() } +// SavedStorage represents a storage associated with the proxy type SavedStorage struct { Id int Storage MessageStorage Description string } +// ListMessageStorage returns a list of storages associated with the proxy func (iproxy *InterceptingProxy) ListMessageStorage() []*SavedStorage { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -193,6 +241,7 @@ func (iproxy *InterceptingProxy) getWSSubs() []*WSIntSub { return iproxy.wsSubs } +// LoadScope loads the scope from the given storage and applies it to the proxy func (iproxy *InterceptingProxy) LoadScope(storageId int) error { // Try and set the scope savedStorage, ok := iproxy.messageStorage[storageId] @@ -210,12 +259,14 @@ func (iproxy *InterceptingProxy) LoadScope(storageId int) error { return nil } +// GetScopeChecker creates a RequestChecker which checks if a request matches the proxy's current scope func (iproxy *InterceptingProxy) GetScopeChecker() RequestChecker { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() return iproxy.scopeChecker } +// SetScopeChecker has the proxy use a specific RequestChecker to check if a request is in scope. If the checker returns true for a request it is considered in scope. Otherwise it is considered out of scope. func (iproxy *InterceptingProxy) SetScopeChecker(checker RequestChecker) error { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -232,12 +283,14 @@ func (iproxy *InterceptingProxy) SetScopeChecker(checker RequestChecker) error { return nil } +// GetScopeQuery returns the query associated with the proxy's scope. If the scope was set using SetScopeChecker, nil is returned func (iproxy *InterceptingProxy) GetScopeQuery() MessageQuery { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() return iproxy.scopeQuery } +// SetScopeQuery sets the scope of the proxy to include any request which matches the given MessageQuery func (iproxy *InterceptingProxy) SetScopeQuery(query MessageQuery) error { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -264,18 +317,40 @@ func (iproxy *InterceptingProxy) setScopeQuery(query MessageQuery) error { return nil } +// ClearScope removes all scope checks from the proxy so that all requests passing through the proxy will be considered in-scope +func (iproxy *InterceptingProxy) ClearScope() error { + iproxy.mtx.Lock() + defer iproxy.mtx.Unlock() + iproxy.scopeChecker = nil + iproxy.scopeChecker = nil + emptyQuery := make(MessageQuery, 0) + savedStorage, ok := iproxy.messageStorage[iproxy.proxyStorage] + if !ok { + savedStorage = nil + } + if savedStorage != nil { + if err := savedStorage.storage.SaveQuery("__scope", emptyQuery); err != nil { + return fmt.Errorf("could not clear scope in storage: %s", err.Error()) + } + } + return nil +} + +// SetNetDial sets the NetDialer that should be used to create outgoing connections when submitting HTTP requests. Overwrites the request's NetDialer func (iproxy *InterceptingProxy) SetNetDial(dialer NetDialer) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() iproxy.netDial = dialer } +// NetDial returns the dialer currently being used to create outgoing connections when submitting HTTP requests func (iproxy *InterceptingProxy) NetDial() NetDialer { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() return iproxy.netDial } +// ClearUpstreamProxy stops the proxy from using an upstream proxy for future connections func (iproxy *InterceptingProxy) ClearUpstreamProxy() { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -285,6 +360,7 @@ func (iproxy *InterceptingProxy) ClearUpstreamProxy() { iproxy.proxyIsSOCKS = false } +// SetUpstreamProxy causes the proxy to begin using an upstream HTTP proxy for submitted HTTP requests func (iproxy *InterceptingProxy) SetUpstreamProxy(proxyHost string, proxyPort int, creds *ProxyCredentials) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -295,6 +371,7 @@ func (iproxy *InterceptingProxy) SetUpstreamProxy(proxyHost string, proxyPort in iproxy.proxyCreds = creds } +// SetUpstreamSOCKSProxy causes the proxy to begin using an upstream SOCKS proxy for submitted HTTP requests func (iproxy *InterceptingProxy) SetUpstreamSOCKSProxy(proxyHost string, proxyPort int, creds *ProxyCredentials) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -305,24 +382,7 @@ func (iproxy *InterceptingProxy) SetUpstreamSOCKSProxy(proxyHost string, proxyPo iproxy.proxyCreds = creds } -func (iproxy *InterceptingProxy) ClearScope() error { - iproxy.mtx.Lock() - defer iproxy.mtx.Unlock() - iproxy.scopeChecker = nil - iproxy.scopeChecker = nil - emptyQuery := make(MessageQuery, 0) - savedStorage, ok := iproxy.messageStorage[iproxy.proxyStorage] - if !ok { - savedStorage = nil - } - if savedStorage != nil { - if err := savedStorage.storage.SaveQuery("__scope", emptyQuery); err != nil { - return fmt.Errorf("could not clear scope in storage: %s", err.Error()) - } - } - return nil -} - +// SubmitRequest submits a ProxyRequest. Does not automatically save the request/results to proxy storage func (iproxy *InterceptingProxy) SubmitRequest(req *ProxyRequest) error { oldDial := req.NetDial defer func() { req.NetDial = oldDial }() @@ -338,6 +398,7 @@ func (iproxy *InterceptingProxy) SubmitRequest(req *ProxyRequest) error { return SubmitRequest(req) } +// WSDial dials a remote server and submits the given request to initiate the handshake func (iproxy *InterceptingProxy) WSDial(req *ProxyRequest) (*WSSession, error) { oldDial := req.NetDial defer func() { req.NetDial = oldDial }() @@ -353,7 +414,8 @@ func (iproxy *InterceptingProxy) WSDial(req *ProxyRequest) (*WSSession, error) { return WSDial(req) } -func (iproxy *InterceptingProxy) AddReqIntSub(f RequestInterceptor) *ReqIntSub { +// AddReqInterceptor adds a RequestInterceptor to the proxy which will be used to modify HTTP requests as they pass through the proxy. Returns a struct representing the active interceptor. +func (iproxy *InterceptingProxy) AddReqInterceptor(f RequestInterceptor) *ReqIntSub { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -365,7 +427,8 @@ func (iproxy *InterceptingProxy) AddReqIntSub(f RequestInterceptor) *ReqIntSub { return sub } -func (iproxy *InterceptingProxy) RemoveReqIntSub(sub *ReqIntSub) { +// RemoveReqInterceptor removes an active request interceptor from the proxy +func (iproxy *InterceptingProxy) RemoveReqInterceptor(sub *ReqIntSub) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -377,7 +440,8 @@ func (iproxy *InterceptingProxy) RemoveReqIntSub(sub *ReqIntSub) { } } -func (iproxy *InterceptingProxy) AddRspIntSub(f ResponseInterceptor) *RspIntSub { +// AddRspInterceptor adds a ResponseInterceptor to the proxy which will be used to modify HTTP responses as they pass through the proxy. Returns a struct representing the active interceptor. +func (iproxy *InterceptingProxy) AddRspInterceptor(f ResponseInterceptor) *RspIntSub { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -389,7 +453,8 @@ func (iproxy *InterceptingProxy) AddRspIntSub(f ResponseInterceptor) *RspIntSub return sub } -func (iproxy *InterceptingProxy) RemoveRspIntSub(sub *RspIntSub) { +// RemoveRspInterceptor removes an active response interceptor from the proxy +func (iproxy *InterceptingProxy) RemoveRspInterceptor(sub *RspIntSub) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -401,7 +466,8 @@ func (iproxy *InterceptingProxy) RemoveRspIntSub(sub *RspIntSub) { } } -func (iproxy *InterceptingProxy) AddWSIntSub(f WSInterceptor) *WSIntSub { +// AddWSInterceptor adds a WSInterceptor to the proxy which will be used to modify both incoming and outgoing websocket messages as they pass through the proxy. Returns a struct representing the active interceptor. +func (iproxy *InterceptingProxy) AddWSInterceptor(f WSInterceptor) *WSIntSub { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -413,7 +479,8 @@ func (iproxy *InterceptingProxy) AddWSIntSub(f WSInterceptor) *WSIntSub { return sub } -func (iproxy *InterceptingProxy) RemoveWSIntSub(sub *WSIntSub) { +// RemoveWSInterceptor removes an active websocket interceptor from the proxy +func (iproxy *InterceptingProxy) RemoveWSInterceptor(sub *WSIntSub) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -425,6 +492,7 @@ func (iproxy *InterceptingProxy) RemoveWSIntSub(sub *WSIntSub) { } } +// SetProxyStorage sets which storage should be used to store messages as they pass through the proxy func (iproxy *InterceptingProxy) SetProxyStorage(storageId int) error { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -440,6 +508,7 @@ func (iproxy *InterceptingProxy) SetProxyStorage(storageId int) error { return nil } +// GetProxyStorage returns the storage being used to save messages as they pass through the proxy func (iproxy *InterceptingProxy) GetProxyStorage() MessageStorage { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() @@ -451,13 +520,15 @@ func (iproxy *InterceptingProxy) GetProxyStorage() MessageStorage { return savedStorage.storage } -func (iproxy *InterceptingProxy) AddHTTPHandler(host string, handler proxyWebUIHandler) { +// AddHTTPHandler causes the proxy to redirect requests to a host to an HTTPHandler. This can be used, for example, to create an internal web inteface. Be careful with what actions are allowed through the interface because the interface could be vulnerable to cross-site request forgery attacks. +func (iproxy *InterceptingProxy) AddHTTPHandler(host string, handler ProxyWebUIHandler) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() iproxy.httpHandlers[host] = handler } -func (iproxy *InterceptingProxy) GetHTTPHandler(host string) (proxyWebUIHandler, error) { +// GetHTTPHandler returns the HTTP handler for a given host +func (iproxy *InterceptingProxy) GetHTTPHandler(host string) (ProxyWebUIHandler, error) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() handler, ok := iproxy.httpHandlers[host] @@ -467,12 +538,14 @@ func (iproxy *InterceptingProxy) GetHTTPHandler(host string) (proxyWebUIHandler, return handler, nil } +// RemoveHTTPHandler removes the HTTP handler for a given host func (iproxy *InterceptingProxy) RemoveHTTPHandler(host string) { iproxy.mtx.Lock() defer iproxy.mtx.Unlock() delete(iproxy.httpHandlers, host) } +// ParseProxyRequest converts an http.Request read from a connection from a ProxyListener into a ProxyRequest func ParseProxyRequest(r *http.Request) (*ProxyRequest, error) { host, port, useTLS, err := DecodeRemoteAddr(r.RemoteAddr) if err != nil { @@ -482,6 +555,7 @@ func ParseProxyRequest(r *http.Request) (*ProxyRequest, error) { return pr, nil } +// BlankResponse writes a blank response to a http.ResponseWriter. Used when a request/response is dropped. func BlankResponse(w http.ResponseWriter) { w.Header().Set("Connection", "close") w.Header().Set("Cache-control", "no-cache") @@ -491,10 +565,17 @@ func BlankResponse(w http.ResponseWriter) { w.WriteHeader(200) } +// ErrResponse writes an error response to the given http.ResponseWriter. Used to give proxy error information to the browser func ErrResponse(w http.ResponseWriter, err error) { + w.Header().Set("Connection", "close") + w.Header().Set("Cache-control", "no-cache") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Cache-control", "no-store") + w.Header().Set("X-Frame-Options", "DENY") http.Error(w, err.Error(), http.StatusInternalServerError) } +// ServeHTTP is used to implement the interface required to have the proxy behave as an HTTP server func (iproxy *InterceptingProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { handler, err := iproxy.GetHTTPHandler(r.Host) if err == nil { diff --git a/proxyhttp.go b/proxyhttp.go index aaca057..dcde977 100644 --- a/proxyhttp.go +++ b/proxyhttp.go @@ -1,4 +1,4 @@ -package main +package puppy /* Wrappers around http.Request and http.Response to add helper functions needed by the proxy @@ -29,56 +29,83 @@ const ( ToClient ) +// A dialer used to create a net.Conn from a network and address type NetDialer func(network, addr string) (net.Conn, error) +// ProxyResponse is an http.Response with additional fields for use within the proxy type ProxyResponse struct { http.Response bodyBytes []byte - DbId string // ID used by storage implementation. Blank string = unsaved + + // Id used to reference this response in its associated MessageStorage. Blank string means it is not saved in any MessageStorage + DbId string + + // If this response was modified by the proxy, Unmangled is the response before it was modified. If the response was not modified, Unmangled is nil. Unmangled *ProxyResponse } +// ProxyRequest is an http.Request with additional fields for use within the proxy type ProxyRequest struct { http.Request - // Destination connection info - DestHost string - DestPort int + // Host where this request is intended to be sent when submitted + DestHost string + // Port that should be used when this request is submitted + DestPort int + // Whether TLS should be used when this request is submitted DestUseTLS bool - // Associated messages + // Response received from the server when this request was submitted. If the request does not have any associated response, ServerResponse is nil. ServerResponse *ProxyResponse - WSMessages []*ProxyWSMessage - Unmangled *ProxyRequest - - // Additional data - bodyBytes []byte - DbId string // ID used by storage implementation. Blank string = unsaved + // If the request was the handshake for a websocket session, WSMessages will be a slice of all the messages that were sent over that session. + WSMessages []*ProxyWSMessage + // If the request was modified by the proxy, Unmangled will point to the unmodified version of the request. Otherwise it is nil. + Unmangled *ProxyRequest + + // ID used to reference to this request in its associated MessageStorage + DbId string + // The time at which this request was submitted StartDatetime time.Time - EndDatetime time.Time + // The time at which the response to this request was received + EndDatetime time.Time - tags mapset.Set + bodyBytes []byte + tags mapset.Set + // The dialer that should be used when this request is submitted NetDial NetDialer } +// WSSession is an extension of websocket.Conn to contain a reference to the ProxyRequest used for the websocket handshake type WSSession struct { websocket.Conn - Request *ProxyRequest // Request used for handshake + // Request used for handshake + Request *ProxyRequest } +// ProxyWSMessage represents one message in a websocket session type ProxyWSMessage struct { - Type int - Message []byte + // The type of websocket message + Type int + + // The contents of the message + Message []byte + + // The direction of the message. Either ToServer or ToClient Direction int + // If the message was modified by the proxy, points to the original unmodified message Unmangled *ProxyWSMessage + // The time at which the message was sent (if sent to the server) or received (if received from the server) Timestamp time.Time - Request *ProxyRequest + // The request used for the handhsake for the session that this session was used for + Request *ProxyRequest - DbId string // ID used by storage implementation. Blank string = unsaved + // ID used to reference to this message in its associated MessageStorage + DbId string } +// PerformConnect submits a CONNECT request for the given host and port over the given connection func PerformConnect(conn net.Conn, destHost string, destPort int) error { connStr := []byte(fmt.Sprintf("CONNECT %s:%d HTTP/1.1\r\nHost: %s\r\nProxy-Connection: Keep-Alive\r\n\r\n", destHost, destPort, destHost)) conn.Write(connStr) @@ -92,6 +119,7 @@ func PerformConnect(conn net.Conn, destHost string, destPort int) error { return nil } +// NewProxyRequest creates a new proxy request with the given destination func NewProxyRequest(r *http.Request, destHost string, destPort int, destUseTLS bool) *ProxyRequest { var retReq *ProxyRequest if r != nil { @@ -111,10 +139,10 @@ func NewProxyRequest(r *http.Request, destHost string, destPort int, destUseTLS nil, make([]*ProxyWSMessage, 0), nil, - make([]byte, 0), "", time.Unix(0, 0), time.Unix(0, 0), + make([]byte, 0), mapset.NewSet(), nil, } @@ -130,10 +158,10 @@ func NewProxyRequest(r *http.Request, destHost string, destPort int, destUseTLS nil, make([]*ProxyWSMessage, 0), nil, - make([]byte, 0), "", time.Unix(0, 0), time.Unix(0, 0), + make([]byte, 0), mapset.NewSet(), nil, } @@ -145,6 +173,7 @@ func NewProxyRequest(r *http.Request, destHost string, destPort int, destUseTLS return retReq } +// ProxyRequestFromBytes parses a slice of bytes containing a well-formed HTTP request into a ProxyRequest. Does NOT correct incorrect Content-Length headers func ProxyRequestFromBytes(b []byte, destHost string, destPort int, destUseTLS bool) (*ProxyRequest, error) { buf := bytes.NewBuffer(b) httpReq, err := http.ReadRequest(bufio.NewReader(buf)) @@ -155,6 +184,7 @@ func ProxyRequestFromBytes(b []byte, destHost string, destPort int, destUseTLS b return NewProxyRequest(httpReq, destHost, destPort, destUseTLS), nil } +// NewProxyResponse creates a new ProxyResponse given an http.Response func NewProxyResponse(r *http.Response) *ProxyResponse { // Write/reread the request to make sure we get all the extra headers Go adds into req.Header oldClose := r.Close @@ -179,6 +209,7 @@ func NewProxyResponse(r *http.Response) *ProxyResponse { return retRsp } +// NewProxyResponse parses a ProxyResponse from a slice of bytes containing a well-formed HTTP response. Does NOT correct incorrect Content-Length headers func ProxyResponseFromBytes(b []byte) (*ProxyResponse, error) { buf := bytes.NewBuffer(b) httpRsp, err := http.ReadResponse(bufio.NewReader(buf), nil) @@ -188,6 +219,7 @@ func ProxyResponseFromBytes(b []byte) (*ProxyResponse, error) { return NewProxyResponse(httpRsp), nil } +// NewProxyWSMessage creates a new WSMessage given a type, message, and direction func NewProxyWSMessage(mtype int, message []byte, direction int) (*ProxyWSMessage, error) { return &ProxyWSMessage{ Type: mtype, @@ -199,6 +231,7 @@ func NewProxyWSMessage(mtype int, message []byte, direction int) (*ProxyWSMessag }, nil } +// DestScheme returns the scheme used by the request (ws, wss, http, or https) func (req *ProxyRequest) DestScheme() string { if req.IsWSUpgrade() { if req.DestUseTLS { @@ -215,9 +248,8 @@ func (req *ProxyRequest) DestScheme() string { } } +// FullURL is the same as req.URL but guarantees it will include the scheme, host, and port if necessary func (req *ProxyRequest) FullURL() *url.URL { - // Same as req.URL but guarantees it will include the scheme, host, and port if necessary - var u url.URL u = *(req.URL) // Copy the original req.URL u.Host = req.Host @@ -225,9 +257,8 @@ func (req *ProxyRequest) FullURL() *url.URL { return &u } +// Same as req.FullURL() but uses DestHost and DestPort for the host and port of the URL func (req *ProxyRequest) DestURL() *url.URL { - // Same as req.FullURL() but uses DestHost and DestPort for the host and port - var u url.URL u = *(req.URL) // Copy the original req.URL u.Scheme = req.DestScheme() @@ -241,10 +272,12 @@ func (req *ProxyRequest) DestURL() *url.URL { return &u } +// Submit submits the request over the given connection. Does not take into account DestHost, DestPort, or DestUseTLS func (req *ProxyRequest) Submit(conn net.Conn) error { return req.submit(conn, false, nil) } +// Submit submits the request in proxy form over the given connection for use with an upstream HTTP proxy. Does not take into account DestHost, DestPort, or DestUseTLS func (req *ProxyRequest) SubmitProxy(conn net.Conn, creds *ProxyCredentials) error { return req.submit(conn, true, creds) } @@ -281,6 +314,7 @@ func (req *ProxyRequest) submit(conn net.Conn, forProxy bool, proxyCreds *ProxyC return nil } +// WSDial performs a websocket handshake over the given connection. Does not take into account DestHost, DestPort, or DestUseTLS func (req *ProxyRequest) WSDial(conn net.Conn) (*WSSession, error) { if !req.IsWSUpgrade() { return nil, fmt.Errorf("could not start websocket session: request is not a websocket handshake request") @@ -317,14 +351,17 @@ func (req *ProxyRequest) WSDial(conn net.Conn) (*WSSession, error) { return wsession, nil } +// WSDial dials the target server and performs a websocket handshake over the new connection. Uses destination information from the request. func WSDial(req *ProxyRequest) (*WSSession, error) { return wsDial(req, false, "", 0, nil, false) } +// WSDialProxy dials the HTTP proxy server, performs a CONNECT handshake to connect to the remote server, then performs a websocket handshake over the new connection. Uses destination information from the request. func WSDialProxy(req *ProxyRequest, proxyHost string, proxyPort int, creds *ProxyCredentials) (*WSSession, error) { return wsDial(req, true, proxyHost, proxyPort, creds, false) } +// WSDialSOCKSProxy connects to the target host through the SOCKS proxy and performs a websocket handshake over the new connection. Uses destination information from the request. func WSDialSOCKSProxy(req *ProxyRequest, proxyHost string, proxyPort int, creds *ProxyCredentials) (*WSSession, error) { return wsDial(req, true, proxyHost, proxyPort, creds, true) } @@ -386,6 +423,7 @@ func wsDial(req *ProxyRequest, useProxy bool, proxyHost string, proxyPort int, p return req.WSDial(conn) } +// IsWSUpgrade returns whether the request is used to initiate a websocket handshake func (req *ProxyRequest) IsWSUpgrade() bool { for k, v := range req.Header { for _, vv := range v { @@ -397,6 +435,7 @@ func (req *ProxyRequest) IsWSUpgrade() bool { return false } +// StripProxyHeaders removes headers associated with requests made to a proxy from the request func (req *ProxyRequest) StripProxyHeaders() { if !req.IsWSUpgrade() { req.Header.Del("Connection") @@ -407,6 +446,7 @@ func (req *ProxyRequest) StripProxyHeaders() { req.Header.Del("Proxy-Authorization") } +// Eq checks whether the request is the same as another request and has the same destination information func (req *ProxyRequest) Eq(other *ProxyRequest) bool { if req.StatusLine() != other.StatusLine() || !reflect.DeepEqual(req.Header, other.Header) || @@ -420,6 +460,7 @@ func (req *ProxyRequest) Eq(other *ProxyRequest) bool { return true } +// Clone returns a request with the same contents and destination information as the original func (req *ProxyRequest) Clone() *ProxyRequest { buf := bytes.NewBuffer(make([]byte, 0)) req.RepeatableWrite(buf) @@ -430,10 +471,11 @@ func (req *ProxyRequest) Clone() *ProxyRequest { newReq.DestHost = req.DestHost newReq.DestPort = req.DestPort newReq.DestUseTLS = req.DestUseTLS - newReq.Header = CopyHeader(req.Header) + newReq.Header = copyHeader(req.Header) return newReq } +// DeepClone returns a request with the same contents, destination, and storage information information as the original along with a deep clone of the associated response, the unmangled version of the request, and any websocket messages func (req *ProxyRequest) DeepClone() *ProxyRequest { // Returns a request with the same request, response, and associated websocket messages newReq := req.Clone() @@ -459,11 +501,13 @@ func (req *ProxyRequest) resetBodyReader() { req.Body = ioutil.NopCloser(bytes.NewBuffer(req.BodyBytes())) } +// RepeatableWrite is the same as http.Request.Write except that it can be safely called multiple times func (req *ProxyRequest) RepeatableWrite(w io.Writer) error { defer req.resetBodyReader() return req.Write(w) } +// RepeatableWrite is the same as http.Request.ProxyWrite except that it can be safely called multiple times func (req *ProxyRequest) RepeatableProxyWrite(w io.Writer, proxyCreds *ProxyCredentials) error { defer req.resetBodyReader() if proxyCreds != nil { @@ -474,11 +518,12 @@ func (req *ProxyRequest) RepeatableProxyWrite(w io.Writer, proxyCreds *ProxyCred return req.WriteProxy(w) } +// BodyBytes returns the bytes of the request body func (req *ProxyRequest) BodyBytes() []byte { return DuplicateBytes(req.bodyBytes) - } +// SetBodyBytes sets the bytes of the request body and updates the Content-Length header func (req *ProxyRequest) SetBodyBytes(bs []byte) { req.bodyBytes = bs req.resetBodyReader() @@ -490,12 +535,14 @@ func (req *ProxyRequest) SetBodyBytes(bs []byte) { req.Header.Set("Content-Length", strconv.Itoa(len(bs))) } +// FullMessage returns a slice of bytes containing the full HTTP message for the request func (req *ProxyRequest) FullMessage() []byte { buf := bytes.NewBuffer(make([]byte, 0)) req.RepeatableWrite(buf) return buf.Bytes() } +// PostParameters attempts to parse POST parameters from the body of the request func (req *ProxyRequest) PostParameters() (url.Values, error) { vals, err := url.ParseQuery(string(req.BodyBytes())) if err != nil { @@ -504,21 +551,25 @@ func (req *ProxyRequest) PostParameters() (url.Values, error) { return vals, nil } +// SetPostParameter sets the value of a post parameter in the message body. If the body does not contain well-formed data, it is deleted replaced with a well-formed body containing only the new parameter func (req *ProxyRequest) SetPostParameter(key string, value string) { req.PostForm.Set(key, value) req.SetBodyBytes([]byte(req.PostForm.Encode())) } +// AddPostParameter adds a post parameter to the body of the request even if a duplicate exists. If the body does not contain well-formed data, it is deleted replaced with a well-formed body containing only the new parameter func (req *ProxyRequest) AddPostParameter(key string, value string) { req.PostForm.Add(key, value) req.SetBodyBytes([]byte(req.PostForm.Encode())) } -func (req *ProxyRequest) DeletePostParameter(key string, value string) { +// DeletePostParameter removes a parameter from the body of the request. If the body does not contain well-formed data, it is deleted replaced with a well-formed body containing only the new parameter +func (req *ProxyRequest) DeletePostParameter(key string) { req.PostForm.Del(key) req.SetBodyBytes([]byte(req.PostForm.Encode())) } +// SetURLParameter sets the value of a URL parameter and updates ProxyRequest.URL func (req *ProxyRequest) SetURLParameter(key string, value string) { q := req.URL.Query() q.Set(key, value) @@ -526,11 +577,13 @@ func (req *ProxyRequest) SetURLParameter(key string, value string) { req.ParseForm() } +// URLParameters returns the values of the request's URL parameters func (req *ProxyRequest) URLParameters() url.Values { vals := req.URL.Query() return vals } +// AddURLParameter adds a URL parameter to the request ignoring duplicates func (req *ProxyRequest) AddURLParameter(key string, value string) { q := req.URL.Query() q.Add(key, value) @@ -538,29 +591,35 @@ func (req *ProxyRequest) AddURLParameter(key string, value string) { req.ParseForm() } -func (req *ProxyRequest) DeleteURLParameter(key string, value string) { +// DeleteURLParameter removes a URL parameter from the request +func (req *ProxyRequest) DeleteURLParameter(key string) { q := req.URL.Query() q.Del(key) req.URL.RawQuery = q.Encode() req.ParseForm() } +// AddTag adds a tag to the request func (req *ProxyRequest) AddTag(tag string) { req.tags.Add(tag) } +// CheckTag returns whether the request has a given tag func (req *ProxyRequest) CheckTag(tag string) bool { return req.tags.Contains(tag) } +// RemoveTag removes a tag from the request func (req *ProxyRequest) RemoveTag(tag string) { req.tags.Remove(tag) } +// ClearTag removes all of the tags associated with the request func (req *ProxyRequest) ClearTags() { req.tags.Clear() } +// Tags returns a slice containing all of the tags associated with the request func (req *ProxyRequest) Tags() []string { items := req.tags.ToSlice() retslice := make([]string, 0) @@ -573,6 +632,7 @@ func (req *ProxyRequest) Tags() []string { return retslice } +// HTTPPath returns the path of the associated with the request func (req *ProxyRequest) HTTPPath() string { // The path used in the http request u := *req.URL @@ -583,10 +643,12 @@ func (req *ProxyRequest) HTTPPath() string { return u.String() } +// StatusLine returns the status line associated with the request func (req *ProxyRequest) StatusLine() string { return fmt.Sprintf("%s %s %s", req.Method, req.HTTPPath(), req.Proto) } +// HeaderSection returns the header section of the request without the additional \r\n at the end func (req *ProxyRequest) HeaderSection() string { retStr := req.StatusLine() retStr += "\r\n" @@ -603,21 +665,25 @@ func (rsp *ProxyResponse) resetBodyReader() { rsp.Body = ioutil.NopCloser(bytes.NewBuffer(rsp.BodyBytes())) } +// RepeatableWrite is the same as http.Response.Write except that it can safely be called multiple times func (rsp *ProxyResponse) RepeatableWrite(w io.Writer) error { defer rsp.resetBodyReader() return rsp.Write(w) } +// BodyBytes returns the bytes contained in the body of the response func (rsp *ProxyResponse) BodyBytes() []byte { return DuplicateBytes(rsp.bodyBytes) } +// SetBodyBytes sets the bytes in the body of the response and updates the Content-Length header func (rsp *ProxyResponse) SetBodyBytes(bs []byte) { rsp.bodyBytes = bs rsp.resetBodyReader() rsp.Header.Set("Content-Length", strconv.Itoa(len(bs))) } +// Clone returns a response with the same status line, headers, and body as the response func (rsp *ProxyResponse) Clone() *ProxyResponse { buf := bytes.NewBuffer(make([]byte, 0)) rsp.RepeatableWrite(buf) @@ -628,6 +694,7 @@ func (rsp *ProxyResponse) Clone() *ProxyResponse { return newRsp } +// DeepClone returns a response with the same status line, headers, and body as the original response along with a deep clone of its unmangled version if it exists func (rsp *ProxyResponse) DeepClone() *ProxyResponse { newRsp := rsp.Clone() newRsp.DbId = rsp.DbId @@ -637,6 +704,7 @@ func (rsp *ProxyResponse) DeepClone() *ProxyResponse { return newRsp } +// Eq returns whether the response has the same contents as another response func (rsp *ProxyResponse) Eq(other *ProxyResponse) bool { if rsp.StatusLine() != other.StatusLine() || !reflect.DeepEqual(rsp.Header, other.Header) || @@ -646,14 +714,16 @@ func (rsp *ProxyResponse) Eq(other *ProxyResponse) bool { return true } +// FullMessage returns the full HTTP message of the response func (rsp *ProxyResponse) FullMessage() []byte { buf := bytes.NewBuffer(make([]byte, 0)) rsp.RepeatableWrite(buf) return buf.Bytes() } +// Returns the status text to be used in the http request func (rsp *ProxyResponse) HTTPStatus() string { - // The status text to be used in the http request + // The status text to be used in the http request. Relies on being the same implementation as http.Response text := rsp.Status if text == "" { text = http.StatusText(rsp.StatusCode) @@ -668,11 +738,13 @@ func (rsp *ProxyResponse) HTTPStatus() string { return text } +// StatusLine returns the status line of the response func (rsp *ProxyResponse) StatusLine() string { // Status line, stolen from net/http/response.go return fmt.Sprintf("HTTP/%d.%d %03d %s", rsp.ProtoMajor, rsp.ProtoMinor, rsp.StatusCode, rsp.HTTPStatus()) } +// HeaderSection returns the header section of the response (without the extra trailing \r\n) func (rsp *ProxyResponse) HeaderSection() string { retStr := rsp.StatusLine() retStr += "\r\n" @@ -683,6 +755,7 @@ func (rsp *ProxyResponse) HeaderSection() string { } return retStr } + func (msg *ProxyWSMessage) String() string { var dirStr string if msg.Direction == ToClient { @@ -693,6 +766,7 @@ func (msg *ProxyWSMessage) String() string { return fmt.Sprintf("{WS Message msg=\"%s\", type=%d, dir=%s}", string(msg.Message), msg.Type, dirStr) } +// Clone returns a copy of the original message. It will have the same type, message, direction, timestamp, and request func (msg *ProxyWSMessage) Clone() *ProxyWSMessage { var retMsg ProxyWSMessage retMsg.Type = msg.Type @@ -703,6 +777,7 @@ func (msg *ProxyWSMessage) Clone() *ProxyWSMessage { return &retMsg } +// DeepClone returns a clone of the original message and a deep clone of the unmangled version if it exists func (msg *ProxyWSMessage) DeepClone() *ProxyWSMessage { retMsg := msg.Clone() retMsg.DbId = msg.DbId @@ -712,6 +787,7 @@ func (msg *ProxyWSMessage) DeepClone() *ProxyWSMessage { return retMsg } +// Eq checks if the message has the same type, direction, and message as another message func (msg *ProxyWSMessage) Eq(other *ProxyWSMessage) bool { if msg.Type != other.Type || msg.Direction != other.Direction || @@ -721,7 +797,7 @@ func (msg *ProxyWSMessage) Eq(other *ProxyWSMessage) bool { return true } -func CopyHeader(hd http.Header) http.Header { +func copyHeader(hd http.Header) http.Header { var ret http.Header = make(http.Header) for k, vs := range hd { for _, v := range vs { @@ -796,14 +872,17 @@ func submitRequest(req *ProxyRequest, useProxy bool, proxyHost string, } } +// SubmitRequest opens a connection to the request's DestHost:DestPort, using TLS if DestUseTLS is set, submits the request, and sets req.Response with the response when a response is received func SubmitRequest(req *ProxyRequest) error { return submitRequest(req, false, "", 0, nil, false) } +// SubmitRequestProxy connects to the given HTTP proxy, performs neccessary handshakes, and submits the request to its destination. req.Response will be set once a response is received func SubmitRequestProxy(req *ProxyRequest, proxyHost string, proxyPort int, creds *ProxyCredentials) error { return submitRequest(req, true, proxyHost, proxyPort, creds, false) } +// SubmitRequestProxy connects to the given SOCKS proxy, performs neccessary handshakes, and submits the request to its destination. req.Response will be set once a response is received func SubmitRequestSOCKSProxy(req *ProxyRequest, proxyHost string, proxyPort int, creds *ProxyCredentials) error { return submitRequest(req, true, proxyHost, proxyPort, creds, true) } diff --git a/proxyhttp_test.go b/proxyhttp_test.go index 02f2a5f..e088bad 100644 --- a/proxyhttp_test.go +++ b/proxyhttp_test.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "net/url" diff --git a/proxylistener.go b/proxylistener.go index 043e287..42d27a5 100644 --- a/proxylistener.go +++ b/proxylistener.go @@ -1,10 +1,11 @@ -package main +package puppy import ( "bufio" "bytes" "crypto/tls" "fmt" + "io/ioutil" "log" "net" "net/http" @@ -16,15 +17,6 @@ import ( "github.com/deckarep/golang-set" ) -/* -func logReq(req *http.Request, logger *log.Logger) { - buf := new(bytes.Buffer) - req.Write(buf) - s := buf.String() - logger.Print(s) -} -*/ - const ( ProxyStopped = iota ProxyStarting @@ -34,16 +26,13 @@ const ( var getNextConnId = IdCounter() var getNextListenerId = IdCounter() -/* -A type representing the "Addr" for our internal connections -*/ -type InternalAddr struct{} +type internalAddr struct{} -func (InternalAddr) Network() string { +func (internalAddr) Network() string { return "" } -func (InternalAddr) String() string { +func (internalAddr) String() string { return "" } @@ -56,8 +45,17 @@ type ProxyConn interface { Id() int Logger() *log.Logger + // Set the CA certificate to be used to sign TLS connections SetCACertificate(*tls.Certificate) + + // If the connection tries to start TLS, attempt to strip it so that further reads will get the decrypted text, otherwise it will just pass the plaintext StartMaybeTLS(hostname string) (bool, error) + + // Have all requests produced by this connection have the given destination information. Removes the need for requests generated by this connection to be aware they are being submitted through a proxy + SetTransparentMode(destHost string, destPort int, useTLS bool) + + // End transparent mode + EndTransparentMode() } type proxyAddr struct { @@ -73,10 +71,12 @@ type proxyConn struct { conn net.Conn // Wrapped connection readReq *http.Request // A replaced request caCert *tls.Certificate -} + mtx sync.Mutex -// ProxyAddr implementations/functions + transparentMode bool +} +// Encode the destination information to be stored in the remote address func EncodeRemoteAddr(host string, port int, useTLS bool) string { var tlsInt int if useTLS { @@ -87,6 +87,7 @@ func EncodeRemoteAddr(host string, port int, useTLS bool) string { return fmt.Sprintf("%s/%d/%d", host, port, tlsInt) } +// Decode destination information from a remote address func DecodeRemoteAddr(addrStr string) (host string, port int, useTLS bool, err error) { parts := strings.Split(addrStr, "/") if len(parts) != 3 { @@ -138,6 +139,7 @@ func (c bufferedConn) Read(p []byte) (int, error) { } //// Implement net.Conn + func (c *proxyConn) Read(b []byte) (n int, err error) { if c.readReq != nil { buf := new(bytes.Buffer) @@ -186,15 +188,25 @@ func (c *proxyConn) RemoteAddr() net.Addr { } //// Implement ProxyConn + func (pconn *proxyConn) Id() int { + pconn.mtx.Lock() + defer pconn.mtx.Unlock() + return pconn.id } func (pconn *proxyConn) Logger() *log.Logger { + pconn.mtx.Lock() + defer pconn.mtx.Unlock() + return pconn.logger } func (pconn *proxyConn) SetCACertificate(cert *tls.Certificate) { + pconn.mtx.Lock() + defer pconn.mtx.Unlock() + pconn.caCert = cert } @@ -202,6 +214,9 @@ func (pconn *proxyConn) StartMaybeTLS(hostname string) (bool, error) { // Prepares to start doing TLS if the client starts. Returns whether TLS was started // Wrap the ProxyConn's net.Conn in a bufferedConn + pconn.mtx.Lock() + defer pconn.mtx.Unlock() + bufConn := bufferedConn{bufio.NewReader(pconn.conn), pconn.conn} usingTLS := false @@ -219,7 +234,7 @@ func (pconn *proxyConn) StartMaybeTLS(hostname string) (bool, error) { return false, err } - cert, err := SignHost(*pconn.caCert, []string{hostname}) + cert, err := signHost(*pconn.caCert, []string{hostname}) if err != nil { return false, err } @@ -237,14 +252,37 @@ func (pconn *proxyConn) StartMaybeTLS(hostname string) (bool, error) { } } -func NewProxyConn(c net.Conn, l *log.Logger) *proxyConn { +func (pconn *proxyConn) SetTransparentMode(destHost string, destPort int, useTLS bool) { + pconn.mtx.Lock() + defer pconn.mtx.Unlock() + + pconn.Addr = &proxyAddr{Host: destHost, + Port: destPort, + UseTLS: useTLS, + } + pconn.transparentMode = true +} + +func (pconn *proxyConn) EndTransparentMode() { + pconn.mtx.Lock() + defer pconn.mtx.Unlock() + + pconn.transparentMode = false +} + +func newProxyConn(c net.Conn, l *log.Logger) *proxyConn { + // converts a connection into a proxyConn a := proxyAddr{Host: "", Port: -1, UseTLS: false} p := proxyConn{Addr: &a, logger: l, conn: c, readReq: nil} p.id = getNextConnId() + p.transparentMode = false return &p } func (pconn *proxyConn) returnRequest(req *http.Request) { + pconn.mtx.Lock() + defer pconn.mtx.Unlock() + pconn.readReq = req } @@ -254,24 +292,33 @@ connections on each listener and read HTTP messages from the connection. Will attempt to spoof TLS from incoming HTTP requests. Accept() returns a ProxyConn which transmists one unencrypted HTTP request and contains the intended destination for -each request. +each request in the RemoteAddr. */ type ProxyListener struct { net.Listener + // The current state of the listener State int inputListeners mapset.Set mtx sync.Mutex logger *log.Logger outputConns chan ProxyConn - inputConns chan net.Conn + inputConns chan *inputConn outputConnDone chan struct{} inputConnDone chan struct{} listenWg sync.WaitGroup caCert *tls.Certificate } +type inputConn struct { + listener *ProxyListener + conn net.Conn + + transparentMode bool + transparentAddr *proxyAddr +} + type listenerData struct { Id int Listener net.Listener @@ -284,12 +331,19 @@ func newListenerData(listener net.Listener) *listenerData { return &l } +// NewProxyListener creates and starts a new proxy listener that will log to the given logger func NewProxyListener(logger *log.Logger) *ProxyListener { - l := ProxyListener{logger: logger, State: ProxyStarting} + var useLogger *log.Logger + if logger != nil { + useLogger = logger + } else { + useLogger = log.New(ioutil.Discard, "[*] ", log.Lshortfile) + } + l := ProxyListener{logger: useLogger, State: ProxyStarting} l.inputListeners = mapset.NewSet() l.outputConns = make(chan ProxyConn) - l.inputConns = make(chan net.Conn) + l.inputConns = make(chan *inputConn) l.outputConnDone = make(chan struct{}) l.inputConnDone = make(chan struct{}) @@ -320,6 +374,7 @@ func NewProxyListener(logger *log.Logger) *ProxyListener { return &l } +// Accept accepts a new connection from any of its listeners func (listener *ProxyListener) Accept() (net.Conn, error) { if listener.outputConns == nil || listener.inputConns == nil || @@ -338,6 +393,7 @@ func (listener *ProxyListener) Accept() (net.Conn, error) { } } +// Close closes all of the listeners associated with the ProxyListener func (listener *ProxyListener) Close() error { listener.mtx.Lock() defer listener.mtx.Unlock() @@ -361,14 +417,29 @@ func (listener *ProxyListener) Close() error { } func (listener *ProxyListener) Addr() net.Addr { - return InternalAddr{} + return internalAddr{} } -// Add a listener for the ProxyListener to listen on +// AddListener adds a listener for the ProxyListener to listen on func (listener *ProxyListener) AddListener(inlisten net.Listener) error { listener.mtx.Lock() defer listener.mtx.Unlock() + return listener.addListener(inlisten, false, nil) +} + +// AddTransparentListener is the same as AddListener, but all of the connections will be in transparent mode +func (listener *ProxyListener) AddTransparentListener(inlisten net.Listener, destHost string, destPort int, useTLS bool) error { + listener.mtx.Lock() + defer listener.mtx.Unlock() + addr := &proxyAddr{ + Host: destHost, + Port: destPort, + UseTLS: useTLS, + } + return listener.addListener(inlisten, true, addr) +} +func (listener *ProxyListener) addListener(inlisten net.Listener, transparentMode bool, destAddr *proxyAddr) error { listener.logger.Println("Adding listener to ProxyListener:", inlisten) il := newListenerData(inlisten) l := listener @@ -383,7 +454,13 @@ func (listener *ProxyListener) AddListener(inlisten net.Listener) error { return } l.logger.Println("Received conn form listener", il.Id) - l.inputConns <- c + newConn := &inputConn{ + conn: c, + listener: nil, + transparentMode: transparentMode, + transparentAddr: destAddr, + } + l.inputConns <- newConn } }() listener.inputListeners.Add(il) @@ -391,7 +468,7 @@ func (listener *ProxyListener) AddListener(inlisten net.Listener) error { return nil } -// Close a listener and remove it from the slistener. Does not kill active connections. +// RemoveListener closes a listener and removes it from the ProxyListener. Does not kill active connections. func (listener *ProxyListener) RemoveListener(inlisten net.Listener) error { listener.mtx.Lock() defer listener.mtx.Unlock() @@ -402,10 +479,16 @@ func (listener *ProxyListener) RemoveListener(inlisten net.Listener) error { return nil } +// TKTK working here // Take in a connection, strip TLS, get destination info, and push a ProxyConn to the listener.outputConnection channel -func (listener *ProxyListener) translateConn(inconn net.Conn) error { - pconn := NewProxyConn(inconn, listener.logger) +func (listener *ProxyListener) translateConn(inconn *inputConn) error { + pconn := newProxyConn(inconn.conn, listener.logger) pconn.SetCACertificate(listener.GetCACertificate()) + if inconn.transparentMode { + pconn.SetTransparentMode(inconn.transparentAddr.Host, + inconn.transparentAddr.Port, + inconn.transparentAddr.UseTLS) + } var host string = "" var port int = -1 @@ -437,7 +520,7 @@ func (listener *ProxyListener) translateConn(inconn net.Conn) error { if request.Method == "CONNECT" { // Respond that we connected resp := http.Response{Status: "Connection established", Proto: "HTTP/1.1", ProtoMajor: 1, StatusCode: 200} - err := resp.Write(inconn) + err := resp.Write(inconn.conn) if err != nil { listener.logger.Println("Could not write CONNECT response:", err) return err @@ -463,9 +546,13 @@ func (listener *ProxyListener) translateConn(inconn net.Conn) error { port = 80 } } - pconn.Addr.Host = host - pconn.Addr.Port = port - pconn.Addr.UseTLS = useTLS + + if !pconn.transparentMode { + pconn.Addr.Host = host + pconn.Addr.Port = port + pconn.Addr.UseTLS = useTLS + } + var useTLSStr string if pconn.Addr.UseTLS { useTLSStr = "YES" @@ -479,6 +566,7 @@ func (listener *ProxyListener) translateConn(inconn net.Conn) error { return nil } +// SetCACertificate sets which certificate the listener should be used when spoofing TLS func (listener *ProxyListener) SetCACertificate(caCert *tls.Certificate) { listener.mtx.Lock() defer listener.mtx.Unlock() @@ -486,6 +574,7 @@ func (listener *ProxyListener) SetCACertificate(caCert *tls.Certificate) { listener.caCert = caCert } +// SetCACertificate gets which certificate the listener is using when spoofing TLS func (listener *ProxyListener) GetCACertificate() *tls.Certificate { listener.mtx.Lock() defer listener.mtx.Unlock() diff --git a/proxymessages.go b/proxymessages.go index c07a07d..a929c24 100644 --- a/proxymessages.go +++ b/proxymessages.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "bufio" @@ -11,7 +11,6 @@ import ( "io" "log" "net" - "os" "sort" "strconv" "strings" @@ -21,6 +20,7 @@ import ( "github.com/gorilla/websocket" ) +// Creates a MessageListener that implements the default puppy handlers. See the message docs for more info func NewProxyMessageListener(logger *log.Logger, iproxy *InterceptingProxy) *MessageListener { l := NewMessageListener(logger, iproxy) @@ -29,6 +29,7 @@ func NewProxyMessageListener(logger *log.Logger, iproxy *InterceptingProxy) *Mes l.AddHandler("savenew", saveNewHandler) l.AddHandler("storagequery", storageQueryHandler) l.AddHandler("validatequery", validateQueryHandler) + l.AddHandler("checkrequest", checkRequestHandler) l.AddHandler("setscope", setScopeHandler) l.AddHandler("viewscope", viewScopeHandler) l.AddHandler("addtag", addTagHandler) @@ -57,7 +58,7 @@ func NewProxyMessageListener(logger *log.Logger, iproxy *InterceptingProxy) *Mes return l } -// Message input structs +// JSON data representing a ProxyRequest type RequestJSON struct { DestHost string DestPort int @@ -79,6 +80,7 @@ type RequestJSON struct { DbId string `json:"DbId,omitempty"` } +// JSON data representing a ProxyResponse type ResponseJSON struct { ProtoMajor int ProtoMinor int @@ -92,6 +94,7 @@ type ResponseJSON struct { DbId string } +// JSON data representing a ProxyWSMessage type WSMessageJSON struct { Message string IsBinary bool @@ -102,6 +105,7 @@ type WSMessageJSON struct { DbId string } +// Check that the RequestJSON contains valid data func (reqd *RequestJSON) Validate() error { if reqd.DestHost == "" { return errors.New("request is missing target host") @@ -114,6 +118,7 @@ func (reqd *RequestJSON) Validate() error { return nil } +// Convert RequestJSON into a ProxyRequest func (reqd *RequestJSON) Parse() (*ProxyRequest, error) { if err := reqd.Validate(); err != nil { return nil, err @@ -187,6 +192,7 @@ func (reqd *RequestJSON) Parse() (*ProxyRequest, error) { return req, nil } +// Convert a ProxyRequest into JSON data. If headersOnly is true, the JSON data will only contain the headers and metadata of the message func NewRequestJSON(req *ProxyRequest, headersOnly bool) *RequestJSON { newHeaders := make(map[string][]string) @@ -243,10 +249,12 @@ func NewRequestJSON(req *ProxyRequest, headersOnly bool) *RequestJSON { return ret } +// Ensure that response JSON data is valid func (rspd *ResponseJSON) Validate() error { return nil } +// Convert response JSON data into a ProxyResponse func (rspd *ResponseJSON) Parse() (*ProxyResponse, error) { if err := rspd.Validate(); err != nil { return nil, err @@ -295,6 +303,7 @@ func (rspd *ResponseJSON) Parse() (*ProxyResponse, error) { return rsp, nil } +// Serialize a ProxyResponse into JSON data. If headersOnly is true, the JSON data will only contain the headers and metadata of the message func NewResponseJSON(rsp *ProxyResponse, headersOnly bool) *ResponseJSON { newHeaders := make(map[string][]string) for k, vs := range rsp.Header { @@ -331,6 +340,7 @@ func NewResponseJSON(rsp *ProxyResponse, headersOnly bool) *ResponseJSON { return ret } +// Parse websocket message JSON data into a ProxyWSMEssage func (wsmd *WSMessageJSON) Parse() (*ProxyWSMessage, error) { var Direction int if wsmd.ToServer { @@ -371,6 +381,7 @@ func (wsmd *WSMessageJSON) Parse() (*ProxyWSMessage, error) { return retData, nil } +// Serialize a websocket message into JSON data func NewWSMessageJSON(wsm *ProxyWSMessage) *WSMessageJSON { isBinary := false if wsm.Type == websocket.BinaryMessage { @@ -399,8 +410,7 @@ func NewWSMessageJSON(wsm *ProxyWSMessage) *WSMessageJSON { return ret } -// Functions to remove extra metadata from submitted messages - +// Clears metadata (start/end time, DbId) and dependent message data (response, websocket messages, and unmangled versions) from the RequestJSON func CleanReqJSON(req *RequestJSON) { req.StartTime = 0 req.EndTime = 0 @@ -410,11 +420,13 @@ func CleanReqJSON(req *RequestJSON) { req.DbId = "" } +// Clears metadata (DbId) and dependent message data (unmangled version) from the ResponseJSON func CleanRspJSON(rsp *ResponseJSON) { rsp.Unmangled = nil rsp.DbId = "" } +// Clears metadata (timestamp, DbId) and dependent message data (unmangled version) from the WSMessageJSON func CleanWSJSON(wsm *WSMessageJSON) { wsm.Timestamp = 0 wsm.Unmangled = nil @@ -461,7 +473,6 @@ func submitHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Interceptin return } - CleanReqJSON(mreq.Request) req, err := mreq.Request.Parse() if err != nil { ErrorResponse(c, fmt.Sprintf("error parsing http request: %s", err.Error())) @@ -499,7 +510,7 @@ func submitHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Interceptin SaveNew */ type saveNewMessage struct { - Request RequestJSON + Request *RequestJSON Storage int } @@ -604,7 +615,7 @@ func storageQueryHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Inter } } else { logger.Println("Search query is multple sets of arguments, creating checker and checking naively...") - goQuery, err := StrQueryToGoQuery(mreq.Query) + goQuery, err := StrQueryToMsgQuery(mreq.Query) if err != nil { ErrorResponse(c, err.Error()) return @@ -648,7 +659,7 @@ func validateQueryHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Inte return } - goQuery, err := StrQueryToGoQuery(mreq.Query) + goQuery, err := StrQueryToMsgQuery(mreq.Query) if err != nil { ErrorResponse(c, err.Error()) return @@ -662,6 +673,53 @@ func validateQueryHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Inte MessageResponse(c, &successResult{Success: true}) } +/* +CheckRequest +*/ + +type checkRequestMessage struct { + Query StrMessageQuery + Request *RequestJSON +} + +type checkRequestResponse struct { + Result bool + Success bool +} + +func checkRequestHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *InterceptingProxy) { + mreq := checkRequestMessage{} + if err := json.Unmarshal(b, &mreq); err != nil { + ErrorResponse(c, "error parsing query message") + return + } + + req, err := mreq.Request.Parse() + if err != nil { + ErrorResponse(c, fmt.Sprintf("error parsing http request: %s", err.Error())) + return + } + + goQuery, err := StrQueryToMsgQuery(mreq.Query) + if err != nil { + ErrorResponse(c, err.Error()) + return + } + + checker, err := CheckerFromMessageQuery(goQuery) + if err != nil { + ErrorResponse(c, err.Error()) + return + } + + result := &checkRequestResponse{ + Success: true, + Result: checker(req), + } + + MessageResponse(c, result) +} + /* SetScope */ @@ -678,7 +736,7 @@ func setScopeHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Intercept return } - goQuery, err := StrQueryToGoQuery(mreq.Query) + goQuery, err := StrQueryToMsgQuery(mreq.Query) if err != nil { ErrorResponse(c, err.Error()) return @@ -719,7 +777,7 @@ func viewScopeHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Intercep } var err error - strQuery, err := GoQueryToStrQuery(scopeQuery) + strQuery, err := MsgQueryToStrQuery(scopeQuery) if err != nil { ErrorResponse(c, err.Error()) return @@ -859,7 +917,6 @@ func clearTagHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Intercept } req.ClearTags() - MainLogger.Println(req.Tags()) err = UpdateRequest(storage, req) if err != nil { ErrorResponse(c, fmt.Sprintf("error saving request: %s", err.Error())) @@ -1039,7 +1096,7 @@ func interceptHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Intercep return ret, nil } - reqSub = iproxy.AddReqIntSub(reqIntFunc) + reqSub = iproxy.AddReqInterceptor(reqIntFunc) } var rspSub *RspIntSub = nil @@ -1096,7 +1153,7 @@ func interceptHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Intercep } return ret, nil } - rspSub = iproxy.AddRspIntSub(rspIntFunc) + rspSub = iproxy.AddRspInterceptor(rspIntFunc) } var wsSub *WSIntSub = nil @@ -1163,23 +1220,23 @@ func interceptHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Intercep } return ret, nil } - wsSub = iproxy.AddWSIntSub(wsIntFunc) + wsSub = iproxy.AddWSInterceptor(wsIntFunc) } closeAll := func() { if reqSub != nil { // close req sub - iproxy.RemoveReqIntSub(reqSub) + iproxy.RemoveReqInterceptor(reqSub) } if rspSub != nil { // close rsp sub - iproxy.RemoveRspIntSub(rspSub) + iproxy.RemoveRspInterceptor(rspSub) } if wsSub != nil { // close websocket sub - iproxy.RemoveWSIntSub(wsSub) + iproxy.RemoveWSInterceptor(wsSub) } // Close all pending requests @@ -1298,7 +1355,7 @@ func allSavedQueriesHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *In Name: q.Name, Query: nil, } - sq, err := GoQueryToStrQuery(q.Query) + sq, err := MsgQueryToStrQuery(q.Query) if err == nil { strSavedQuery.Query = sq savedQueries = append(savedQueries, strSavedQuery) @@ -1340,7 +1397,7 @@ func saveQueryHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Intercep return } - goQuery, err := StrQueryToGoQuery(mreq.Query) + goQuery, err := StrQueryToMsgQuery(mreq.Query) if err != nil { ErrorResponse(c, err.Error()) return @@ -1399,7 +1456,7 @@ func loadQueryHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Intercep return } - strQuery, err := GoQueryToStrQuery(query) + strQuery, err := MsgQueryToStrQuery(query) if err != nil { ErrorResponse(c, err.Error()) return @@ -1458,6 +1515,11 @@ type activeListener struct { type addListenerMessage struct { Type string Addr string + + TransparentMode bool + DestHost string + DestPort int + DestUseTLS bool } type addListenerResult struct { @@ -1496,7 +1558,12 @@ func addListenerHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *Interc return } - iproxy.AddListener(listener) + if mreq.TransparentMode { + iproxy.AddTransparentListener(listener, mreq.DestHost, + mreq.DestPort, mreq.DestUseTLS) + } else { + iproxy.AddListener(listener) + } alistener := &activeListener{ Id: getNextMsgListenerId(), @@ -1581,13 +1648,12 @@ func loadCertificatesHandler(b []byte, c net.Conn, logger *log.Logger, iproxy *I return } - caCert, err := tls.LoadX509KeyPair(mreq.CertificateFile, mreq.KeyFile) + err := iproxy.LoadCACertificates(mreq.CertificateFile, mreq.KeyFile) if err != nil { ErrorResponse(c, err.Error()) return } - iproxy.SetCACertificate(&caCert) MessageResponse(c, &successResult{Success: true}) } @@ -1635,30 +1701,8 @@ func generateCertificatesHandler(b []byte, c net.Conn, logger *log.Logger, iprox return } - pair, err := GenerateCACerts() + _, err := GenerateCACertsToDisk(mreq.KeyFile, mreq.CertFile) if err != nil { - ErrorResponse(c, "error generating certificates: "+err.Error()) - return - } - - pkeyFile, err := os.OpenFile(mreq.KeyFile, os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - ErrorResponse(c, "could not save private key: "+err.Error()) - return - } - pkeyFile.Write(pair.PrivateKeyPEM()) - if err := pkeyFile.Close(); err != nil { - ErrorResponse(c, err.Error()) - return - } - - certFile, err := os.OpenFile(mreq.CertFile, os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - ErrorResponse(c, "could not save private key: "+err.Error()) - return - } - certFile.Write(pair.CACertPEM()) - if err := certFile.Close(); err != nil { ErrorResponse(c, err.Error()) return } diff --git a/python/puppy/.gitignore b/python/puppy/.gitignore deleted file mode 100644 index 319fcbf..0000000 --- a/python/puppy/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.egg-info -*.pyc -.DS_store diff --git a/python/puppy/puppyproxy/clip.py b/python/puppy/puppyproxy/clip.py deleted file mode 100644 index daceebb..0000000 --- a/python/puppy/puppyproxy/clip.py +++ /dev/null @@ -1,386 +0,0 @@ -""" -Copyright (c) 2014, Al Sweigart -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the {organization} nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" - -import contextlib -import ctypes -import os -import platform -import subprocess -import sys -import time - -from ctypes import c_size_t, sizeof, c_wchar_p, get_errno, c_wchar - -EXCEPT_MSG = """ - Pyperclip could not find a copy/paste mechanism for your system. - For more information, please visit https://pyperclip.readthedocs.org """ -PY2 = sys.version_info[0] == 2 -text_type = unicode if PY2 else str - -class PyperclipException(RuntimeError): - pass - - -class PyperclipWindowsException(PyperclipException): - def __init__(self, message): - message += " (%s)" % ctypes.WinError() - super(PyperclipWindowsException, self).__init__(message) - -def init_osx_clipboard(): - def copy_osx(text): - p = subprocess.Popen(['pbcopy', 'w'], - stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=text) - - def paste_osx(): - p = subprocess.Popen(['pbpaste', 'r'], - stdout=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode() - - return copy_osx, paste_osx - - -def init_gtk_clipboard(): - import gtk - - def copy_gtk(text): - global cb - cb = gtk.Clipboard() - cb.set_text(text) - cb.store() - - def paste_gtk(): - clipboardContents = gtk.Clipboard().wait_for_text() - # for python 2, returns None if the clipboard is blank. - if clipboardContents is None: - return '' - else: - return clipboardContents - - return copy_gtk, paste_gtk - - -def init_qt_clipboard(): - # $DISPLAY should exist - from PyQt4.QtGui import QApplication - - app = QApplication([]) - - def copy_qt(text): - cb = app.clipboard() - cb.setText(text) - - def paste_qt(): - cb = app.clipboard() - return text_type(cb.text()) - - return copy_qt, paste_qt - - -def init_xclip_clipboard(): - def copy_xclip(text): - p = subprocess.Popen(['xclip', '-selection', 'c'], - stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=text) - - def paste_xclip(): - p = subprocess.Popen(['xclip', '-selection', 'c', '-o'], - stdout=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode() - - return copy_xclip, paste_xclip - - -def init_xsel_clipboard(): - def copy_xsel(text): - p = subprocess.Popen(['xsel', '-b', '-i'], - stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=text) - - def paste_xsel(): - p = subprocess.Popen(['xsel', '-b', '-o'], - stdout=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode() - - return copy_xsel, paste_xsel - - -def init_klipper_clipboard(): - def copy_klipper(text): - p = subprocess.Popen( - ['qdbus', 'org.kde.klipper', '/klipper', 'setClipboardContents', - text], - stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=None) - - def paste_klipper(): - p = subprocess.Popen( - ['qdbus', 'org.kde.klipper', '/klipper', 'getClipboardContents'], - stdout=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - - # Workaround for https://bugs.kde.org/show_bug.cgi?id=342874 - # TODO: https://github.com/asweigart/pyperclip/issues/43 - clipboardContents = stdout.decode() - # even if blank, Klipper will append a newline at the end - assert len(clipboardContents) > 0 - # make sure that newline is there - assert clipboardContents.endswith('\n') - if clipboardContents.endswith('\n'): - clipboardContents = clipboardContents[:-1] - return clipboardContents - - return copy_klipper, paste_klipper - - -def init_no_clipboard(): - class ClipboardUnavailable(object): - def __call__(self, *args, **kwargs): - raise PyperclipException(EXCEPT_MSG) - - if PY2: - def __nonzero__(self): - return False - else: - def __bool__(self): - return False - - return ClipboardUnavailable(), ClipboardUnavailable() - -class CheckedCall(object): - def __init__(self, f): - super(CheckedCall, self).__setattr__("f", f) - - def __call__(self, *args): - ret = self.f(*args) - if not ret and get_errno(): - raise PyperclipWindowsException("Error calling " + self.f.__name__) - return ret - - def __setattr__(self, key, value): - setattr(self.f, key, value) - - -def init_windows_clipboard(): - from ctypes.wintypes import (HGLOBAL, LPVOID, DWORD, LPCSTR, INT, HWND, - HINSTANCE, HMENU, BOOL, UINT, HANDLE) - - windll = ctypes.windll - - safeCreateWindowExA = CheckedCall(windll.user32.CreateWindowExA) - safeCreateWindowExA.argtypes = [DWORD, LPCSTR, LPCSTR, DWORD, INT, INT, - INT, INT, HWND, HMENU, HINSTANCE, LPVOID] - safeCreateWindowExA.restype = HWND - - safeDestroyWindow = CheckedCall(windll.user32.DestroyWindow) - safeDestroyWindow.argtypes = [HWND] - safeDestroyWindow.restype = BOOL - - OpenClipboard = windll.user32.OpenClipboard - OpenClipboard.argtypes = [HWND] - OpenClipboard.restype = BOOL - - safeCloseClipboard = CheckedCall(windll.user32.CloseClipboard) - safeCloseClipboard.argtypes = [] - safeCloseClipboard.restype = BOOL - - safeEmptyClipboard = CheckedCall(windll.user32.EmptyClipboard) - safeEmptyClipboard.argtypes = [] - safeEmptyClipboard.restype = BOOL - - safeGetClipboardData = CheckedCall(windll.user32.GetClipboardData) - safeGetClipboardData.argtypes = [UINT] - safeGetClipboardData.restype = HANDLE - - safeSetClipboardData = CheckedCall(windll.user32.SetClipboardData) - safeSetClipboardData.argtypes = [UINT, HANDLE] - safeSetClipboardData.restype = HANDLE - - safeGlobalAlloc = CheckedCall(windll.kernel32.GlobalAlloc) - safeGlobalAlloc.argtypes = [UINT, c_size_t] - safeGlobalAlloc.restype = HGLOBAL - - safeGlobalLock = CheckedCall(windll.kernel32.GlobalLock) - safeGlobalLock.argtypes = [HGLOBAL] - safeGlobalLock.restype = LPVOID - - safeGlobalUnlock = CheckedCall(windll.kernel32.GlobalUnlock) - safeGlobalUnlock.argtypes = [HGLOBAL] - safeGlobalUnlock.restype = BOOL - - GMEM_MOVEABLE = 0x0002 - CF_UNICODETEXT = 13 - - @contextlib.contextmanager - def window(): - """ - Context that provides a valid Windows hwnd. - """ - # we really just need the hwnd, so setting "STATIC" - # as predefined lpClass is just fine. - hwnd = safeCreateWindowExA(0, b"STATIC", None, 0, 0, 0, 0, 0, - None, None, None, None) - try: - yield hwnd - finally: - safeDestroyWindow(hwnd) - - @contextlib.contextmanager - def clipboard(hwnd): - """ - Context manager that opens the clipboard and prevents - other applications from modifying the clipboard content. - """ - # We may not get the clipboard handle immediately because - # some other application is accessing it (?) - # We try for at least 500ms to get the clipboard. - t = time.time() + 0.5 - success = False - while time.time() < t: - success = OpenClipboard(hwnd) - if success: - break - time.sleep(0.01) - if not success: - raise PyperclipWindowsException("Error calling OpenClipboard") - - try: - yield - finally: - safeCloseClipboard() - - def copy_windows(text): - # This function is heavily based on - # http://msdn.com/ms649016#_win32_Copying_Information_to_the_Clipboard - with window() as hwnd: - # http://msdn.com/ms649048 - # If an application calls OpenClipboard with hwnd set to NULL, - # EmptyClipboard sets the clipboard owner to NULL; - # this causes SetClipboardData to fail. - # => We need a valid hwnd to copy something. - with clipboard(hwnd): - safeEmptyClipboard() - - if text: - # http://msdn.com/ms649051 - # If the hMem parameter identifies a memory object, - # the object must have been allocated using the - # function with the GMEM_MOVEABLE flag. - count = len(text) + 1 - handle = safeGlobalAlloc(GMEM_MOVEABLE, - count * sizeof(c_wchar)) - locked_handle = safeGlobalLock(handle) - - ctypes.memmove(c_wchar_p(locked_handle), c_wchar_p(text), count * sizeof(c_wchar)) - - safeGlobalUnlock(handle) - safeSetClipboardData(CF_UNICODETEXT, handle) - - def paste_windows(): - with clipboard(None): - handle = safeGetClipboardData(CF_UNICODETEXT) - if not handle: - # GetClipboardData may return NULL with errno == NO_ERROR - # if the clipboard is empty. - # (Also, it may return a handle to an empty buffer, - # but technically that's not empty) - return "" - return c_wchar_p(handle).value - - return copy_windows, paste_windows - -# `import PyQt4` sys.exit()s if DISPLAY is not in the environment. -# Thus, we need to detect the presence of $DISPLAY manually -# and not load PyQt4 if it is absent. -HAS_DISPLAY = os.getenv("DISPLAY", False) -CHECK_CMD = "where" if platform.system() == "Windows" else "which" - - -def _executable_exists(name): - return subprocess.call([CHECK_CMD, name], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 - - -def determine_clipboard(): - # Determine the OS/platform and set - # the copy() and paste() functions accordingly. - if 'cygwin' in platform.system().lower(): - # FIXME: pyperclip currently does not support Cygwin, - # see https://github.com/asweigart/pyperclip/issues/55 - pass - elif os.name == 'nt' or platform.system() == 'Windows': - return init_windows_clipboard() - if os.name == 'mac' or platform.system() == 'Darwin': - return init_osx_clipboard() - if HAS_DISPLAY: - # Determine which command/module is installed, if any. - try: - import gtk # check if gtk is installed - except ImportError: - pass - else: - return init_gtk_clipboard() - - try: - import PyQt4 # check if PyQt4 is installed - except ImportError: - pass - else: - return init_qt_clipboard() - - if _executable_exists("xclip"): - return init_xclip_clipboard() - if _executable_exists("xsel"): - return init_xsel_clipboard() - if _executable_exists("klipper") and _executable_exists("qdbus"): - return init_klipper_clipboard() - - return init_no_clipboard() - - -def set_clipboard(clipboard): - global copy, paste - - clipboard_types = {'osx': init_osx_clipboard, - 'gtk': init_gtk_clipboard, - 'qt': init_qt_clipboard, - 'xclip': init_xclip_clipboard, - 'xsel': init_xsel_clipboard, - 'klipper': init_klipper_clipboard, - 'windows': init_windows_clipboard, - 'no': init_no_clipboard} - - copy, paste = clipboard_types[clipboard]() - - -copy, paste = determine_clipboard() diff --git a/python/puppy/puppyproxy/colors.py b/python/puppy/puppyproxy/colors.py deleted file mode 100644 index c09f1b1..0000000 --- a/python/puppy/puppyproxy/colors.py +++ /dev/null @@ -1,197 +0,0 @@ -import re -import itertools - -from pygments import highlight -from pygments.lexers.data import JsonLexer -from pygments.lexers.html import XmlLexer -from pygments.lexers import get_lexer_for_mimetype, HttpLexer -from pygments.formatters import TerminalFormatter - -def clen(s): - ansi_escape = re.compile(r'\x1b[^m]*m') - return len(ansi_escape.sub('', s)) - -class Colors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - # Effects - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - - # Colors - BLACK = '\033[30m' - RED = '\033[31m' - GREEN = '\033[32m' - YELLOW = '\033[33m' - BLUE = '\033[34m' - MAGENTA = '\033[35m' - CYAN = '\033[36m' - WHITE = '\033[37m' - - # BG Colors - BGBLACK = '\033[40m' - BGRED = '\033[41m' - BGGREEN = '\033[42m' - BGYELLOW = '\033[43m' - BGBLUE = '\033[44m' - BGMAGENTA = '\033[45m' - BGCYAN = '\033[46m' - BGWHITE = '\033[47m' - - # Light Colors - LBLACK = '\033[90m' - LRED = '\033[91m' - LGREEN = '\033[92m' - LYELLOW = '\033[93m' - LBLUE = '\033[94m' - LMAGENTA = '\033[95m' - LCYAN = '\033[96m' - LWHITE = '\033[97m' - -class Styles: - - ################ - # Request tables - TABLE_HEADER = Colors.BOLD+Colors.UNDERLINE - VERB_GET = Colors.CYAN - VERB_POST = Colors.YELLOW - VERB_OTHER = Colors.BLUE - STATUS_200 = Colors.CYAN - STATUS_300 = Colors.MAGENTA - STATUS_400 = Colors.YELLOW - STATUS_500 = Colors.RED - PATH_COLORS = [Colors.CYAN, Colors.BLUE] - - KV_KEY = Colors.GREEN - KV_VAL = Colors.ENDC - - UNPRINTABLE_DATA = Colors.CYAN - - -def verb_color(verb): - if verb and verb == 'GET': - return Styles.VERB_GET - elif verb and verb == 'POST': - return Styles.VERB_POST - else: - return Styles.VERB_OTHER - -def scode_color(scode): - if scode and scode[0] == '2': - return Styles.STATUS_200 - elif scode and scode[0] == '3': - return Styles.STATUS_300 - elif scode and scode[0] == '4': - return Styles.STATUS_400 - elif scode and scode[0] == '5': - return Styles.STATUS_500 - else: - return Colors.ENDC - -def path_formatter(path, width=-1): - if len(path) > width and width != -1: - path = path[:width] - path = path[:-3]+'...' - parts = path.split('/') - colparts = [] - for p, c in zip(parts, itertools.cycle(Styles.PATH_COLORS)): - colparts.append(c+p+Colors.ENDC) - return '/'.join(colparts) - -def color_string(s, color_only=False): - """ - Return the string with a a color/ENDC. The same string will always be the same color. - """ - from .util import str_hash_code - # Give each unique host a different color (ish) - if not s: - return "" - strcols = [Colors.RED, - Colors.GREEN, - Colors.YELLOW, - Colors.BLUE, - Colors.MAGENTA, - Colors.CYAN, - Colors.LRED, - Colors.LGREEN, - Colors.LYELLOW, - Colors.LBLUE, - Colors.LMAGENTA, - Colors.LCYAN] - col = strcols[str_hash_code(s)%(len(strcols)-1)] - if color_only: - return col - else: - return col + s + Colors.ENDC - -def pretty_msg(msg): - to_ret = pretty_headers(msg) + '\r\n' + pretty_body(msg) - return to_ret - -def pretty_headers(msg): - to_ret = msg.headers_section() - to_ret = highlight(to_ret, HttpLexer(), TerminalFormatter()) - return to_ret - -def pretty_body(msg): - from .util import printable_data - to_ret = printable_data(msg.body, colors=False) - if 'content-type' in msg.headers: - try: - lexer = get_lexer_for_mimetype(msg.headers.get('content-type').split(';')[0]) - to_ret = highlight(to_ret, lexer, TerminalFormatter()) - except: - pass - return to_ret - -def url_formatter(req, colored=False, always_have_path=False, explicit_path=False, explicit_port=False): - retstr = '' - - if not req.use_tls: - if colored: - retstr += Colors.RED - retstr += 'http' - if colored: - retstr += Colors.ENDC - retstr += '://' - else: - retstr += 'https://' - - if colored: - retstr += color_string(req.dest_host) - else: - retstr += req.dest_host - if not ((req.use_tls and req.dest_port == 443) or \ - (not req.use_tls and req.dest_port == 80) or \ - explicit_port): - if colored: - retstr += ':' - retstr += Colors.MAGENTA - retstr += str(req.dest_port) - retstr += Colors.ENDC - else: - retstr += ':{}'.format(req.dest_port) - if (req.url.path and req.url.path != '/') or always_have_path: - if colored: - retstr += path_formatter(req.url.path) - else: - retstr += req.url.path - if req.url.params: - retstr += '?' - params = req.url.params.split("&") - pairs = [tuple(param.split("=")) for param in params] - paramstrs = [] - for k, v in pairs: - if colored: - paramstrs += (Colors.GREEN + '{}' + Colors.ENDC + '=' + Colors.LGREEN + '{}' + Colors.ENDC).format(k, v) - else: - paramstrs += '{}={}'.format(k, v) - retstr += '&'.join(paramstrs) - if req.url.fragment: - retstr += '#%s' % req.url.fragment - return retstr - diff --git a/python/puppy/puppyproxy/config.py b/python/puppy/puppyproxy/config.py deleted file mode 100644 index 3eb395d..0000000 --- a/python/puppy/puppyproxy/config.py +++ /dev/null @@ -1,119 +0,0 @@ -import copy -import json - -default_config = """{ - "listeners": [ - {"iface": "127.0.0.1", "port": 8080} - ], - "proxy": {"use_proxy": false, "host": "", "port": 0, "is_socks": false} -}""" - - -class ProxyConfig: - - def __init__(self): - self._listeners = [('127.0.0.1', '8080')] - self._proxy = {'use_proxy': False, 'host': '', 'port': 0, 'is_socks': False} - - def load(self, fname): - try: - with open(fname, 'r') as f: - config_info = json.loads(f.read()) - except IOError: - config_info = json.loads(default_config) - with open(fname, 'w') as f: - f.write(default_config) - - # Listeners - if 'listeners' in config_info: - self._listeners = [] - for info in config_info['listeners']: - if 'port' in info: - port = info['port'] - else: - port = 8080 - - if 'interface' in info: - iface = info['interface'] - elif 'iface' in info: - iface = info['iface'] - else: - iface = '127.0.0.1' - - self._listeners.append((iface, port)) - - if 'proxy' in config_info: - self._proxy = config_info['proxy'] - - - @property - def listeners(self): - return copy.deepcopy(self._listeners) - - @listeners.setter - def listeners(self, val): - self._listeners = val - - @property - def proxy(self): - # don't use this, use the getters to get the parsed values - return self._proxy - - @proxy.setter - def proxy(self, val): - self._proxy = val - - @property - def use_proxy(self): - if self._proxy is None: - return False - if 'use_proxy' in self._proxy: - if self._proxy['use_proxy']: - return True - return False - - @property - def proxy_host(self): - if self._proxy is None: - return '' - if 'host' in self._proxy: - return self._proxy['host'] - return '' - - @property - def proxy_port(self): - if self._proxy is None: - return '' - if 'port' in self._proxy: - return self._proxy['port'] - return '' - - @property - def proxy_username(self): - if self._proxy is None: - return '' - if 'username' in self._proxy: - return self._proxy['username'] - return '' - - @property - def proxy_password(self): - if self._proxy is None: - return '' - if 'password' in self._proxy: - return self._proxy['password'] - return '' - - @property - def use_proxy_creds(self): - return ('username' in self._proxy or 'password' in self._proxy) - - @property - def is_socks_proxy(self): - if self._proxy is None: - return False - if 'is_socks' in self._proxy: - if self._proxy['is_socks']: - return True - return False - diff --git a/python/puppy/puppyproxy/console.py b/python/puppy/puppyproxy/console.py deleted file mode 100644 index f7df899..0000000 --- a/python/puppy/puppyproxy/console.py +++ /dev/null @@ -1,203 +0,0 @@ -""" -Contains helpers for interacting with the console. Includes definition for the -class that is used to run the console. -""" - -import atexit -import cmd2 -import os -import readline -#import string -import shlex -import sys - -from .colors import Colors -from .proxy import MessageError - -################### -## Helper Functions - -def print_errors(func): - def catch(*args, **kwargs): - try: - func(*args, **kwargs) - except (CommandError, MessageError) as e: - print(str(e)) - return catch - -def interface_loop(client): - cons = Cmd(client=client) - load_interface(cons) - sys.argv = [] - cons.cmdloop() - -def load_interface(cons): - from .interface import test, view, decode, misc, context, mangle, macros, tags - test.load_cmds(cons) - view.load_cmds(cons) - decode.load_cmds(cons) - misc.load_cmds(cons) - context.load_cmds(cons) - mangle.load_cmds(cons) - macros.load_cmds(cons) - tags.load_cmds(cons) - -########## -## Classes - -class SessionEnd(Exception): - pass - -class CommandError(Exception): - pass - -class Cmd(cmd2.Cmd): - """ - An object representing the console interface. Provides methods to add - commands and aliases to the console. Implemented as a hack around cmd2.Cmd - """ - - def __init__(self, *args, **kwargs): - # the \x01/\x02 are to make the prompt behave properly with the readline library - self.prompt = 'puppy\x01' + Colors.YELLOW + '\x02> \x01' + Colors.ENDC + '\x02' - self.debug = True - self.histsize = 0 - if 'histsize' in kwargs: - self.histsize = kwargs['histsize'] - del kwargs['histsize'] - if 'client' not in kwargs: - raise Exception("client argument is required") - self.client = kwargs['client'] - del kwargs['client'] - - self._cmds = {} - self._aliases = {} - - atexit.register(self.save_histfile) - readline.set_history_length(self.histsize) - if os.path.exists('cmdhistory'): - if self.histsize != 0: - readline.read_history_file('cmdhistory') - else: - os.remove('cmdhistory') - - cmd2.Cmd.__init__(self, *args, **kwargs) - - - def __dir__(self): - # Hack to get cmd2 to detect that we can run a command - ret = set(dir(self.__class__)) - ret.update(self.__dict__.keys()) - ret.update(['do_'+k for k in self._cmds.keys()]) - ret.update(['help_'+k for k in self._cmds.keys()]) - ret.update(['complete_'+k for k, v in self._cmds.items() if self._cmds[k][1]]) - for k, v in self._aliases.items(): - ret.add('do_' + k) - ret.add('help_' + k) - if self._cmds[self._aliases[k]][1]: - ret.add('complete_'+k) - return sorted(ret) - - def __getattr__(self, attr): - def gen_helpfunc(func): - def f(): - if not func.__doc__: - to_print = 'No help exists for function' - else: - lines = func.__doc__.splitlines() - if len(lines) > 0 and lines[0] == '': - lines = lines[1:] - if len(lines) > 0 and lines[-1] == '': - lines = lines[-1:] - to_print = '\n'.join(l.lstrip() for l in lines) - - aliases = set() - aliases.add(attr[5:]) - for i in range(2): - for k, v in self._aliases.items(): - if k in aliases or v in aliases: - aliases.add(k) - aliases.add(v) - to_print += '\nAliases: ' + ', '.join(aliases) - print(to_print) - return f - - def gen_dofunc(func, client): - def f(line): - args = shlex.split(line) - func(client, args) - return print_errors(f) - - if attr.startswith('do_'): - command = attr[3:] - if command in self._cmds: - return gen_dofunc(self._cmds[command][0], self.client) - elif command in self._aliases: - real_command = self._aliases[command] - if real_command in self._cmds: - return gen_dofunc(self._cmds[real_command][0], self.client) - elif attr.startswith('help_'): - command = attr[5:] - if command in self._cmds: - return gen_helpfunc(self._cmds[command][0]) - elif command in self._aliases: - real_command = self._aliases[command] - if real_command in self._cmds: - return gen_helpfunc(self._cmds[real_command][0]) - elif attr.startswith('complete_'): - command = attr[9:] - if command in self._cmds: - if self._cmds[command][1]: - return self._cmds[command][1] - elif command in self._aliases: - real_command = self._aliases[command] - if real_command in self._cmds: - if self._cmds[real_command][1]: - return self._cmds[real_command][1] - raise AttributeError(attr) - - def save_histfile(self): - # Write the command to the history file - if self.histsize != 0: - readline.set_history_length(self.histsize) - readline.write_history_file('cmdhistory') - - def get_names(self): - # Hack to get cmd to recognize do_/etc functions as functions for things - # like autocomplete - return dir(self) - - def set_cmd(self, command, func, autocomplete_func=None): - """ - Add a command to the console. - """ - self._cmds[command] = (func, autocomplete_func) - - def set_cmds(self, cmd_dict): - """ - Set multiple commands from a dictionary. Format is: - {'command': (do_func, autocomplete_func)} - Use autocomplete_func=None for no autocomplete function - """ - for command, vals in cmd_dict.items(): - do_func, ac_func = vals - self.set_cmd(command, do_func, ac_func) - - def add_alias(self, command, alias): - """ - Add an alias for a command. - ie add_alias("foo", "f") will let you run the 'foo' command with 'f' - """ - if command not in self._cmds: - raise KeyError() - self._aliases[alias] = command - - def add_aliases(self, alias_list): - """ - Pass in a list of tuples to add them all as aliases. - ie add_aliases([('foo', 'f'), ('foo', 'fo')]) will add 'f' and 'fo' as - aliases for 'foo' - """ - for command, alias in alias_list: - self.add_alias(command, alias) - diff --git a/python/puppy/puppyproxy/interface/context.py b/python/puppy/puppyproxy/interface/context.py deleted file mode 100644 index d16eed3..0000000 --- a/python/puppy/puppyproxy/interface/context.py +++ /dev/null @@ -1,245 +0,0 @@ -from itertools import groupby - -from ..proxy import InvalidQuery, time_to_nsecs -from ..colors import Colors, Styles - -# class BuiltinFilters(object): -# _filters = { -# 'not_image': ( -# ['path nctr "(\.png$|\.jpg$|\.gif$)"'], -# 'Filter out image requests', -# ), -# 'not_jscss': ( -# ['path nctr "(\.js$|\.css$)"'], -# 'Filter out javascript and css files', -# ), -# } - -# @staticmethod -# @defer.inlineCallbacks -# def get(name): -# if name not in BuiltinFilters._filters: -# raise PappyException('%s not a bult in filter' % name) -# if name in BuiltinFilters._filters: -# filters = [pappyproxy.context.Filter(f) for f in BuiltinFilters._filters[name][0]] -# for f in filters: -# yield f.generate() -# defer.returnValue(filters) -# raise PappyException('"%s" is not a built-in filter' % name) - -# @staticmethod -# def list(): -# return [k for k, v in BuiltinFilters._filters.iteritems()] - -# @staticmethod -# def help(name): -# if name not in BuiltinFilters._filters: -# raise PappyException('"%s" is not a built-in filter' % name) -# return pappyproxy.context.Filter(BuiltinFilters._filters[name][1]) - - -# def complete_filtercmd(text, line, begidx, endidx): -# strs = [k for k, v in pappyproxy.context.Filter._filter_functions.iteritems()] -# strs += [k for k, v in pappyproxy.context.Filter._async_filter_functions.iteritems()] -# return autocomplete_startswith(text, strs) - -# def complete_builtin_filter(text, line, begidx, endidx): -# all_names = BuiltinFilters.list() -# if not text: -# ret = all_names[:] -# else: -# ret = [n for n in all_names if n.startswith(text)] -# return ret - -# @crochet.wait_for(timeout=None) -# @defer.inlineCallbacks -# def builtin_filter(line): -# if not line: -# raise PappyException("Filter name required") - -# filters_to_add = yield BuiltinFilters.get(line) -# for f in filters_to_add: -# print f.filter_string -# yield pappyproxy.pappy.main_context.add_filter(f) -# defer.returnValue(None) - -def filtercmd(client, args): - """ - Apply a filter to the current context - Usage: filter - See README.md for information on filter strings - """ - try: - phrases = [list(group) for k, group in groupby(args, lambda x: x == "OR") if not k] - for phrase in phrases: - # we do before/after by id not by timestamp - if phrase[0] in ('before', 'b4', 'after', 'af') and len(phrase) > 1: - r = client.req_by_id(phrase[1], headers_only=True) - phrase[1] = str(time_to_nsecs(r.time_start)) - client.context.apply_phrase(phrases) - except InvalidQuery as e: - print(e) - -def filter_up(client, args): - """ - Remove the last applied filter - Usage: filter_up - """ - client.context.pop_phrase() - -def filter_clear(client, args): - """ - Reset the context so that it contains no filters (ignores scope) - Usage: filter_clear - """ - client.context.set_query([]) - -def filter_list(client, args): - """ - Print the filters that make up the current context - Usage: filter_list - """ - from ..util import print_query - print_query(client.context.query) - -def scope_save(client, args): - """ - Set the scope to be the current context. Saved between launches - Usage: scope_save - """ - client.set_scope(client.context.query) - -def scope_reset(client, args): - """ - Set the context to be the scope (view in-scope items) - Usage: scope_reset - """ - result = client.get_scope() - if result.is_custom: - print("Proxy is using a custom function to check scope. Cannot set context to scope.") - return - client.context.set_query(result.filter) - -def scope_delete(client, args): - """ - Delete the scope so that it contains all request/response pairs - Usage: scope_delete - """ - client.set_scope([]) - -def scope_list(client, args): - """ - Print the filters that make up the scope - Usage: scope_list - """ - from ..util import print_query - result = client.get_scope() - if result.is_custom: - print("Proxy is using a custom function to check scope") - return - print_query(result.filter) - -def list_saved_queries(client, args): - from ..util import print_query - queries = client.all_saved_queries() - print('') - for q in queries: - print(Styles.TABLE_HEADER + q.name + Colors.ENDC) - print_query(q.query) - print('') - -def save_query(client, args): - from ..util import print_query - if len(args) != 1: - print("Must give name to save filters as") - return - client.save_query(args[0], client.context.query) - print('') - print(Styles.TABLE_HEADER + args[0] + Colors.ENDC) - print_query(client.context.query) - print('') - -def load_query(client, args): - from ..util import print_query - if len(args) != 1: - print("Must give name of query to load") - return - new_query = client.load_query(args[0]) - client.context.set_query(new_query) - print('') - print(Styles.TABLE_HEADER + args[0] + Colors.ENDC) - print_query(new_query) - print('') - -def delete_query(client, args): - if len(args) != 1: - print("Must give name of filter") - return - client.delete_query(args[0]) - -# @crochet.wait_for(timeout=None) -# @defer.inlineCallbacks -# def filter_prune(line): -# """ -# Delete all out of context requests from the data file. -# CANNOT BE UNDONE!! Be careful! -# Usage: filter_prune -# """ -# # Delete filtered items from datafile -# print '' -# print 'Currently active filters:' -# for f in pappyproxy.pappy.main_context.active_filters: -# print '> %s' % f.filter_string - -# # We copy so that we're not removing items from a set we're iterating over -# act_reqs = yield pappyproxy.pappy.main_context.get_reqs() -# inact_reqs = set(Request.cache.req_ids()).difference(set(act_reqs)) -# message = 'This will delete %d/%d requests. You can NOT undo this!! Continue?' % (len(inact_reqs), (len(inact_reqs) + len(act_reqs))) -# #print message -# if not confirm(message, 'n'): -# defer.returnValue(None) - -# for reqid in inact_reqs: -# try: -# req = yield pappyproxy.http.Request.load_request(reqid) -# yield req.deep_delete() -# except PappyException as e: -# print e -# print 'Deleted %d requests' % len(inact_reqs) -# defer.returnValue(None) - -############### -## Plugin hooks - -def load_cmds(cmd): - cmd.set_cmds({ - #'filter': (filtercmd, complete_filtercmd), - 'filter': (filtercmd, None), - 'filter_up': (filter_up, None), - 'filter_list': (filter_list, None), - 'filter_clear': (filter_clear, None), - 'scope_list': (scope_list, None), - 'scope_delete': (scope_delete, None), - 'scope_reset': (scope_reset, None), - 'scope_save': (scope_save, None), - 'list_saved_queries': (list_saved_queries, None), - # 'filter_prune': (filter_prune, None), - # 'builtin_filter': (builtin_filter, complete_builtin_filter), - 'save_query': (save_query, None), - 'load_query': (load_query, None), - 'delete_query': (delete_query, None), - }) - cmd.add_aliases([ - ('filter', 'f'), - ('filter', 'fl'), - ('filter_up', 'fu'), - ('filter_list', 'fls'), - ('filter_clear', 'fc'), - ('scope_list', 'sls'), - ('scope_reset', 'sr'), - ('list_saved_queries', 'sqls'), - # ('builtin_filter', 'fbi'), - ('save_query', 'sq'), - ('load_query', 'lq'), - ('delete_query', 'dq'), - ]) diff --git a/python/puppy/puppyproxy/interface/decode.py b/python/puppy/puppyproxy/interface/decode.py deleted file mode 100644 index 786ff39..0000000 --- a/python/puppy/puppyproxy/interface/decode.py +++ /dev/null @@ -1,326 +0,0 @@ -import html -import base64 -import datetime -import gzip -import shlex -import string -import urllib - -from ..util import hexdump, printable_data, copy_to_clipboard, clipboard_contents, encode_basic_auth, parse_basic_auth -from ..console import CommandError -from io import StringIO - -def print_maybe_bin(s): - binary = False - for c in s: - if chr(c) not in string.printable: - binary = True - break - if binary: - print(hexdump(s)) - else: - print(s.decode()) - -def asciihex_encode_helper(s): - return ''.join('{0:x}'.format(c) for c in s).encode() - -def asciihex_decode_helper(s): - ret = [] - try: - for a, b in zip(s[0::2], s[1::2]): - c = chr(a)+chr(b) - ret.append(chr(int(c, 16))) - return ''.join(ret).encode() - except Exception as e: - raise CommandError(e) - -def gzip_encode_helper(s): - out = StringIO.StringIO() - with gzip.GzipFile(fileobj=out, mode="w") as f: - f.write(s) - return out.getvalue() - -def gzip_decode_helper(s): - dec_data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(s)) - dec_data = dec_data.read() - return dec_data - -def base64_decode_helper(s): - try: - return base64.b64decode(s) - except TypeError: - for i in range(1, 5): - try: - s_padded = base64.b64decode(s + '='*i) - return s_padded - except: - pass - raise CommandError("Unable to base64 decode string") - -def url_decode_helper(s): - bs = s.decode() - return urllib.parse.unquote(bs).encode() - -def url_encode_helper(s): - bs = s.decode() - return urllib.parse.quote_plus(bs).encode() - -def html_encode_helper(s): - return ''.join(['&#x{0:x};'.format(c) for c in s]).encode() - -def html_decode_helper(s): - return html.unescape(s.decode()).encode() - -def _code_helper(args, func, copy=True): - if len(args) == 0: - s = clipboard_contents().encode() - print('Will decode:') - print(printable_data(s)) - s = func(s) - if copy: - try: - copy_to_clipboard(s) - except Exception as e: - print('Result cannot be copied to the clipboard. Result not copied.') - raise e - return s - else: - s = func(args[0].encode()) - if copy: - try: - copy_to_clipboard(s) - except Exception as e: - print('Result cannot be copied to the clipboard. Result not copied.') - raise e - return s - -def base64_decode(client, args): - """ - Base64 decode a string. - If no string is given, will decode the contents of the clipboard. - Results are copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, base64_decode_helper)) - -def base64_encode(client, args): - """ - Base64 encode a string. - If no string is given, will encode the contents of the clipboard. - Results are copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, base64.b64encode)) - -def url_decode(client, args): - """ - URL decode a string. - If no string is given, will decode the contents of the clipboard. - Results are copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, url_decode_helper)) - -def url_encode(client, args): - """ - URL encode special characters in a string. - If no string is given, will encode the contents of the clipboard. - Results are copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, url_encode_helper)) - -def asciihex_decode(client, args): - """ - Decode an ascii hex string. - If no string is given, will decode the contents of the clipboard. - Results are copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, asciihex_decode_helper)) - -def asciihex_encode(client, args): - """ - Convert all the characters in a line to hex and combine them. - If no string is given, will encode the contents of the clipboard. - Results are copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, asciihex_encode_helper)) - -def html_decode(client, args): - """ - Decode an html encoded string. - If no string is given, will decode the contents of the clipboard. - Results are copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, html_decode_helper)) - -def html_encode(client, args): - """ - Encode a string and escape html control characters. - If no string is given, will encode the contents of the clipboard. - Results are copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, html_encode_helper)) - -def gzip_decode(client, args): - """ - Un-gzip a string. - If no string is given, will decompress the contents of the clipboard. - Results are copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, gzip_decode_helper)) - -def gzip_encode(client, args): - """ - Gzip a string. - If no string is given, will decompress the contents of the clipboard. - Results are NOT copied to the clipboard. - """ - print_maybe_bin(_code_helper(args, gzip_encode_helper, copy=False)) - -def base64_decode_raw(client, args): - """ - Same as base64_decode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, base64_decode_helper, copy=False)) - -def base64_encode_raw(client, args): - """ - Same as base64_encode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, base64.b64encode, copy=False)) - -def url_decode_raw(client, args): - """ - Same as url_decode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, url_decode_helper, copy=False)) - -def url_encode_raw(client, args): - """ - Same as url_encode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, url_encode_helper, copy=False)) - -def asciihex_decode_raw(client, args): - """ - Same as asciihex_decode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, asciihex_decode_helper, copy=False)) - -def asciihex_encode_raw(client, args): - """ - Same as asciihex_encode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, asciihex_encode_helper, copy=False)) - -def html_decode_raw(client, args): - """ - Same as html_decode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, html_decode_helper, copy=False)) - -def html_encode_raw(client, args): - """ - Same as html_encode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, html_encode_helper, copy=False)) - -def gzip_decode_raw(client, args): - """ - Same as gzip_decode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, gzip_decode_helper, copy=False)) - -def gzip_encode_raw(client, args): - """ - Same as gzip_encode but the output will never be printed as a hex dump and - results will not be copied. It is suggested you redirect the output - to a file. - """ - print(_code_helper(args, gzip_encode_helper, copy=False)) - -def unix_time_decode_helper(line): - unix_time = int(line.strip()) - dtime = datetime.datetime.fromtimestamp(unix_time) - return dtime.strftime('%Y-%m-%d %H:%M:%S') - -def unix_time_decode(client, args): - print(_code_helper(args, unix_time_decode_helper)) - -def http_auth_encode(client, args): - if len(args) != 2: - raise CommandError('Usage: http_auth_encode ') - username, password = args - print(encode_basic_auth(username, password)) - -def http_auth_decode(client, args): - username, password = decode_basic_auth(args[0]) - print(username) - print(password) - -def load_cmds(cmd): - cmd.set_cmds({ - 'base64_decode': (base64_decode, None), - 'base64_encode': (base64_encode, None), - 'asciihex_decode': (asciihex_decode, None), - 'asciihex_encode': (asciihex_encode, None), - 'url_decode': (url_decode, None), - 'url_encode': (url_encode, None), - 'html_decode': (html_decode, None), - 'html_encode': (html_encode, None), - 'gzip_decode': (gzip_decode, None), - 'gzip_encode': (gzip_encode, None), - 'base64_decode_raw': (base64_decode_raw, None), - 'base64_encode_raw': (base64_encode_raw, None), - 'asciihex_decode_raw': (asciihex_decode_raw, None), - 'asciihex_encode_raw': (asciihex_encode_raw, None), - 'url_decode_raw': (url_decode_raw, None), - 'url_encode_raw': (url_encode_raw, None), - 'html_decode_raw': (html_decode_raw, None), - 'html_encode_raw': (html_encode_raw, None), - 'gzip_decode_raw': (gzip_decode_raw, None), - 'gzip_encode_raw': (gzip_encode_raw, None), - 'unixtime_decode': (unix_time_decode, None), - 'httpauth_encode': (http_auth_encode, None), - 'httpauth_decode': (http_auth_decode, None) - }) - cmd.add_aliases([ - ('base64_decode', 'b64d'), - ('base64_encode', 'b64e'), - ('asciihex_decode', 'ahd'), - ('asciihex_encode', 'ahe'), - ('url_decode', 'urld'), - ('url_encode', 'urle'), - ('html_decode', 'htmld'), - ('html_encode', 'htmle'), - ('gzip_decode', 'gzd'), - ('gzip_encode', 'gze'), - ('base64_decode_raw', 'b64dr'), - ('base64_encode_raw', 'b64er'), - ('asciihex_decode_raw', 'ahdr'), - ('asciihex_encode_raw', 'aher'), - ('url_decode_raw', 'urldr'), - ('url_encode_raw', 'urler'), - ('html_decode_raw', 'htmldr'), - ('html_encode_raw', 'htmler'), - ('gzip_decode_raw', 'gzdr'), - ('gzip_encode_raw', 'gzer'), - ('unixtime_decode', 'uxtd'), - ('httpauth_encode', 'hae'), - ('httpauth_decode', 'had'), - ]) diff --git a/python/puppy/puppyproxy/interface/macros.py b/python/puppy/puppyproxy/interface/macros.py deleted file mode 100644 index f87829d..0000000 --- a/python/puppy/puppyproxy/interface/macros.py +++ /dev/null @@ -1,61 +0,0 @@ -from ..macros import macro_from_requests, MacroTemplate, load_macros - -macro_dict = {} - -def generate_macro(client, args): - if len(args) == 0: - print("usage: gma [name] [reqids]") - return - macro_name = args[0] - - reqs = [] - if len(args) > 1: - ids = args[1].split(',') - for reqid in ids: - req = client.req_by_id(reqid) - reqs.append(req) - - script_string = macro_from_requests(reqs) - fname = MacroTemplate.template_filename('macro', macro_name) - with open(fname, 'w') as f: - f.write(script_string) - print("Macro written to {}".format(fname)) - -def load_macros_cmd(client, args): - global macro_dict - - load_dir = '.' - if len(args) > 0: - load_dir = args[0] - - loaded_macros, loaded_int_macros = load_macros(load_dir) - for macro in loaded_macros: - macro_dict[macro.name] = macro - print("Loaded {} ({})".format(macro.name, macro.file_name)) - -def complete_run_macro(text, line, begidx, endidx): - from ..util import autocomplete_starts_with - - global macro_dict - strs = [k for k,v in macro_dict.iteritems()] - return autocomplete_startswith(text, strs) - -def run_macro(client, args): - global macro_dict - if len(args) == 0: - print("usage: rma [macro name]") - return - macro = macro_dict[args[0]] - macro.execute(client, args[1:]) - -def load_cmds(cmd): - cmd.set_cmds({ - 'generate_macro': (generate_macro, None), - 'load_macros': (load_macros_cmd, None), - 'run_macro': (run_macro, complete_run_macro), - }) - cmd.add_aliases([ - ('generate_macro', 'gma'), - ('load_macros', 'lma'), - ('run_macro', 'rma'), - ]) diff --git a/python/puppy/puppyproxy/interface/mangle.py b/python/puppy/puppyproxy/interface/mangle.py deleted file mode 100644 index c652039..0000000 --- a/python/puppy/puppyproxy/interface/mangle.py +++ /dev/null @@ -1,325 +0,0 @@ -import curses -import os -import subprocess -import tempfile -import threading -from ..macros import InterceptMacro -from ..proxy import MessageError, parse_request, parse_response -from ..colors import url_formatter - -edit_queue = [] - -class InterceptorMacro(InterceptMacro): - """ - A class representing a macro that modifies requests as they pass through the - proxy - """ - def __init__(self): - InterceptMacro.__init__(self) - self.name = "InterceptorMacro" - - def mangle_request(self, request): - # This function gets called to mangle/edit requests passed through the proxy - - # Write original request to the temp file - with tempfile.NamedTemporaryFile(delete=False) as tf: - tfName = tf.name - tf.write(request.full_message()) - - mangled_req = request - front = False - while True: - # Have the console edit the file - event = edit_file(tfName, front=front) - event.wait() - if event.canceled: - return request - - # Create new mangled request from edited file - with open(tfName, 'rb') as f: - text = f.read() - - os.remove(tfName) - - # Check if dropped - if text == '': - return None - - try: - mangled_req = parse_request(text) - except MessageError as e: - print("could not parse request: %s" % str(e)) - front = True - continue - mangled_req.dest_host = request.dest_host - mangled_req.dest_port = request.dest_port - mangled_req.use_tls = request.use_tls - break - return mangled_req - - def mangle_response(self, request, response): - # This function gets called to mangle/edit respones passed through the proxy - - # Write original response to the temp file - with tempfile.NamedTemporaryFile(delete=False) as tf: - tfName = tf.name - tf.write(response.full_message()) - - mangled_rsp = response - while True: - # Have the console edit the file - event = edit_file(tfName, front=True) - event.wait() - if event.canceled: - return response - - # Create new mangled response from edited file - with open(tfName, 'rb') as f: - text = f.read() - - os.remove(tfName) - - # Check if dropped - if text == '': - return None - - try: - mangled_rsp = parse_response(text) - except MessageError as e: - print("could not parse response: %s" % str(e)) - front = True - continue - break - return mangled_rsp - - def mangle_websocket(self, request, response, message): - # This function gets called to mangle/edit respones passed through the proxy - - # Write original response to the temp file - with tempfile.NamedTemporaryFile(delete=False) as tf: - tfName = tf.name - tf.write(b"# ") - if message.to_server: - tf.write(b"OUTGOING to") - else: - tf.write(b"INCOMING from") - desturl = 'ws' + url_formatter(request)[4:] # replace http:// with ws:// - tf.write(b' ' + desturl.encode()) - tf.write(b" -- Note that this line is ignored\n") - tf.write(message.message) - - mangled_msg = message - while True: - # Have the console edit the file - event = edit_file(tfName, front=True) - event.wait() - if event.canceled: - return message - - # Create new mangled response from edited file - with open(tfName, 'rb') as f: - text = f.read() - _, text = text.split(b'\n', 1) - - os.remove(tfName) - - # Check if dropped - if text == '': - return None - - mangled_msg.message = text - # if messages can be invalid, check for it here and continue if invalid - break - return mangled_msg - - -class EditEvent: - - def __init__(self): - self.e = threading.Event() - self.canceled = False - - def wait(self): - self.e.wait() - - def set(self): - self.e.set() - - def cancel(self): - self.canceled = True - self.set() - -############### -## Helper funcs - -def edit_file(fname, front=False): - global edit_queue - # Adds the filename to the edit queue. Returns an event that is set once - # the file is edited and the editor is closed - #e = threading.Event() - e = EditEvent() - if front: - edit_queue = [(fname, e, threading.current_thread())] + edit_queue - else: - edit_queue.append((fname, e, threading.current_thread())) - return e - -def execute_repeater(client, reqid): - #script_loc = os.path.join(pappy.session.config.pappy_dir, "plugins", "vim_repeater", "repeater.vim") - maddr = client.maddr - if maddr is None: - print("Client has no message address, cannot run repeater") - return - storage, reqid = client.parse_reqid(reqid) - script_loc = os.path.join(os.path.dirname(os.path.realpath(__file__)), - "repeater", "repeater.vim") - args = (["vim", "-S", script_loc, "-c", "RepeaterSetup %s %s %s"%(reqid, storage.storage_id, client.maddr)]) - subprocess.call(args) - -class CloudToButt(InterceptMacro): - - def __init__(self): - InterceptMacro.__init__(self) - self.name = 'cloudtobutt' - self.intercept_requests = True - self.intercept_responses = True - self.intercept_ws = True - - def mangle_response(self, request, response): - response.body = response.body.replace(b"cloud", b"butt") - response.body = response.body.replace(b"Cloud", b"Butt") - return response - - def mangle_request(self, request): - request.body = request.body.replace(b"foo", b"bar") - request.body = request.body.replace(b"Foo", b"Bar") - return request - - def mangle_websocket(self, request, response, wsm): - wsm.message = wsm.message.replace(b"world", b"zawarudo") - wsm.message = wsm.message.replace(b"zawarudo", b"ZAWARUDO") - return wsm - -def repeater(client, args): - """ - Open a request in the repeater - Usage: repeater - """ - # This is not async on purpose. start_editor acts up if this is called - # with inline callbacks. As a result, check_reqid and get_unmangled - # cannot be async - reqid = args[0] - req = client.req_by_id(reqid) - execute_repeater(client, reqid) - -def intercept(client, args): - """ - Intercept requests and/or responses and edit them with before passing them along - Usage: intercept - """ - global edit_queue - - req_names = ('req', 'request', 'requests') - rsp_names = ('rsp', 'response', 'responses') - ws_names = ('ws', 'websocket') - - mangle_macro = InterceptorMacro() - if any(a in req_names for a in args): - mangle_macro.intercept_requests = True - if any(a in rsp_names for a in args): - mangle_macro.intercept_responses = True - if any(a in ws_names for a in args): - mangle_macro.intercept_ws = True - if not args: - mangle_macro.intercept_requests = True - - intercepting = [] - if mangle_macro.intercept_requests: - intercepting.append('Requests') - if mangle_macro.intercept_responses: - intercepting.append('Responses') - if mangle_macro.intercept_ws: - intercepting.append('Websocket Messages') - if not mangle_macro.intercept_requests and not mangle_macro.intercept_responses and not mangle_macro.intercept_ws: - intercept_str = 'NOTHING WHY ARE YOU DOING THIS' # WHYYYYYYYY - else: - intercept_str = ', '.join(intercepting) - - ## Interceptor loop - stdscr = curses.initscr() - curses.noecho() - curses.cbreak() - stdscr.nodelay(True) - - conn = client.new_conn() - try: - conn.intercept(mangle_macro) - editnext = False - while True: - stdscr.addstr(0, 0, "Currently intercepting: %s" % intercept_str) - stdscr.clrtoeol() - stdscr.addstr(1, 0, "%d item(s) in queue." % len(edit_queue)) - stdscr.clrtoeol() - if editnext: - stdscr.addstr(2, 0, "Waiting for next item... Press 'q' to quit or 'b' to quit waiting") - else: - stdscr.addstr(2, 0, "Press 'n' to edit the next item or 'q' to quit interceptor.") - stdscr.clrtoeol() - - c = stdscr.getch() - if c == ord('q'): - return - elif c == ord('n'): - editnext = True - elif c == ord('b'): - editnext = False - - if editnext and edit_queue: - editnext = False - (to_edit, event, t) = edit_queue.pop(0) - editor = 'vi' - if 'EDITOR' in os.environ: - editor = os.environ['EDITOR'] - additional_args = [] - if editor == 'vim': - # prevent adding additional newline - additional_args.append('-b') - subprocess.call([editor, to_edit] + additional_args) - stdscr.clear() - event.set() - t.join() - finally: - conn.close() - # Now that the connection is closed, make sure the rest of the threads finish/error out - while len(edit_queue) > 0: - (fname, event, t) = edit_queue.pop(0) - event.cancel() - t.join() - curses.nocbreak() - stdscr.keypad(0) - curses.echo() - curses.endwin() - -############### -## Plugin hooks - -def test_macro(client, args): - c2b = CloudToButt() - conn = client.new_conn() - with client.new_conn() as conn: - conn.intercept(c2b) - print("intercept started") - input("Press enter to quit...") - print("past raw input") - -def load_cmds(cmd): - cmd.set_cmds({ - 'intercept': (intercept, None), - 'c2b': (test_macro, None), - 'repeater': (repeater, None), - }) - cmd.add_aliases([ - ('intercept', 'ic'), - ('repeater', 'rp'), - ]) - diff --git a/python/puppy/puppyproxy/interface/misc.py b/python/puppy/puppyproxy/interface/misc.py deleted file mode 100644 index b926825..0000000 --- a/python/puppy/puppyproxy/interface/misc.py +++ /dev/null @@ -1,172 +0,0 @@ -import argparse -import sys -from ..util import copy_to_clipboard, confirm, printable_data -from ..console import CommandError -from ..proxy import InterceptMacro -from ..colors import url_formatter, verb_color, Colors, scode_color - -class WatchMacro(InterceptMacro): - - def __init__(self): - InterceptMacro.__init__(self) - self.name = "WatchMacro" - - def mangle_request(self, request): - printstr = "> " - printstr += verb_color(request.method) + request.method + Colors.ENDC + " " - printstr += url_formatter(request, colored=True) - print(printstr) - - return request - - def mangle_response(self, request, response): - printstr = "< " - printstr += verb_color(request.method) + request.method + Colors.ENDC + ' ' - printstr += url_formatter(request, colored=True) - printstr += " \u2192 " - response_code = str(response.status_code) + ' ' + response.reason - response_code = scode_color(response_code) + response_code + Colors.ENDC - printstr += response_code - print(printstr) - - return response - - def mangle_websocket(self, request, response, message): - printstr = "" - if message.to_server: - printstr += ">" - else: - printstr += "<" - printstr += "ws(b={}) ".format(message.is_binary) - printstr += printable_data(message.message) - print(printstr) - - return message - -def message_address(client, args): - msg_addr = client.maddr - if msg_addr is None: - print("Client has no message address") - return - print(msg_addr) - if len(args) > 0 and args[0] == "-c": - try: - copy_to_clipboard(msg_addr.encode()) - print("Copied to clipboard!") - except: - print("Could not copy address to clipboard") - -def cpinmem(client, args): - req = client.req_by_id(args[0]) - client.save_new(req, client.inmem_storage.storage_id) - -def ping(client, args): - print(client.ping()) - -def watch(client, args): - macro = WatchMacro() - macro.intercept_requests = True - macro.intercept_responses = True - macro.intercept_ws = True - - with client.new_conn() as conn: - conn.intercept(macro) - print("Watching requests. Press to quit...") - input() - -def submit(client, cargs): - """ - Resubmit some requests, optionally with modified headers and cookies. - - Usage: submit [-h] [-m] [-u] [-p] [-o REQID] [-c [COOKIES [COOKIES ...]]] [-d [HEADERS [HEADERS ...]]] - """ - #Usage: submit reqids [-h] [-m] [-u] [-p] [-o REQID] [-c [COOKIES [COOKIES ...]]] [-d [HEADERS [HEADERS ...]]] - - parser = argparse.ArgumentParser(prog="submit", usage=submit.__doc__) - #parser.add_argument('reqids') - parser.add_argument('-m', '--inmem', action='store_true', help='Store resubmitted requests in memory without storing them in the data file') - parser.add_argument('-u', '--unique', action='store_true', help='Only resubmit one request per endpoint (different URL parameters are different endpoints)') - parser.add_argument('-p', '--uniquepath', action='store_true', help='Only resubmit one request per endpoint (ignoring URL parameters)') - parser.add_argument('-c', '--cookies', nargs='*', help='Apply a cookie to requests before submitting') - parser.add_argument('-d', '--headers', nargs='*', help='Apply a header to requests before submitting') - parser.add_argument('-o', '--copycookies', help='Copy the cookies used in another request') - args = parser.parse_args(cargs) - - headers = {} - cookies = {} - clear_cookies = False - - if args.headers: - for h in args.headers: - k, v = h.split('=', 1) - headers[k] = v - - if args.copycookies: - reqid = args.copycookies - req = client.req_by_id(reqid) - clear_cookies = True - for k, v in req.cookie_iter(): - cookies[k] = v - - if args.cookies: - for c in args.cookies: - k, v = c.split('=', 1) - cookies[k] = v - - if args.unique and args.uniquepath: - raise CommandError('Both -u and -p cannot be given as arguments') - - # Get requests to submit - #reqs = [r.copy() for r in client.in_context_requests()] - reqs = client.in_context_requests() - - # Apply cookies and headers - for req in reqs: - if clear_cookies: - req.headers.delete("Cookie") - for k, v in cookies.items(): - req.set_cookie(k, v) - for k, v in headers.items(): - req.headers.set(k, v) - - conf_message = "You're about to submit %d requests, continue?" % len(reqs) - if not confirm(conf_message): - return - - # Filter unique paths - if args.uniquepath or args.unique: - endpoints = set() - new_reqs = [] - for r in reqs: - if unique_path_and_args: - s = r.url.geturl() - else: - s = r.url.geturl(include_params=False) - - if not s in endpoints: - new_reqs.append(r) - endpoints.add(s) - reqs = new_reqs - - # Tag and send them - for req in reqs: - req.tags.add('resubmitted') - sys.stdout.write(client.prefixed_reqid(req) + " ") - sys.stdout.flush() - - storage = client.disk_storage.storage_id - if args.inmem: - storage = client.inmem_storage.storage_id - - client.submit(req, storage=storage) - sys.stdout.write("\n") - sys.stdout.flush() - -def load_cmds(cmd): - cmd.set_cmds({ - 'maddr': (message_address, None), - 'ping': (ping, None), - 'submit': (submit, None), - 'cpim': (cpinmem, None), - 'watch': (watch, None), - }) diff --git a/python/puppy/puppyproxy/interface/repeater/__init__.py b/python/puppy/puppyproxy/interface/repeater/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/puppy/puppyproxy/interface/repeater/repeater.py b/python/puppy/puppyproxy/interface/repeater/repeater.py deleted file mode 100644 index 9daab53..0000000 --- a/python/puppy/puppyproxy/interface/repeater/repeater.py +++ /dev/null @@ -1,1607 +0,0 @@ -import base64 -import copy -import datetime -import json -import math -import re -import shlex -import socket -import sys -import vim -import threading - -from collections import namedtuple -from urlparse import urlparse, ParseResult, parse_qs -from urllib import urlencode -import Cookie as hcookies - -## STRIPPED DOWN COPY OF HTTP OBJECTS / COMMS - -class MessageError(Exception): - pass - - -class ProxyException(Exception): - pass - - -class InvalidQuery(Exception): - pass - -class SocketClosed(Exception): - pass - -class SockBuffer: - # I can't believe I have to implement this - - def __init__(self, sock): - self.buf = [] # a list of chunks of strings - self.s = sock - self.closed = False - - def close(self): - self.s.shutdown(socket.SHUT_RDWR) - self.s.close() - self.closed = True - - def _check_newline(self): - for chunk in self.buf: - if '\n' in chunk: - return True - return False - - def readline(self): - # Receive until we get a newline, raise SocketClosed if socket is closed - while True: - try: - data = self.s.recv(8192) - except OSError: - raise SocketClosed() - if not data: - raise SocketClosed() - self.buf.append(data) - if b'\n' in data: - break - - # Combine chunks - retbytes = bytes() - n = 0 - for chunk in self.buf: - n += 1 - if b'\n' in chunk: - head, tail = chunk.split(b'\n', 1) - retbytes += head - self.buf = self.buf[n:] - self.buf = [tail] + self.buf - break - else: - retbytes += chunk - return retbytes.decode() - - def send(self, data): - try: - self.s.send(data) - except OSError: - raise SocketClosed() - -class Headers: - def __init__(self, headers=None): - if headers is None: - self.headers = {} - else: - self.headers = headers - - def __contains__(self, hd): - for k, _ in self.headers.items(): - if k.lower() == hd.lower(): - return True - return False - - def add(self, k, v): - try: - l = self.headers[k.lower()] - l.append((k,v)) - except KeyError: - self.headers[k.lower()] = [(k,v)] - - def set(self, k, v): - self.headers[k.lower()] = [(k,v)] - - def get(self, k): - return self.headers[k.lower()][0][1] - - def delete(self, k): - del self.headers[k.lower()] - - def pairs(self, key=None): - for _, kvs in self.headers.items(): - for k, v in kvs: - if key is None or k.lower() == key.lower(): - yield (k, v) - - def dict(self): - retdict = {} - for _, kvs in self.headers.items(): - for k, v in kvs: - if k in retdict: - retdict[k].append(v) - else: - retdict[k] = [v] - return retdict - -class RequestContext: - def __init__(self, client, query=None): - self._current_query = [] - self.client = client - if query is not None: - self._current_query = query - - def _validate(self, query): - self.client.validate_query(query) - - def set_query(self, query): - self._validate(query) - self._current_query = query - - def apply_phrase(self, phrase): - self._validate([phrase]) - self._current_query.append(phrase) - - def pop_phrase(self): - if len(self._current_query) > 0: - self._current_query.pop() - - def apply_filter(self, filt): - self._validate([[filt]]) - self._current_query.append([filt]) - - @property - def query(self): - return copy.deepcopy(self._current_query) - - -class URL: - def __init__(self, url): - parsed = urlparse(url) - if url is not None: - parsed = urlparse(url) - self.scheme = parsed.scheme - self.netloc = parsed.netloc - self.path = parsed.path - self.params = parsed.params - self.query = parsed.query - self.fragment = parsed.fragment - else: - self.scheme = "" - self.netloc = "" - self.path = "/" - self.params = "" - self.query = "" - self.fragment = "" - - def geturl(self, include_params=True): - params = self.params - query = self.query - fragment = self.fragment - - if not include_params: - params = "" - query = "" - fragment = "" - - r = ParseResult(scheme=self.scheme, - netloc=self.netloc, - path=self.path, - params=params, - query=query, - fragment=fragment) - return r.geturl() - - def parameters(self): - try: - return parse_qs(self.query, keep_blank_values=True) - except Exception: - return [] - - def param_iter(self): - for k, vs in self.parameters().items(): - for v in vs: - yield k, v - - def set_param(self, key, val): - params = self.parameters() - params[key] = [val] - self.query = urlencode(params) - - def add_param(self, key, val): - params = self.parameters() - if key in params: - params[key].append(val) - else: - params[key] = [val] - self.query = urlencode(params) - - def del_param(self, key): - params = self.parameters() - del params[key] - self.query = urlencode(params) - - def set_params(self, params): - self.query = urlencode(params) - - -class InterceptMacro: - """ - A class representing a macro that modifies requests as they pass through the - proxy - """ - - def __init__(self): - self.name = '' - self.intercept_requests = False - self.intercept_responses = False - self.intercept_ws = False - - def __repr__(self): - return "" % self.name - - def mangle_request(self, request): - return request - - def mangle_response(self, request, response): - return response - - def mangle_websocket(self, request, response, message): - return message - - -class HTTPRequest: - def __init__(self, method="GET", path="/", proto_major=1, proto_minor=1, - headers=None, body=bytes(), dest_host="", dest_port=80, - use_tls=False, time_start=None, time_end=None, db_id="", - tags=None, headers_only=False, storage_id=0): - # http info - self.method = method - self.url = URL(path) - self.proto_major = proto_major - self.proto_minor = proto_minor - - self.headers = Headers() - if headers is not None: - for k, vs in headers.items(): - for v in vs: - self.headers.add(k, v) - - self.headers_only = headers_only - self._body = bytes() - if not headers_only: - self.body = body - - # metadata - self.dest_host = dest_host - self.dest_port = dest_port - self.use_tls = use_tls - self.time_start = time_start or datetime.datetime(1970, 1, 1) - self.time_end = time_end or datetime.datetime(1970, 1, 1) - - self.response = None - self.unmangled = None - self.ws_messages = [] - - self.db_id = db_id - self.storage_id = storage_id - if tags is not None: - self.tags = set(tags) - else: - self.tags = set() - - @property - def body(self): - return self._body - - @body.setter - def body(self, bs): - self.headers_only = False - if type(bs) is str: - self._body = bs.encode() - elif type(bs) is bytes: - self._body = bs - else: - raise Exception("invalid body type: {}".format(type(bs))) - self.headers.set("Content-Length", str(len(self._body))) - - @property - def content_length(self): - if 'content-length' in self.headers: - return int(self.headers.get('content-length')) - return len(self.body) - - def status_line(self): - sline = "{method} {path} HTTP/{proto_major}.{proto_minor}".format( - method=self.method, path=self.url.geturl(), proto_major=self.proto_major, - proto_minor=self.proto_minor).encode() - return sline - - def headers_section(self): - message = self.status_line() + b"\r\n" - for k, v in self.headers.pairs(): - message += "{}: {}\r\n".format(k, v).encode() - return message - - def full_message(self): - message = self.headers_section() - message += b"\r\n" - message += self.body - return message - - def parameters(self): - try: - return parse_qs(self.body.decode(), keep_blank_values=True) - except Exception: - return [] - - def param_iter(self, ignore_content_type=False): - if not ignore_content_type: - if "content-type" not in self.headers: - return - if "www-form-urlencoded" not in self.headers.get("content-type").lower(): - return - for k, vs in self.parameters().items(): - for v in vs: - yield k, v - - def set_param(self, key, val): - params = self.parameters() - params[key] = [val] - self.body = urlencode(params) - - def add_param(self, key, val): - params = self.parameters() - if key in params: - params[key].append(val) - else: - params[key] = [val] - self.body = urlencode(params) - - def del_param(self, key): - params = self.parameters() - del params[key] - self.body = urlencode(params) - - def set_params(self, params): - self.body = urlencode(params) - - def cookies(self): - try: - cookie = hcookies.BaseCookie() - cookie.load(self.headers.get("cookie")) - return cookie - except Exception as e: - return hcookies.BaseCookie() - - def cookie_iter(self): - c = self.cookies() - for k in c: - yield k, c[k].value - - def set_cookie(self, key, val): - c = self.cookies() - c[key] = val - self.set_cookies(c) - - def del_cookie(self, key): - c = self.cookies() - del c[key] - self.set_cookies(c) - - def set_cookies(self, c): - cookie_pairs = [] - if isinstance(c, hcookies.BaseCookie()): - # it's a basecookie - for k in c: - cookie_pairs.append('{}={}'.format(k, c[k].value)) - else: - # it's a dictionary - for k, v in c.items(): - cookie_pairs.append('{}={}'.format(k, v)) - header_str = '; '.join(cookie_pairs) - self.headers.set("Cookie", header_str) - - def copy(self): - return HTTPRequest( - method=self.method, - path=self.url.geturl(), - proto_major=self.proto_major, - proto_minor=self.proto_minor, - headers=self.headers.headers, - body=self.body, - dest_host=self.dest_host, - dest_port=self.dest_port, - use_tls=self.use_tls, - tags=copy.deepcopy(self.tags), - headers_only=self.headers_only, - ) - - -class HTTPResponse: - def __init__(self, status_code=200, reason="OK", proto_major=1, proto_minor=1, - headers=None, body=bytes(), db_id="", headers_only=False): - self.status_code = status_code - self.reason = reason - self.proto_major = proto_major - self.proto_minor = proto_minor - - self.headers = Headers() - if headers is not None: - for k, vs in headers.items(): - for v in vs: - self.headers.add(k, v) - - self.headers_only = headers_only - self._body = bytes() - if not headers_only: - self.body = body - - self.unmangled = None - self.db_id = db_id - - @property - def body(self): - return self._body - - @body.setter - def body(self, bs): - self.headers_only = False - if type(bs) is str: - self._body = bs.encode() - elif type(bs) is bytes: - self._body = bs - else: - raise Exception("invalid body type: {}".format(type(bs))) - self.headers.set("Content-Length", str(len(self._body))) - - @property - def content_length(self): - if 'content-length' in self.headers: - return int(self.headers.get('content-length')) - return len(self.body) - - def status_line(self): - sline = "HTTP/{proto_major}.{proto_minor} {status_code} {reason}".format( - proto_major=self.proto_major, proto_minor=self.proto_minor, - status_code=self.status_code, reason=self.reason).encode() - return sline - - def headers_section(self): - message = self.status_line() + b"\r\n" - for k, v in self.headers.pairs(): - message += "{}: {}\r\n".format(k, v).encode() - return message - - def full_message(self): - message = self.headers_section() - message += b"\r\n" - message += self.body - return message - - def cookies(self): - try: - cookie = hcookies.BaseCookie() - for _, v in self.headers.pairs('set-cookie'): - cookie.load(v) - return cookie - except Exception as e: - return hcookies.BaseCookie() - - def cookie_iter(self): - c = self.cookies() - for k in c: - yield k, c[k].value - - def set_cookie(self, key, val): - c = self.cookies() - c[key] = val - self.set_cookies(c) - - def del_cookie(self, key): - c = self.cookies() - del c[key] - self.set_cookies(c) - - def set_cookies(self, c): - self.headers.delete("set-cookie") - if isinstance(c, hcookies.BaseCookie): - cookies = c - else: - cookies = hcookies.BaseCookie() - for k, v in c.items(): - cookies[k] = v - for _, m in c.items(): - self.headers.add("Set-Cookie", m.OutputString()) - - def copy(self): - return HTTPResponse( - status_code=self.status_code, - reason=self.reason, - proto_major=self.proto_major, - proto_minor=self.proto_minor, - headers=self.headers.headers, - body=self.body, - headers_only=self.headers_only, - ) - -class WSMessage: - def __init__(self, is_binary=True, message=bytes(), to_server=True, - timestamp=None, db_id=""): - self.is_binary = is_binary - self.message = message - self.to_server = to_server - self.timestamp = timestamp or datetime.datetime(1970, 1, 1) - - self.unmangled = None - self.db_id = db_id - - def copy(self): - return WSMessage( - is_binary=self.is_binary, - message=self.message, - to_server=self.to_server, - ) - -ScopeResult = namedtuple("ScopeResult", ["is_custom", "filter"]) -ListenerResult = namedtuple("ListenerResult", ["lid", "addr"]) -GenPemCertsResult = namedtuple("GenPemCertsResult", ["key_pem", "cert_pem"]) -SavedQuery = namedtuple("SavedQuery", ["name", "query"]) -SavedStorage = namedtuple("SavedStorage", ["storage_id", "description"]) - -def messagingFunction(func): - def f(self, *args, **kwargs): - if self.is_interactive: - raise MessageError("cannot be called while other message is interactive") - if self.closed: - raise MessageError("connection is closed") - return func(self, *args, **kwargs) - return f - -class ProxyConnection: - next_id = 1 - def __init__(self, kind="", addr=""): - self.connid = ProxyConnection.next_id - ProxyConnection.next_id += 1 - self.sbuf = None - self.buf = bytes() - self.parent_client = None - self.debug = False - self.is_interactive = False - self.closed = True - self.sock_lock_read = threading.Lock() - self.sock_lock_write = threading.Lock() - self.kind = None - self.addr = None - - if kind.lower() == "tcp": - tcpaddr, port = addr.rsplit(":", 1) - self.connect_tcp(tcpaddr, int(port)) - elif kind.lower() == "unix": - self.connect_unix(addr) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def connect_tcp(self, addr, port): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((addr, port)) - self.sbuf = SockBuffer(s) - self.closed = False - self.kind = "tcp" - self.addr = "{}:{}".format(addr, port) - - def connect_unix(self, addr): - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - s.connect(addr) - self.sbuf = SockBuffer(s) - self.closed = False - self.kind = "unix" - self.addr = addr - - @property - def maddr(self): - if self.kind is not None: - return "{}:{}".format(self.kind, self.addr) - else: - return None - - def close(self): - self.sbuf.close() - if self.parent_client is not None: - self.parent_client.conns.remove(self) - self.closed = True - - def read_message(self): - with self.sock_lock_read: - l = self.sbuf.readline() - if self.debug: - print("<({}) {}".format(self.connid, l)) - j = json.loads(l) - if "Success" in j and j["Success"] == False: - if "Reason" in j: - raise MessageError(j["Reason"]) - raise MessageError("unknown error") - return j - - def submit_command(self, cmd): - with self.sock_lock_write: - ln = json.dumps(cmd).encode()+b"\n" - if self.debug: - print(">({}) {} ".format(self.connid, ln.decode())) - self.sbuf.send(ln) - - def reqrsp_cmd(self, cmd): - self.submit_command(cmd) - ret = self.read_message() - if ret is None: - raise Exception() - return ret - - ########### - ## Commands - - @messagingFunction - def ping(self): - cmd = {"Command": "Ping"} - result = self.reqrsp_cmd(cmd) - return result["Ping"] - - @messagingFunction - def submit(self, req, storage=None): - cmd = { - "Command": "Submit", - "Request": encode_req(req), - "Storage": 0, - } - if storage is not None: - cmd["Storage"] = storage - result = self.reqrsp_cmd(cmd) - if "SubmittedRequest" not in result: - raise MessageError("no request returned") - req = decode_req(result["SubmittedRequest"]) - req.storage_id = storage - return req - - @messagingFunction - def save_new(self, req, storage): - reqd = encode_req(req) - cmd = { - "Command": "SaveNew", - "Request": encode_req(req), - "Storage": storage, - } - result = self.reqrsp_cmd(cmd) - req.db_id = result["DbId"] - req.storage_id = storage - return result["DbId"] - - def _query_storage(self, q, storage, headers_only=False, max_results=0): - cmd = { - "Command": "StorageQuery", - "Query": q, - "HeadersOnly": headers_only, - "MaxResults": max_results, - "Storage": storage, - } - result = self.reqrsp_cmd(cmd) - reqs = [] - for reqd in result["Results"]: - req = decode_req(reqd, headers_only=headers_only) - req.storage_id = storage - reqs.append(req) - return reqs - - @messagingFunction - def query_storage(self, q, storage, max_results=0, headers_only=False): - return self._query_storage(q, storage, headers_only=headers_only, max_results=max_results) - - @messagingFunction - def req_by_id(self, reqid, storage, headers_only=False): - results = self._query_storage([[["dbid", "is", reqid]]], storage, - headers_only=headers_only, max_results=1) - if len(results) == 0: - raise MessageError("request with id {} does not exist".format(reqid)) - return results[0] - - @messagingFunction - def set_scope(self, filt): - cmd = { - "Command": "SetScope", - "Query": filt, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def get_scope(self): - cmd = { - "Command": "ViewScope", - } - result = self.reqrsp_cmd(cmd) - ret = ScopeResult(result["IsCustom"], result["Query"]) - return ret - - @messagingFunction - def add_tag(self, reqid, tag, storage): - cmd = { - "Command": "AddTag", - "ReqId": reqid, - "Tag": tag, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def remove_tag(self, reqid, tag, storage): - cmd = { - "Command": "RemoveTag", - "ReqId": reqid, - "Tag": tag, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def clear_tag(self, reqid, storage): - cmd = { - "Command": "ClearTag", - "ReqId": reqid, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def all_saved_queries(self, storage): - cmd = { - "Command": "AllSavedQueries", - "Storage": storage, - } - results = self.reqrsp_cmd(cmd) - queries = [] - for result in results["Queries"]: - queries.append(SavedQuery(name=result["Name"], query=result["Query"])) - return queries - - @messagingFunction - def save_query(self, name, filt, storage): - cmd = { - "Command": "SaveQuery", - "Name": name, - "Query": filt, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def load_query(self, name, storage): - cmd = { - "Command": "LoadQuery", - "Name": name, - "Storage": storage, - } - result = self.reqrsp_cmd(cmd) - return result["Query"] - - @messagingFunction - def delete_query(self, name, storage): - cmd = { - "Command": "DeleteQuery", - "Name": name, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def add_listener(self, addr, port): - laddr = "{}:{}".format(addr, port) - cmd = { - "Command": "AddListener", - "Type": "tcp", - "Addr": laddr, - } - result = self.reqrsp_cmd(cmd) - lid = result["Id"] - return lid - - @messagingFunction - def remove_listener(self, lid): - cmd = { - "Command": "RemoveListener", - "Id": lid, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def get_listeners(self): - cmd = { - "Command": "GetListeners", - } - result = self.reqrsp_cmd(cmd) - results = [] - for r in result["Results"]: - results.append(r["Id"], r["Addr"]) - return results - - @messagingFunction - def load_certificates(self, pkey_file, cert_file): - cmd = { - "Command": "LoadCerts", - "KeyFile": pkey_file, - "CertificateFile": cert_file, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def set_certificates(self, pkey_pem, cert_pem): - cmd = { - "Command": "SetCerts", - "KeyPEMData": pkey_pem, - "CertificatePEMData": cert_pem, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def clear_certificates(self): - cmd = { - "Command": "ClearCerts", - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def generate_certificates(self, pkey_file, cert_file): - cmd = { - "Command": "GenCerts", - "KeyFile": pkey_file, - "CertFile": cert_file, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def generate_pem_certificates(self): - cmd = { - "Command": "GenPEMCerts", - } - result = self.reqrsp_cmd(cmd) - ret = GenPemCertsResult(result["KeyPEMData"], result["CertificatePEMData"]) - return ret - - @messagingFunction - def validate_query(self, query): - cmd = { - "Command": "ValidateQuery", - "Query": query, - } - try: - result = self.reqrsp_cmd(cmd) - except MessageError as e: - raise InvalidQuery(str(e)) - - @messagingFunction - def add_sqlite_storage(self, path, desc): - cmd = { - "Command": "AddSQLiteStorage", - "Path": path, - "Description": desc - } - result = self.reqrsp_cmd(cmd) - return result["StorageId"] - - @messagingFunction - def add_in_memory_storage(self, desc): - cmd = { - "Command": "AddInMemoryStorage", - "Description": desc - } - result = self.reqrsp_cmd(cmd) - return result["StorageId"] - - @messagingFunction - def close_storage(self, strage_id): - cmd = { - "Command": "CloseStorage", - "StorageId": storage_id, - } - result = self.reqrsp_cmd(cmd) - - @messagingFunction - def set_proxy_storage(self, storage_id): - cmd = { - "Command": "SetProxyStorage", - "StorageId": storage_id, - } - result = self.reqrsp_cmd(cmd) - - @messagingFunction - def list_storage(self): - cmd = { - "Command": "ListStorage", - } - result = self.reqrsp_cmd(cmd) - ret = [] - for ss in result["Storages"]: - ret.append(SavedStorage(ss["Id"], ss["Description"])) - return ret - - @messagingFunction - def intercept(self, macro): - # Run an intercepting macro until closed - - from .util import log_error - # Start intercepting - self.is_interactive = True - cmd = { - "Command": "Intercept", - "InterceptRequests": macro.intercept_requests, - "InterceptResponses": macro.intercept_responses, - "InterceptWS": macro.intercept_ws, - } - try: - self.reqrsp_cmd(cmd) - except Exception as e: - self.is_interactive = False - raise e - - def run_macro(): - while True: - try: - msg = self.read_message() - except MessageError as e: - log_error(str(e)) - return - except SocketClosed: - return - - def mangle_and_respond(msg): - retCmd = None - if msg["Type"] == "httprequest": - req = decode_req(msg["Request"]) - newReq = macro.mangle_request(req) - - if newReq is None: - retCmd = { - "Id": msg["Id"], - "Dropped": True, - } - else: - newReq.unmangled = None - newReq.response = None - newReq.ws_messages = [] - - retCmd = { - "Id": msg["Id"], - "Dropped": False, - "Request": encode_req(newReq), - } - elif msg["Type"] == "httpresponse": - req = decode_req(msg["Request"]) - rsp = decode_rsp(msg["Response"]) - newRsp = macro.mangle_response(req, rsp) - - if newRsp is None: - retCmd = { - "Id": msg["Id"], - "Dropped": True, - } - else: - newRsp.unmangled = None - - retCmd = { - "Id": msg["Id"], - "Dropped": False, - "Response": encode_rsp(newRsp), - } - elif msg["Type"] == "wstoserver" or msg["Type"] == "wstoclient": - req = decode_req(msg["Request"]) - rsp = decode_rsp(msg["Response"]) - wsm = decode_ws(msg["WSMessage"]) - newWsm = macro.mangle_websocket(req, rsp, wsm) - - if newWsm is None: - retCmd = { - "Id": msg["Id"], - "Dropped": True, - } - else: - newWsm.unmangled = None - - retCmd = { - "Id": msg["Id"], - "Dropped": False, - "WSMessage": encode_ws(newWsm), - } - else: - raise Exception("Unknown message type: " + msg["Type"]) - if retCmd is not None: - try: - self.submit_command(retCmd) - except SocketClosed: - return - - mangle_thread = threading.Thread(target=mangle_and_respond, - args=(msg,)) - mangle_thread.start() - - self.int_thread = threading.Thread(target=run_macro) - self.int_thread.start() - - -ActiveStorage = namedtuple("ActiveStorage", ["type", "storage_id", "prefix"]) - -def _serialize_storage(stype, prefix): - return "{}|{}".format(stype, prefix) - -class ProxyClient: - def __init__(self, binary=None, debug=False, conn_addr=None): - self.binloc = binary - self.proxy_proc = None - self.ltype = None - self.laddr = None - self.debug = debug - self.conn_addr = conn_addr - - self.conns = set() - self.msg_conn = None # conn for single req/rsp messages - - self.context = RequestContext(self) - - self.storage_by_id = {} - self.storage_by_prefix = {} - self.proxy_storage = None - - self.reqrsp_methods = { - "submit_command", - #"reqrsp_cmd", - "ping", - #"submit", - #"save_new", - #"query_storage", - #"req_by_id", - "set_scope", - "get_scope", - # "add_tag", - # "remove_tag", - # "clear_tag", - "all_saved_queries", - "save_query", - "load_query", - "delete_query", - "add_listener", - "remove_listener", - "get_listeners", - "load_certificates", - "set_certificates", - "clear_certificates", - "generate_certificates", - "generate_pem_certificates", - "validate_query", - "list_storage", - # "add_sqlite_storage", - # "add_in_memory_storage", - # "close_storage", - # "set_proxy_storage", - } - - def __enter__(self): - if self.conn_addr is not None: - self.msg_connect(self.conn_addr) - else: - self.execute_binary(binary=self.binloc, debug=self.debug) - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def __getattr__(self, name): - if name in self.reqrsp_methods: - return getattr(self.msg_conn, name) - raise NotImplementedError(name) - - @property - def maddr(self): - if self.ltype is not None: - return "{}:{}".format(self.ltype, self.laddr) - else: - return None - - def execute_binary(self, binary=None, debug=False, listen_addr=None): - self.binloc = binary - args = [self.binloc] - if listen_addr is not None: - args += ["--msglisten", listen_addr] - else: - args += ["--msgauto"] - - if debug: - args += ["--dbg"] - self.proxy_proc = Popen(args, stdout=PIPE, stderr=PIPE) - - # Wait for it to start and make connection - listenstr = self.proxy_proc.stdout.readline().rstrip() - self.msg_connect(listenstr.decode()) - - def msg_connect(self, addr): - self.ltype, self.laddr = addr.split(":", 1) - self.msg_conn = self.new_conn() - self._get_storage() - - def close(self): - conns = list(self.conns) - for conn in conns: - conn.close() - if self.proxy_proc is not None: - self.proxy_proc.terminate() - - def new_conn(self): - conn = ProxyConnection(kind=self.ltype, addr=self.laddr) - conn.parent_client = self - conn.debug = self.debug - self.conns.add(conn) - return conn - - # functions involving storage - - def _add_storage(self, storage, prefix): - self.storage_by_prefix[prefix] = storage - self.storage_by_id[storage.storage_id] = storage - - def _clear_storage(self): - self.storage_by_prefix = {} - self.storage_by_id = {} - - def _get_storage(self): - self._clear_storage() - storages = self.list_storage() - for s in storages: - stype, prefix = s.description.split("|") - storage = ActiveStorage(stype, s.storage_id, prefix) - self._add_storage(storage, prefix) - - def parse_reqid(self, reqid): - if reqid[0].isalpha(): - prefix = reqid[0] - realid = reqid[1:] - else: - prefix = "" - realid = reqid - storage = self.storage_by_prefix[prefix] - return storage, realid - - def storage_iter(self): - for _, s in self.storage_by_id.items(): - yield s - - def _stg_or_def(self, storage): - if storage is None: - return self.proxy_storage - return storage - - def in_context_requests(self, headers_only=False, max_results=0): - results = self.query_storage(self.context.query, - headers_only=headers_only, - max_results=max_results) - ret = results - if max_results > 0 and len(results) > max_results: - ret = results[:max_results] - return ret - - def prefixed_reqid(self, req): - prefix = "" - if req.storage_id in self.storage_by_id: - s = self.storage_by_id[req.storage_id] - prefix = s.prefix - return "{}{}".format(prefix, req.db_id) - - # functions that don't just pass through to underlying conn - - def add_sqlite_storage(self, path, prefix): - desc = _serialize_storage("sqlite", prefix) - sid = self.msg_conn.add_sqlite_storage(path, desc) - s = ActiveStorage(type="sqlite", storage_id=sid, prefix=prefix) - self._add_storage(s, prefix) - return s - - def add_in_memory_storage(self, prefix): - desc = _serialize_storage("inmem", prefix) - sid = self.msg_conn.add_in_memory_storage(desc) - s = ActiveStorage(type="inmem", storage_id=sid, prefix=prefix) - self._add_storage(s, prefix) - return s - - def close_storage(self, storage_id): - s = self.storage_by_id[storage_id] - self.msg_conn.close_storage(s.storage_id) - del self.storage_by_id[s.storage_id] - del self.storage_by_prefix[s.storage_prefix] - - def set_proxy_storage(self, storage_id): - s = self.storage_by_id[storage_id] - self.msg_conn.set_proxy_storage(s.storage_id) - self.proxy_storage = storage_id - - def save_new(self, req, storage=None): - self.msg_conn.save_new(req, storage=self._stg_or_def(storage)) - - def submit(self, req, storage=None): - self.msg_conn.submit(req, storage=self._stg_or_def(storage)) - - def query_storage(self, q, max_results=0, headers_only=False, storage=None): - results = [] - if storage is None: - for s in self.storage_iter(): - results += self.msg_conn.query_storage(q, max_results=max_results, - headers_only=headers_only, - storage=s.storage_id) - else: - results += self.msg_conn.query_storage(q, max_results=max_results, - headers_only=headers_only, - storage=storage) - results.sort(key=lambda req: req.time_start) - results = [r for r in reversed(results)] - return results - - def req_by_id(self, reqid, headers_only=False): - storage, rid = self.parse_reqid(reqid) - return self.msg_conn.req_by_id(rid, headers_only=headers_only, - storage=storage.storage_id) - - # for these and submit, might need storage stored on the request itself - def add_tag(self, reqid, tag, storage=None): - self.msg_conn.add_tag(reqid, tag, storage=self._stg_or_def(storage)) - - def remove_tag(self, reqid, tag, storage=None): - self.msg_conn.remove_tag(reqid, tag, storage=self._stg_or_def(storage)) - - def clear_tag(self, reqid, storage=None): - self.msg_conn.clear_tag(reqid, storage=self._stg_or_def(storage)) - - def all_saved_queries(self, storage=None): - self.msg_conn.all_saved_queries(storage=None) - - def save_query(self, name, filt, storage=None): - self.msg_conn.save_query(name, filt, storage=self._stg_or_def(storage)) - - def load_query(self, name, storage=None): - self.msg_conn.load_query(name, storage=self._stg_or_def(storage)) - - def delete_query(self, name, storage=None): - self.msg_conn.delete_query(name, storage=self._stg_or_def(storage)) - - -def decode_req(result, headers_only=False): - if "StartTime" in result: - time_start = time_from_nsecs(result["StartTime"]) - else: - time_start = None - - if "EndTime" in result: - time_end = time_from_nsecs(result["EndTime"]) - else: - time_end = None - - if "DbId" in result: - db_id = result["DbId"] - else: - db_id = "" - - if "Tags" in result: - tags = result["Tags"] - else: - tags = "" - - ret = HTTPRequest( - method=result["Method"], - path=result["Path"], - proto_major=result["ProtoMajor"], - proto_minor=result["ProtoMinor"], - headers=copy.deepcopy(result["Headers"]), - body=base64.b64decode(result["Body"]), - dest_host=result["DestHost"], - dest_port=result["DestPort"], - use_tls=result["UseTLS"], - time_start=time_start, - time_end=time_end, - tags=tags, - headers_only=headers_only, - db_id=db_id, - ) - - if "Unmangled" in result: - ret.unmangled = decode_req(result["Unmangled"], headers_only=headers_only) - if "Response" in result: - ret.response = decode_rsp(result["Response"], headers_only=headers_only) - if "WSMessages" in result: - for wsm in result["WSMessages"]: - ret.ws_messages.append(decode_ws(wsm)) - return ret - -def decode_rsp(result, headers_only=False): - ret = HTTPResponse( - status_code=result["StatusCode"], - reason=result["Reason"], - proto_major=result["ProtoMajor"], - proto_minor=result["ProtoMinor"], - headers=copy.deepcopy(result["Headers"]), - body=base64.b64decode(result["Body"]), - headers_only=headers_only, - ) - - if "Unmangled" in result: - ret.unmangled = decode_rsp(result["Unmangled"], headers_only=headers_only) - return ret - -def decode_ws(result): - timestamp = None - db_id = "" - - if "Timestamp" in result: - timestamp = time_from_nsecs(result["Timestamp"]) - if "DbId" in result: - db_id = result["DbId"] - - ret = WSMessage( - is_binary=result["IsBinary"], - message=base64.b64decode(result["Message"]), - to_server=result["ToServer"], - timestamp=timestamp, - db_id=db_id, - ) - - if "Unmangled" in result: - ret.unmangled = decode_ws(result["Unmangled"]) - - return ret - -def encode_req(req, int_rsp=False): - msg = { - "DestHost": req.dest_host, - "DestPort": req.dest_port, - "UseTLS": req.use_tls, - "Method": req.method, - "Path": req.url.geturl(), - "ProtoMajor": req.proto_major, - "ProtoMinor": req.proto_major, - "Headers": req.headers.dict(), - "Body": base64.b64encode(copy.copy(req.body)).decode(), - } - - if not int_rsp: - msg["StartTime"] = time_to_nsecs(req.time_start) - msg["EndTime"] = time_to_nsecs(req.time_end) - if req.unmangled is not None: - msg["Unmangled"] = encode_req(req.unmangled) - if req.response is not None: - msg["Response"] = encode_rsp(req.response) - msg["WSMessages"] = [] - for wsm in req.ws_messages: - msg["WSMessages"].append(encode_ws(wsm)) - return msg - -def encode_rsp(rsp, int_rsp=False): - msg = { - "ProtoMajor": rsp.proto_major, - "ProtoMinor": rsp.proto_minor, - "StatusCode": rsp.status_code, - "Reason": rsp.reason, - "Headers": rsp.headers.dict(), - "Body": base64.b64encode(copy.copy(rsp.body)).decode(), - } - - if not int_rsp: - if rsp.unmangled is not None: - msg["Unmangled"] = encode_rsp(rsp.unmangled) - return msg - -def encode_ws(ws, int_rsp=False): - msg = { - "Message": base64.b64encode(ws.message).decode(), - "IsBinary": ws.is_binary, - "toServer": ws.to_server, - } - if not int_rsp: - if ws.unmangled is not None: - msg["Unmangled"] = encode_ws(ws.unmangled) - msg["Timestamp"] = time_to_nsecs(ws.timestamp) - msg["DbId"] = ws.db_id - return msg - -def time_from_nsecs(nsecs): - secs = nsecs/1000000000 - t = datetime.datetime.utcfromtimestamp(secs) - return t - -def time_to_nsecs(t): - if t is None: - return None - secs = (t-datetime.datetime(1970,1,1)).total_seconds() - return int(math.floor(secs * 1000000000)) - -RequestStatusLine = namedtuple("RequestStatusLine", ["method", "path", "proto_major", "proto_minor"]) -ResponseStatusLine = namedtuple("ResponseStatusLine", ["proto_major", "proto_minor", "status_code", "reason"]) - -def parse_req_sline(sline): - if len(sline.split(b' ')) == 3: - verb, path, version = sline.split(b' ') - elif len(parts) == 2: - verb, version = parts.split(b' ') - path = b'' - else: - raise ParseError("malformed statusline") - raw_version = version[5:] # strip HTTP/ - pmajor, pminor = raw_version.split(b'.', 1) - return RequestStatusLine(verb.decode(), path.decode(), int(pmajor), int(pminor)) - -def parse_rsp_sline(sline): - if len(sline.split(b' ')) > 2: - version, status_code, reason = sline.split(b' ', 2) - else: - version, status_code = sline.split(b' ', 1) - reason = '' - raw_version = version[5:] # strip HTTP/ - pmajor, pminor = raw_version.split(b'.', 1) - return ResponseStatusLine(int(pmajor), int(pminor), int(status_code), reason.decode()) - -def _parse_message(bs, sline_parser): - header_env, body = re.split(b"\r?\n\r?\n", bs, 1) - status_line, header_bytes = re.split(b"\r?\n", header_env, 1) - h = Headers() - for l in re.split(b"\r?\n", header_bytes): - k, v = l.split(b": ", 1) - if k.lower != 'content-length': - h.add(k.decode(), v.decode()) - h.add("Content-Length", str(len(body))) - return (sline_parser(status_line), h, body) - -def parse_request(bs, dest_host='', dest_port=80, use_tls=False): - req_sline, headers, body = _parse_message(bs, parse_req_sline) - req = HTTPRequest( - method=req_sline.method, - path=req_sline.path, - proto_major=req_sline.proto_major, - proto_minor=req_sline.proto_minor, - headers=headers.dict(), - body=body, - dest_host=dest_host, - dest_port=dest_port, - use_tls=use_tls, - ) - return req - -def parse_response(bs): - rsp_sline, headers, body = _parse_message(bs, parse_rsp_sline) - rsp = HTTPResponse( - status_code=rsp_sline.status_code, - reason=rsp_sline.reason, - proto_major=rsp_sline.proto_major, - proto_minor=rsp_sline.proto_minor, - headers=headers.dict(), - body=body, - ) - return rsp - -## ACTUAL PLUGIN DATA ## - -def escape(s): - return s.replace("'", "''") - -def run_command(command): - funcs = { - "setup": set_up_windows, - "submit": submit_current_buffer, - } - if command in funcs: - funcs[command]() - -def set_buffer_content(buf, text): - buf[:] = None - first = True - for l in text.split('\n'): - if first: - buf[0] = l - first = False - else: - buf.append(l) - -def update_buffers(req): - b1_id = int(vim.eval("s:b1")) - b1 = vim.buffers[b1_id] - - b2_id = int(vim.eval("s:b2")) - b2 = vim.buffers[b2_id] - - # Set up the buffers - set_buffer_content(b1, req.full_message()) - - if req.response is not None: - set_buffer_content(b2, req.response.full_message()) - - # Save the port, ssl, host setting - vim.command("let s:dest_port=%d" % req.dest_port) - vim.command("let s:dest_host='%s'" % escape(req.dest_host)) - - if req.use_tls: - vim.command("let s:use_tls=1") - else: - vim.command("let s:use_tls=0") - -def set_conn(conn_type, conn_addr): - conn_type = vim.command("let s:conn_type='%s'" % escape(conn_type)) - conn_addr = vim.command("let s:conn_addr='%s'" % escape(conn_addr)) - -def get_conn_addr(): - conn_type = vim.eval("s:conn_type") - conn_addr = vim.eval("s:conn_addr") - return (conn_type, conn_addr) - -def set_up_windows(): - reqid = vim.eval("a:2") - storage_id = vim.eval("a:3") - msg_addr = vim.eval("a:4") - - vim.command("let s:storage_id=%d" % int(storage_id)) - - # Get the left buffer - vim.command("new") - vim.command("only") - b2 = vim.current.buffer - vim.command("let s:b2=bufnr('$')") - - # Vsplit new file - vim.command("vnew") - b1 = vim.current.buffer - vim.command("let s:b1=bufnr('$')") - - print msg_addr - comm_type, comm_addr = msg_addr.split(":", 1) - set_conn(comm_type, comm_addr) - with ProxyConnection(kind=comm_type, addr=comm_addr) as conn: - # Get the request - req = conn.req_by_id(reqid, int(storage_id)) - update_buffers(req) - -def dest_loc(): - dest_host = vim.eval("s:dest_host") - dest_port = int(vim.eval("s:dest_port")) - tls_num = vim.eval("s:use_tls") - storage_id = int(vim.eval("s:storage_id")) - if tls_num == "1": - use_tls = True - else: - use_tls = False - return (dest_host, dest_port, use_tls, storage_id) - -def submit_current_buffer(): - curbuf = vim.current.buffer - b2_id = int(vim.eval("s:b2")) - b2 = vim.buffers[b2_id] - vim.command("let s:b1=bufnr('$')") - vim.command("only") - vim.command("rightbelow vertical new") - vim.command("b %d" % b2_id) - vim.command("wincmd h") - full_request = '\n'.join(curbuf) - - req = parse_request(full_request) - dest_host, dest_port, use_tls, storage_id = dest_loc() - req.dest_host = dest_host - req.dest_port = dest_port - req.use_tls = use_tls - - comm_type, comm_addr = get_conn_addr() - with ProxyConnection(kind=comm_type, addr=comm_addr) as conn: - new_req = conn.submit(req, storage=storage_id) - conn.add_tag(new_req.db_id, "repeater", storage_id) - update_buffers(new_req) - -# (left, right) = set_up_windows() -# set_buffer_content(left, 'Hello\nWorld') -# set_buffer_content(right, 'Hello\nOther\nWorld') -#print "Arg is %s" % vim.eval("a:arg") -run_command(vim.eval("a:1")) diff --git a/python/puppy/puppyproxy/interface/repeater/repeater.vim b/python/puppy/puppyproxy/interface/repeater/repeater.vim deleted file mode 100644 index 756ca51..0000000 --- a/python/puppy/puppyproxy/interface/repeater/repeater.vim +++ /dev/null @@ -1,20 +0,0 @@ -if !has('python') - echo "Vim must support python in order to use the repeater" - finish -endif - -" Settings to make life easier -set hidden - -let s:pyscript = resolve(expand(':p:h') . '/repeater.py') - -function! RepeaterAction(...) - execute 'pyfile ' . s:pyscript -endfunc - -command! -nargs=* RepeaterSetup call RepeaterAction('setup', ) -command! RepeaterSubmitBuffer call RepeaterAction('submit') - -" Bind forward to f -nnoremap f :RepeaterSubmitBuffer - diff --git a/python/puppy/puppyproxy/interface/tags.py b/python/puppy/puppyproxy/interface/tags.py deleted file mode 100644 index ff617bc..0000000 --- a/python/puppy/puppyproxy/interface/tags.py +++ /dev/null @@ -1,62 +0,0 @@ -from ..util import confirm - -def tag_cmd(client, args): - if len(args) == 0: - raise CommandError("Usage: tag [reqid1] [reqid2] ...") - if not args[0]: - raise CommandError("Tag cannot be empty") - tag = args[0] - reqids = [] - if len(args) > 1: - for reqid in args[1:]: - client.add_tag(reqid, tag) - else: - icr = client.in_context_requests(headers_only=True) - cnt = confirm("You are about to tag {} requests with \"{}\". Continue?".format(len(icr), tag)) - if not cnt: - return - for reqh in icr: - reqid = client.prefixed_reqid(reqh) - client.remove_tag(reqid, tag) - -def untag_cmd(client, args): - if len(args) == 0: - raise CommandError("Usage: untag [reqid1] [reqid2] ...") - if not args[0]: - raise CommandError("Tag cannot be empty") - tag = args[0] - reqids = [] - if len(args) > 0: - for reqid in args[1:]: - client.remove_tag(reqid, tag) - else: - icr = client.in_context_requests(headers_only=True) - cnt = confirm("You are about to remove the \"{}\" tag from {} requests. Continue?".format(tag, len(icr))) - if not cnt: - return - for reqh in icr: - reqid = client.prefixed_reqid(reqh) - client.add_tag(reqid, tag) - -def clrtag_cmd(client, args): - if len(args) == 0: - raise CommandError("Usage: clrtag [reqid1] [reqid2] ...") - reqids = [] - if len(args) > 0: - for reqid in args: - client.clear_tag(reqid) - else: - icr = client.in_context_requests(headers_only=True) - cnt = confirm("You are about to clear ALL TAGS from {} requests. Continue?".format(len(icr))) - if not cnt: - return - for reqh in icr: - reqid = client.prefixed_reqid(reqh) - client.clear_tag(reqid) - -def load_cmds(cmd): - cmd.set_cmds({ - 'clrtag': (clrtag_cmd, None), - 'untag': (untag_cmd, None), - 'tag': (tag_cmd, None), - }) diff --git a/python/puppy/puppyproxy/interface/test.py b/python/puppy/puppyproxy/interface/test.py deleted file mode 100644 index 5847b25..0000000 --- a/python/puppy/puppyproxy/interface/test.py +++ /dev/null @@ -1,7 +0,0 @@ - -def test_cmd(client, args): - print("args:", ', '.join(args)) - print("ping:", client.ping()) - -def load_cmds(cons): - cons.set_cmd("test", test_cmd) diff --git a/python/puppy/puppyproxy/interface/view.py b/python/puppy/puppyproxy/interface/view.py deleted file mode 100644 index 175c17c..0000000 --- a/python/puppy/puppyproxy/interface/view.py +++ /dev/null @@ -1,674 +0,0 @@ -import datetime -import json -import pygments -import pprint -import re -import shlex -import urllib - -from ..util import print_table, print_request_rows, get_req_data_row, datetime_string, maybe_hexdump -from ..colors import Colors, Styles, verb_color, scode_color, path_formatter, color_string, url_formatter, pretty_msg, pretty_headers -from ..console import CommandError -from pygments.formatters import TerminalFormatter -from pygments.lexers.data import JsonLexer -from pygments.lexers.html import XmlLexer -from urllib.parse import parse_qs, unquote - -################### -## Helper functions - -def view_full_message(request, headers_only=False, try_ws=False): - def _print_message(mes): - print_str = '' - if mes.to_server == False: - print_str += Colors.BLUE - print_str += '< Incoming' - else: - print_str += Colors.GREEN - print_str += '> Outgoing' - print_str += Colors.ENDC - if mes.unmangled: - print_str += ', ' + Colors.UNDERLINE + 'mangled' + Colors.ENDC - t_plus = "??" - if request.time_start: - t_plus = mes.timestamp - request.time_start - print_str += ', binary = %s, T+%ss\n' % (mes.is_binary, t_plus.total_seconds()) - - print_str += Colors.ENDC - print_str += maybe_hexdump(mes.message).decode() - print_str += '\n' - return print_str - - if headers_only: - print(pretty_headers(request)) - else: - if try_ws and request.ws_messages: - print_str = '' - print_str += Styles.TABLE_HEADER - print_str += "Websocket session handshake\n" - print_str += Colors.ENDC - print_str += pretty_msg(request) - print_str += '\n' - print_str += Styles.TABLE_HEADER - print_str += "Websocket session \n" - print_str += Colors.ENDC - for wsm in request.ws_messages: - print_str += _print_message(wsm) - if wsm.unmangled: - print_str += Colors.YELLOW - print_str += '-'*10 - print_str += Colors.ENDC - print_str += ' vv UNMANGLED vv ' - print_str += Colors.YELLOW - print_str += '-'*10 - print_str += Colors.ENDC - print_str += '\n' - print_str += _print_message(wsm.unmangled) - print_str += Colors.YELLOW - print_str += '-'*20 + '-'*len(' ^^ UNMANGLED ^^ ') - print_str += '\n' - print_str += Colors.ENDC - print(print_str) - else: - print(pretty_msg(request)) - -def print_request_extended(client, request): - # Prints extended info for the request - title = "Request Info (reqid=%s)" % client.prefixed_reqid(request) - print(Styles.TABLE_HEADER + title + Colors.ENDC) - reqlen = len(request.body) - reqlen = '%d bytes' % reqlen - rsplen = 'No response' - - mangle_str = 'Nothing mangled' - if request.unmangled: - mangle_str = 'Request' - - if request.response: - response_code = str(request.response.status_code) + \ - ' ' + request.response.reason - response_code = scode_color(response_code) + response_code + Colors.ENDC - rsplen = request.response.content_length - rsplen = '%d bytes' % rsplen - - if request.response.unmangled: - if mangle_str == 'Nothing mangled': - mangle_str = 'Response' - else: - mangle_str += ' and Response' - else: - response_code = '' - - time_str = '--' - if request.response is not None: - time_delt = request.time_end - request.time_start - time_str = "%.2f sec" % time_delt.total_seconds() - - if request.use_tls: - is_ssl = 'YES' - else: - is_ssl = Colors.RED + 'NO' + Colors.ENDC - - if request.time_start: - time_made_str = datetime_string(request.time_start) - else: - time_made_str = '--' - - verb = verb_color(request.method) + request.method + Colors.ENDC - host = color_string(request.dest_host) - - colored_tags = [color_string(t) for t in request.tags] - - print_pairs = [] - print_pairs.append(('Made on', time_made_str)) - print_pairs.append(('ID', client.prefixed_reqid(request))) - print_pairs.append(('URL', url_formatter(request, colored=True))) - print_pairs.append(('Host', host)) - print_pairs.append(('Path', path_formatter(request.url.path))) - print_pairs.append(('Verb', verb)) - print_pairs.append(('Status Code', response_code)) - print_pairs.append(('Request Length', reqlen)) - print_pairs.append(('Response Length', rsplen)) - if request.response and request.response.unmangled: - print_pairs.append(('Unmangled Response Length', request.response.unmangled.content_length)) - print_pairs.append(('Time', time_str)) - print_pairs.append(('Port', request.dest_port)) - print_pairs.append(('SSL', is_ssl)) - print_pairs.append(('Mangled', mangle_str)) - print_pairs.append(('Tags', ', '.join(colored_tags))) - - for k, v in print_pairs: - print(Styles.KV_KEY+str(k)+': '+Styles.KV_VAL+str(v)) - -def pretty_print_body(fmt, body): - try: - bstr = body.decode() - if fmt.lower() == 'json': - d = json.loads(bstr.strip()) - s = json.dumps(d, indent=4, sort_keys=True) - print(pygments.highlight(s, JsonLexer(), TerminalFormatter())) - elif fmt.lower() == 'form': - qs = parse_qs(bstr) - for k, vs in qs.items(): - for v in vs: - s = Colors.GREEN - s += '%s: ' % unquote(k) - s += Colors.ENDC - s += unquote(v) - print(s) - elif fmt.lower() == 'text': - print(bstr) - elif fmt.lower() == 'xml': - import xml.dom.minidom - xml = xml.dom.minidom.parseString(bstr) - print(pygments.highlight(xml.toprettyxml(), XmlLexer(), TerminalFormatter())) - else: - raise CommandError('"%s" is not a valid format' % fmt) - except CommandError as e: - raise e - except Exception as e: - raise CommandError('Body could not be parsed as "{}": {}'.format(fmt, e)) - -def print_params(client, req, params=None): - if not req.url.parameters() and not req.body: - print('Request %s has no url or data parameters' % client.prefixed_reqid(req)) - print('') - if req.url.parameters(): - print(Styles.TABLE_HEADER + "Url Params" + Colors.ENDC) - for k, v in req.url.param_iter(): - if params is None or (params and k in params): - print(Styles.KV_KEY+str(k)+': '+Styles.KV_VAL+str(v)) - print('') - if req.body: - print(Styles.TABLE_HEADER + "Body/POST Params" + Colors.ENDC) - pretty_print_body(guess_pretty_print_fmt(req), req.body) - print('') - if 'cookie' in req.headers: - print(Styles.TABLE_HEADER + "Cookies" + Colors.ENDC) - for k, v in req.cookie_iter(): - if params is None or (params and k in params): - print(Styles.KV_KEY+str(k)+': '+Styles.KV_VAL+str(v)) - print('') - # multiform request when we support it - -def guess_pretty_print_fmt(msg): - if 'content-type' in msg.headers: - if 'json' in msg.headers.get('content-type'): - return 'json' - elif 'www-form' in msg.headers.get('content-type'): - return 'form' - elif 'application/xml' in msg.headers.get('content-type'): - return 'xml' - return 'text' - -def print_tree(tree): - # Prints a tree. Takes in a sorted list of path tuples - _print_tree_helper(tree, 0, []) - -def _get_tree_prefix(depth, print_bars, last): - if depth == 0: - return u'' - else: - ret = u'' - pb = print_bars + [True] - for i in range(depth): - if pb[i]: - ret += u'\u2502 ' - else: - ret += u' ' - if last: - ret += u'\u2514\u2500 ' - else: - ret += u'\u251c\u2500 ' - return ret - -def _print_tree_helper(tree, depth, print_bars): - # Takes in a tree and prints it at the given depth - if tree == [] or tree == [()]: - return - while tree[0] == (): - tree = tree[1:] - if tree == [] or tree == [()]: - return - if len(tree) == 1 and len(tree[0]) == 1: - print(_get_tree_prefix(depth, print_bars + [False], True) + tree[0][0]) - return - - curkey = tree[0][0] - subtree = [] - for row in tree: - if row[0] != curkey: - if curkey == '': - curkey = '/' - print(_get_tree_prefix(depth, print_bars, False) + curkey) - if depth == 0: - _print_tree_helper(subtree, depth+1, print_bars + [False]) - else: - _print_tree_helper(subtree, depth+1, print_bars + [True]) - curkey = row[0] - subtree = [] - subtree.append(row[1:]) - if curkey == '': - curkey = '/' - print(_get_tree_prefix(depth, print_bars, True) + curkey) - _print_tree_helper(subtree, depth+1, print_bars + [False]) - - -def add_param(found_params, kind: str, k: str, v: str, reqid: str): - if type(k) is not str: - raise Exception("BAD") - if not k in found_params: - found_params[k] = {} - if kind in found_params[k]: - found_params[k][kind].append((reqid, v)) - else: - found_params[k][kind] = [(reqid, v)] - -def print_param_info(param_info): - for k, d in param_info.items(): - print(Styles.TABLE_HEADER + k + Colors.ENDC) - for param_type, valpairs in d.items(): - print(param_type) - value_ids = {} - for reqid, val in valpairs: - ids = value_ids.get(val, []) - ids.append(reqid) - value_ids[val] = ids - for val, ids in value_ids.items(): - if len(ids) <= 15: - idstr = ', '.join(ids) - else: - idstr = ', '.join(ids[:15]) + '...' - if val == '': - printstr = (Colors.RED + 'BLANK' + Colors.ENDC + 'x%d (%s)') % (len(ids), idstr) - else: - printstr = (Colors.GREEN + '%s' + Colors.ENDC + 'x%d (%s)') % (val, len(ids), idstr) - print(printstr) - print('') - -def path_tuple(url): - return tuple(url.path.split('/')) - -#################### -## Command functions - -def list_reqs(client, args): - """ - List the most recent in-context requests. By default shows the most recent 25 - Usage: list [a|num] - - If `a` is given, all the in-context requests are shown. If a number is given, - that many requests will be shown. - """ - if len(args) > 0: - if args[0][0].lower() == 'a': - print_count = 0 - else: - try: - print_count = int(args[0]) - except: - print("Please enter a valid argument for list") - return - else: - print_count = 25 - - rows = [] - reqs = client.in_context_requests(headers_only=True, max_results=print_count) - for req in reqs: - rows.append(get_req_data_row(req, client=client)) - print_request_rows(rows) - -def view_full_request(client, args): - """ - View the full data of the request - Usage: view_full_request - """ - if not args: - raise CommandError("Request id is required") - reqid = args[0] - req = client.req_by_id(reqid) - view_full_message(req, try_ws=True) - -def view_full_response(client, args): - """ - View the full data of the response associated with a request - Usage: view_full_response - """ - if not args: - raise CommandError("Request id is required") - reqid = args[0] - req = client.req_by_id(reqid) - if not req.response: - raise CommandError("request {} does not have an associated response".format(reqid)) - view_full_message(req.response) - -def view_request_headers(client, args): - """ - View the headers of the request - Usage: view_request_headers - """ - if not args: - raise CommandError("Request id is required") - reqid = args[0] - req = client.req_by_id(reqid, headers_only=True) - view_full_message(req, True) - -def view_response_headers(client, args): - """ - View the full data of the response associated with a request - Usage: view_full_response - """ - if not args: - raise CommandError("Request id is required") - reqid = args[0] - req = client.req_by_id(reqid) - if not req.response: - raise CommandError("request {} does not have an associated response".format(reqid)) - view_full_message(req.response, headers_only=True) - -def view_request_info(client, args): - """ - View information about request - Usage: view_request_info - """ - if not args: - raise CommandError("Request id is required") - reqid = args[0] - req = client.req_by_id(reqid, headers_only=True) - print_request_extended(client, req) - print('') - -def pretty_print_request(client, args): - """ - Print the body of the request pretty printed. - Usage: pretty_print_request - """ - if len(args) < 2: - raise CommandError("Usage: pretty_print_request ") - print_type = args[0] - reqid = args[1] - req = client.req_by_id(reqid) - pretty_print_body(print_type, req.body) - -def pretty_print_response(client, args): - """ - Print the body of the response pretty printed. - Usage: pretty_print_response - """ - if len(args) < 2: - raise CommandError("Usage: pretty_print_response ") - print_type = args[0] - reqid = args[1] - req = client.req_by_id(reqid) - if not req.response: - raise CommandError("request {} does not have an associated response".format(reqid)) - pretty_print_body(print_type, req.response.body) - -def print_params_cmd(client, args): - """ - View the parameters of a request - Usage: print_params [key 1] [key 2] ... - """ - if not args: - raise CommandError("Request id is required") - if len(args) > 1: - keys = args[1:] - else: - keys = None - - reqid = args[0] - req = client.req_by_id(reqid) - print_params(client, req, keys) - -def get_param_info(client, args): - if args and args[0] == 'ct': - contains = True - args = args[1:] - else: - contains = False - - if args: - params = tuple(args) - else: - params = None - - def check_key(k, params, contains): - if contains: - for p in params: - if p.lower() in k.lower(): - return True - else: - if params is None or k in params: - return True - return False - - found_params = {} - - reqs = client.in_context_requests() - for req in reqs: - prefixed_id = client.prefixed_reqid(req) - for k, v in req.url.param_iter(): - if type(k) is not str: - raise Exception("BAD") - if check_key(k, params, contains): - add_param(found_params, 'Url Parameter', k, v, prefixed_id) - for k, v in req.param_iter(): - if check_key(k, params, contains): - add_param(found_params, 'POST Parameter', k, v, prefixed_id) - for k, v in req.cookie_iter(): - if check_key(k, params, contains): - add_param(found_params, 'Cookie', k, v, prefixed_id) - print_param_info(found_params) - -def find_urls(client, args): - reqs = client.in_context_requests() # update to take reqlist - - url_regexp = b'((?:http|ftp|https)://(?:[\w_-]+(?:(?:\.[\w_-]+)+))(?:[\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?)' - urls = set() - for req in reqs: - urls |= set(re.findall(url_regexp, req.full_message())) - if req.response: - urls |= set(re.findall(url_regexp, req.response.full_message())) - for url in sorted(urls): - print(url.decode()) - -def site_map(client, args): - """ - Print the site map. Only includes requests in the current context. - Usage: site_map - """ - if len(args) > 0 and args[0] == 'p': - paths = True - else: - paths = False - all_reqs = client.in_context_requests(headers_only=True) - reqs_by_host = {} - for req in all_reqs: - reqs_by_host.setdefault(req.dest_host, []).append(req) - for host, reqs in reqs_by_host.items(): - paths_set = set() - for req in reqs: - if req.response and req.response.status_code != 404: - paths_set.add(path_tuple(req.url)) - tree = sorted(list(paths_set)) - print(host) - if paths: - for p in tree: - print ('/'.join(list(p))) - else: - print_tree(tree) - print("") - -def dump_response(client, args): - """ - Dump the data of the response to a file. - Usage: dump_response - """ - # dump the data of a response - if not args: - raise CommandError("Request id is required") - req = client.req_by_id(args[0]) - if req.response: - rsp = req.response - if len(args) >= 2: - fname = args[1] - else: - fname = req.url.path.split('/')[-1] - - with open(fname, 'wb') as f: - f.write(rsp.body) - print('Response data written to {}'.format(fname)) - else: - print('Request {} does not have a response'.format(req.reqid)) - -def get_surrounding_lines(s, n, lines): - left = n - right = n - lines_left = 0 - lines_right = 0 - - # move left until we find enough lines or hit the edge - while left > 0 and lines_left < lines: - if s[left] == '\n': - lines_left += 1 - left -= 1 - - # move right until we find enough lines or hit the edge - while right < len(s) and lines_right < lines: - if s[right] == '\n': - lines_right += 1 - right += 1 - - return s[left:right] - -def print_search_header(reqid, locstr): - printstr = Styles.TABLE_HEADER - printstr += "Result(s) for request {} ({})".format(reqid, locstr) - printstr += Colors.ENDC - print(printstr) - -def highlight_str(s, substr): - highlighted = Colors.BGYELLOW + Colors.BLACK + Colors.BOLD + substr + Colors.ENDC - return s.replace(substr, highlighted) - -def search_message(mes, substr, lines, reqid, locstr): - header_printed = False - for m in re.finditer(substr, mes): - if not header_printed: - print_search_header(reqid, locstr) - header_printed = True - n = m.start() - linestr = get_surrounding_lines(mes, n, lines) - linelist = linestr.split('\n') - linestr = '\n'.join(line[:500] for line in linelist) - toprint = highlight_str(linestr, substr) - print(toprint) - print('-'*50) - -def search(client, args): - search_str = args[0] - lines = 2 - if len(args) > 1: - lines = int(args[1]) - for req in client.in_context_requests_iter(): - reqid = client.get_reqid(req) - reqheader_printed = False - try: - mes = req.full_message().decode() - search_message(mes, search_str, lines, reqid, "Request") - except UnicodeDecodeError: - pass - if req.response: - try: - mes = req.response.full_message().decode() - search_message(mes, search_str, lines, reqid, "Response") - except UnicodeDecodeError: - pass - - wsheader_printed = False - for wsm in req.ws_messages: - if not wsheader_printed: - print_search_header(client.get_reqid(req), reqid, "Websocket Messages") - wsheader_printed = True - if search_str in wsm.message: - print(highlight_str(wsm.message, search_str)) - - -# @crochet.wait_for(timeout=None) -# @defer.inlineCallbacks -# def view_request_bytes(line): -# """ -# View the raw bytes of the request. Use this if you want to redirect output to a file. -# Usage: view_request_bytes -# """ -# args = shlex.split(line) -# if not args: -# raise CommandError("Request id is required") -# reqid = args[0] - -# reqs = yield load_reqlist(reqid) -# for req in reqs: -# if len(reqs) > 1: -# print 'Request %s:' % req.reqid -# print req.full_message -# if len(reqs) > 1: -# print '-'*30 -# print '' - -# @crochet.wait_for(timeout=None) -# @defer.inlineCallbacks -# def view_response_bytes(line): -# """ -# View the full data of the response associated with a request -# Usage: view_request_bytes -# """ -# reqs = yield load_reqlist(line) -# for req in reqs: -# if req.response: -# if len(reqs) > 1: -# print '-'*15 + (' %s ' % req.reqid) + '-'*15 -# print req.response.full_message -# else: -# print "Request %s does not have a response" % req.reqid - - -############### -## Plugin hooks - -def load_cmds(cmd): - cmd.set_cmds({ - 'list': (list_reqs, None), - 'view_full_request': (view_full_request, None), - 'view_full_response': (view_full_response, None), - 'view_request_headers': (view_request_headers, None), - 'view_response_headers': (view_response_headers, None), - 'view_request_info': (view_request_info, None), - 'pretty_print_request': (pretty_print_request, None), - 'pretty_print_response': (pretty_print_response, None), - 'print_params': (print_params_cmd, None), - 'param_info': (get_param_info, None), - 'urls': (find_urls, None), - 'site_map': (site_map, None), - 'dump_response': (dump_response, None), - 'search': (search, None), - # 'view_request_bytes': (view_request_bytes, None), - # 'view_response_bytes': (view_response_bytes, None), - }) - cmd.add_aliases([ - ('list', 'ls'), - ('view_full_request', 'vfq'), - ('view_full_request', 'kjq'), - ('view_request_headers', 'vhq'), - ('view_response_headers', 'vhs'), - ('view_full_response', 'vfs'), - ('view_full_response', 'kjs'), - ('view_request_info', 'viq'), - ('pretty_print_request', 'ppq'), - ('pretty_print_response', 'pps'), - ('print_params', 'pprm'), - ('param_info', 'pri'), - ('site_map', 'sm'), - # ('view_request_bytes', 'vbq'), - # ('view_response_bytes', 'vbs'), - # #('dump_response', 'dr'), - ]) diff --git a/python/puppy/puppyproxy/macros.py b/python/puppy/puppyproxy/macros.py deleted file mode 100644 index 8ba40bc..0000000 --- a/python/puppy/puppyproxy/macros.py +++ /dev/null @@ -1,313 +0,0 @@ -import glob -import imp -import os -import random -import re -import stat -from jinja2 import Environment, FileSystemLoader -from collections import namedtuple - -from .proxy import InterceptMacro - -class MacroException(Exception): - pass - -class FileInterceptMacro(InterceptMacro): - """ - An intercepting macro that loads a macro from a file. - """ - def __init__(self, filename=''): - InterceptMacro.__init__(self) - self.file_name = '' # name from the file - self.filename = filename or '' # filename we load from - self.source = None - - if self.filename: - self.load() - - def __repr__(self): - s = self.name - names = [] - names.append(self.file_name) - s += ' (%s)' % ('/'.join(names)) - return "" % s - - def load(self): - if self.filename: - match = re.findall('.*int_(.*).py$', self.filename) - if len(match) > 0: - self.file_name = match[0] - else: - self.file_name = self.filename - - # yes there's a race condition here, but it's better than nothing - st = os.stat(self.filename) - if (st.st_mode & stat.S_IWOTH): - raise MacroException("Refusing to load world-writable macro: %s" % self.filename) - module_name = os.path.basename(os.path.splitext(self.filename)[0]) - self.source = imp.load_source('%s'%module_name, self.filename) - if self.source and hasattr(self.source, 'MACRO_NAME'): - self.name = self.source.MACRO_NAME - else: - self.name = module_name - else: - self.source = None - - # Update what we can do - if self.source and hasattr(self.source, 'mangle_request'): - self.intercept_requests = True - else: - self.intercept_requests = False - - if self.source and hasattr(self.source, 'mangle_response'): - self.intercept_responses = True - else: - self.intercept_responses = False - - if self.source and hasattr(self.source, 'mangle_websocket'): - self.intercept_ws = True - else: - self.intercept_ws = False - - def init(self, args): - if hasattr(self.source, 'init'): - self.source.init(args) - - def mangle_request(self, request): - if hasattr(self.source, 'mangle_request'): - req = self.source.mangle_request(request) - return req - return request - - def mangle_response(self, request): - if hasattr(self.source, 'mangle_response'): - rsp = self.source.mangle_response(request, request.response) - return rsp - return request.response - - def mangle_websocket(self, request, message): - if hasattr(self.source, 'mangle_websocket'): - mangled_ws = self.source.mangle_websocket(request, request.response, message) - return mangled_ws - return message - -class MacroFile: - """ - A class representing a file that can be executed to automate actions - """ - - def __init__(self, filename=''): - self.name = '' # name from the file - self.file_name = filename or '' # filename we load from - self.source = None - - if self.file_name: - self.load() - - def load(self): - if self.file_name: - match = re.findall('.*macro_(.*).py$', self.file_name) - self.name = match[0] - st = os.stat(self.file_name) - if (st.st_mode & stat.S_IWOTH): - raise PappyException("Refusing to load world-writable macro: %s" % self.file_name) - module_name = os.path.basename(os.path.splitext(self.file_name)[0]) - self.source = imp.load_source('%s'%module_name, self.file_name) - else: - self.source = None - - def execute(self, client, args): - # Execute the macro - if self.source: - self.source.run_macro(client, args) - -MacroTemplateData = namedtuple("MacroTemplateData", ["filename", "description", "argdesc", "fname_fmt"]) - -class MacroTemplate(object): - _template_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "templates") - _template_data = { - 'macro': MacroTemplateData('macro.py.tmpl', - 'Generic macro template', - '[reqids]', - 'macro_{fname}.py'), - - 'intmacro': MacroTemplateData('intmacro.py.tmpl', - 'Generic intercepting macro template', - '', - 'int_{fname}.py'), - } - - @classmethod - def fill_template(cls, template, subs): - loader = FileSystemLoader(cls._template_dir) - env = Environment(loader=loader) - template = env.get_template(cls._template_data[template].filename) - return template.render(zip=zip, **subs) - - @classmethod - def template_filename(cls, template, fname): - return cls._template_data[template].fname_fmt.format(fname=fname) - - @classmethod - def template_names(cls): - for k, v in cls._template_data.iteritems(): - yield k - - @classmethod - def template_description(cls, template): - return cls._template_data[template].description - - @classmethod - def template_argstring(cls, template): - return cls._template_data[template].argdesc - -## Other functions - -def load_macros(loc): - """ - Loads the macros stored in the location and returns a list of Macro objects - """ - macro_files = glob.glob(loc + "/macro_*.py") - macro_objs = [] - for f in macro_files: - macro_objs.append(MacroFile(f)) - - # int_macro_files = glob.glob(loc + "/int_*.py") - # int_macro_objs = [] - # for f in int_macro_files: - # try: - # int_macro_objs.append(FileInterceptMacro(f)) - # except PappyException as e: - # print(str(e)) - #return (macro_objs, int_macro_objs) - return (macro_objs, []) - -def macro_from_requests(reqs): - # Generates a macro that defines request objects for each of the requests - # in reqs - subs = {} - - req_lines = [] - req_params = [] - for req in reqs: - lines = req.full_message().splitlines(True) - #esclines = [line.encode('unicode_escape') for line in lines] - esclines = [line for line in lines] - req_lines.append(esclines) - - params = [] - params.append('dest_host="{}"'.format(req.dest_host)) - params.append('dest_port={}'.format(req.dest_port)) - params.append('use_tls={}'.format(req.use_tls)) - req_params.append(', '.join(params)) - subs['req_lines'] = req_lines - subs['req_params'] = req_params - - return MacroTemplate.fill_template('macro', subs) - -# @defer.inlineCallbacks -# def mangle_request(request, intmacros): -# """ -# Mangle a request with a list of intercepting macros. -# Returns a tuple that contains the resulting request (with its unmangled -# value set if needed) and a bool that states whether the request was modified -# Returns (None, True) if the request was dropped. - -# :rtype: (Request, Bool) -# """ -# # Mangle requests with list of intercepting macros -# if not intmacros: -# defer.returnValue((request, False)) - -# cur_req = request.copy() -# for macro in intmacros: -# if macro.intercept_requests: -# if macro.async_req: -# cur_req = yield macro.async_mangle_request(cur_req.copy()) -# else: -# cur_req = macro.mangle_request(cur_req.copy()) - -# if cur_req is None: -# defer.returnValue((None, True)) - -# mangled = False -# if not cur_req == request or \ -# not cur_req.host == request.host or \ -# not cur_req.port == request.port or \ -# not cur_req.is_ssl == request.is_ssl: -# # copy unique data to new request and clear it off old one -# cur_req.unmangled = request -# cur_req.unmangled.is_unmangled_version = True -# if request.response: -# cur_req.response = request.response -# request.response = None -# mangled = True -# else: -# # return the original request -# cur_req = request -# defer.returnValue((cur_req, mangled)) - -# @defer.inlineCallbacks -# def mangle_response(request, intmacros): -# """ -# Mangle a request's response with a list of intercepting macros. -# Returns a bool stating whether the request's response was modified. -# Unmangled values will be updated as needed. - -# :rtype: Bool -# """ -# if not intmacros: -# defer.returnValue(False) - -# old_rsp = request.response -# for macro in intmacros: -# if macro.intercept_responses: -# # We copy so that changes to request.response doesn't mangle the original response -# request.response = request.response.copy() -# if macro.async_rsp: -# request.response = yield macro.async_mangle_response(request) -# else: -# request.response = macro.mangle_response(request) - -# if request.response is None: -# defer.returnValue(True) - -# mangled = False -# if not old_rsp == request.response: -# request.response.rspid = old_rsp -# old_rsp.rspid = None -# request.response.unmangled = old_rsp -# request.response.unmangled.is_unmangled_version = True -# mangled = True -# else: -# request.response = old_rsp -# defer.returnValue(mangled) - -# @defer.inlineCallbacks -# def mangle_websocket_message(message, request, intmacros): -# # Mangle messages with list of intercepting macros -# if not intmacros: -# defer.returnValue((message, False)) - -# cur_msg = message.copy() -# for macro in intmacros: -# if macro.intercept_ws: -# if macro.async_ws: -# cur_msg = yield macro.async_mangle_ws(request, cur_msg.copy()) -# else: -# cur_msg = macro.mangle_ws(request, cur_msg.copy()) - -# if cur_msg is None: -# defer.returnValue((None, True)) - -# mangled = False -# if not cur_msg == message: -# # copy unique data to new request and clear it off old one -# cur_msg.unmangled = message -# cur_msg.unmangled.is_unmangled_version = True -# mangled = True -# else: -# # return the original request -# cur_msg = message -# defer.returnValue((cur_msg, mangled)) diff --git a/python/puppy/puppyproxy/proxy.py b/python/puppy/puppyproxy/proxy.py deleted file mode 100644 index c5137b4..0000000 --- a/python/puppy/puppyproxy/proxy.py +++ /dev/null @@ -1,1523 +0,0 @@ -#!/usr/bin/env python3 - -import base64 -import copy -import datetime -import json -import math -import re -import socket -import shlex -import threading - -from collections import namedtuple -from urllib.parse import urlparse, ParseResult, parse_qs, urlencode -from subprocess import Popen, PIPE, TimeoutExpired -from http import cookies as hcookies - - -class MessageError(Exception): - pass - - -class ProxyException(Exception): - pass - - -class InvalidQuery(Exception): - pass - -class SocketClosed(Exception): - pass - -class SockBuffer: - # I can't believe I have to implement this - - def __init__(self, sock): - self.buf = [] # a list of chunks of strings - self.s = sock - self.closed = False - - def close(self): - self.s.shutdown(socket.SHUT_RDWR) - self.s.close() - self.closed = True - - def _check_newline(self): - for chunk in self.buf: - if '\n' in chunk: - return True - return False - - def readline(self): - # Receive until we get a newline, raise SocketClosed if socket is closed - while True: - try: - data = self.s.recv(8192) - except OSError: - raise SocketClosed() - if not data: - raise SocketClosed() - self.buf.append(data) - if b'\n' in data: - break - - # Combine chunks - retbytes = bytes() - n = 0 - for chunk in self.buf: - n += 1 - if b'\n' in chunk: - head, tail = chunk.split(b'\n', 1) - retbytes += head - self.buf = self.buf[n:] - self.buf = [tail] + self.buf - break - else: - retbytes += chunk - return retbytes.decode() - - def send(self, data): - try: - self.s.send(data) - except OSError: - raise SocketClosed() - -class Headers: - def __init__(self, headers=None): - self.headers = {} - if headers is not None: - if isinstance(headers, Headers): - for _, pairs in headers.headers.items(): - for k, v in pairs: - self.add(k, v) - else: - for k, vs in headers.items(): - for v in vs: - self.add(k, v) - - def __contains__(self, hd): - for k, _ in self.headers.items(): - if k.lower() == hd.lower(): - return True - return False - - def add(self, k, v): - try: - l = self.headers[k.lower()] - l.append((k,v)) - except KeyError: - self.headers[k.lower()] = [(k,v)] - - def set(self, k, v): - self.headers[k.lower()] = [(k,v)] - - def get(self, k): - return self.headers[k.lower()][0][1] - - def delete(self, k): - del self.headers[k.lower()] - - def pairs(self, key=None): - for _, kvs in self.headers.items(): - for k, v in kvs: - if key is None or k.lower() == key.lower(): - yield (k, v) - - def dict(self): - retdict = {} - for _, kvs in self.headers.items(): - for k, v in kvs: - if k in retdict: - retdict[k].append(v) - else: - retdict[k] = [v] - return retdict - -class RequestContext: - def __init__(self, client, query=None): - self._current_query = [] - self.client = client - if query is not None: - self._current_query = query - - def _validate(self, query): - self.client.validate_query(query) - - def set_query(self, query): - self._validate(query) - self._current_query = query - - def apply_phrase(self, phrase): - self._validate([phrase]) - self._current_query.append(phrase) - - def pop_phrase(self): - if len(self._current_query) > 0: - self._current_query.pop() - - def apply_filter(self, filt): - self._validate([[filt]]) - self._current_query.append([filt]) - - @property - def query(self): - return copy.deepcopy(self._current_query) - - -class URL: - def __init__(self, url): - parsed = urlparse(url) - if url is not None: - parsed = urlparse(url) - self.scheme = parsed.scheme - self.netloc = parsed.netloc - self.path = parsed.path - self.params = parsed.params - self.query = parsed.query - self.fragment = parsed.fragment - else: - self.scheme = "" - self.netloc = "" - self.path = "/" - self.params = "" - self.query = "" - self.fragment = "" - - def geturl(self, include_params=True): - params = self.params - query = self.query - fragment = self.fragment - - if not include_params: - params = "" - query = "" - fragment = "" - - r = ParseResult(scheme=self.scheme, - netloc=self.netloc, - path=self.path, - params=params, - query=query, - fragment=fragment) - return r.geturl() - - def parameters(self): - try: - return parse_qs(self.query, keep_blank_values=True) - except Exception: - return [] - - def param_iter(self): - for k, vs in self.parameters().items(): - for v in vs: - yield k, v - - def set_param(self, key, val): - params = self.parameters() - params[key] = [val] - self.query = urlencode(params) - - def add_param(self, key, val): - params = self.parameters() - if key in params: - params[key].append(val) - else: - params[key] = [val] - self.query = urlencode(params) - - def del_param(self, key): - params = self.parameters() - del params[key] - self.query = urlencode(params) - - def set_params(self, params): - self.query = urlencode(params) - - -class InterceptMacro: - """ - A class representing a macro that modifies requests as they pass through the - proxy - """ - - def __init__(self): - self.name = '' - self.intercept_requests = False - self.intercept_responses = False - self.intercept_ws = False - - def __repr__(self): - return "" % self.name - - def mangle_request(self, request): - return request - - def mangle_response(self, request, response): - return response - - def mangle_websocket(self, request, response, message): - return message - - -class HTTPRequest: - def __init__(self, method="GET", path="/", proto_major=1, proto_minor=1, - headers=None, body=bytes(), dest_host="", dest_port=80, - use_tls=False, time_start=None, time_end=None, db_id="", - tags=None, headers_only=False, storage_id=0): - # http info - self.method = method - self.url = URL(path) - self.proto_major = proto_major - self.proto_minor = proto_minor - - self.headers = Headers(headers) - - self.headers_only = headers_only - self._body = bytes() - if not headers_only: - self.body = body - - # metadata - self.dest_host = dest_host - self.dest_port = dest_port - self.use_tls = use_tls - self.time_start = time_start - self.time_end = time_end - - self.response = None - self.unmangled = None - self.ws_messages = [] - - self.db_id = db_id - self.storage_id = storage_id - if tags is not None: - self.tags = set(tags) - else: - self.tags = set() - - @property - def body(self): - return self._body - - @body.setter - def body(self, bs): - self.headers_only = False - if type(bs) is str: - self._body = bs.encode() - elif type(bs) is bytes: - self._body = bs - else: - raise Exception("invalid body type: {}".format(type(bs))) - self.headers.set("Content-Length", str(len(self._body))) - - @property - def content_length(self): - if 'content-length' in self.headers: - return int(self.headers.get('content-length')) - return len(self.body) - - def status_line(self): - sline = "{method} {path} HTTP/{proto_major}.{proto_minor}".format( - method=self.method, path=self.url.geturl(), proto_major=self.proto_major, - proto_minor=self.proto_minor).encode() - return sline - - def headers_section(self): - message = self.status_line() + b"\r\n" - for k, v in self.headers.pairs(): - message += "{}: {}\r\n".format(k, v).encode() - return message - - def full_message(self): - message = self.headers_section() - message += b"\r\n" - message += self.body - return message - - def parameters(self): - try: - return parse_qs(self.body.decode(), keep_blank_values=True) - except Exception: - return [] - - def param_iter(self, ignore_content_type=False): - if not ignore_content_type: - if "content-type" not in self.headers: - return - if "www-form-urlencoded" not in self.headers.get("content-type").lower(): - return - for k, vs in self.parameters().items(): - for v in vs: - yield k, v - - def set_param(self, key, val): - params = self.parameters() - params[key] = [val] - self.body = urlencode(params) - - def add_param(self, key, val): - params = self.parameters() - if key in params: - params[key].append(val) - else: - params[key] = [val] - self.body = urlencode(params) - - def del_param(self, key): - params = self.parameters() - del params[key] - self.body = urlencode(params) - - def set_params(self, params): - self.body = urlencode(params) - - def cookies(self): - try: - cookie = hcookies.BaseCookie() - cookie.load(self.headers.get("cookie")) - return cookie - except Exception as e: - return hcookies.BaseCookie() - - def cookie_iter(self): - c = self.cookies() - for k in c: - yield k, c[k].value - - def set_cookie(self, key, val): - c = self.cookies() - c[key] = val - self.set_cookies(c) - - def del_cookie(self, key): - c = self.cookies() - del c[key] - self.set_cookies(c) - - def set_cookies(self, c): - cookie_pairs = [] - if isinstance(c, hcookies.BaseCookie()): - # it's a basecookie - for k in c: - cookie_pairs.append('{}={}'.format(k, c[k].value)) - else: - # it's a dictionary - for k, v in c.items(): - cookie_pairs.append('{}={}'.format(k, v)) - header_str = '; '.join(cookie_pairs) - self.headers.set("Cookie", header_str) - - def copy(self): - return HTTPRequest( - method=self.method, - path=self.url.geturl(), - proto_major=self.proto_major, - proto_minor=self.proto_minor, - headers=self.headers, - body=self.body, - dest_host=self.dest_host, - dest_port=self.dest_port, - use_tls=self.use_tls, - tags=copy.deepcopy(self.tags), - headers_only=self.headers_only, - ) - - -class HTTPResponse: - def __init__(self, status_code=200, reason="OK", proto_major=1, proto_minor=1, - headers=None, body=bytes(), db_id="", headers_only=False): - self.status_code = status_code - self.reason = reason - self.proto_major = proto_major - self.proto_minor = proto_minor - - self.headers = Headers() - if headers is not None: - for k, vs in headers.items(): - for v in vs: - self.headers.add(k, v) - - self.headers_only = headers_only - self._body = bytes() - if not headers_only: - self.body = body - - self.unmangled = None - self.db_id = db_id - - @property - def body(self): - return self._body - - @body.setter - def body(self, bs): - self.headers_only = False - if type(bs) is str: - self._body = bs.encode() - elif type(bs) is bytes: - self._body = bs - else: - raise Exception("invalid body type: {}".format(type(bs))) - self.headers.set("Content-Length", str(len(self._body))) - - @property - def content_length(self): - if 'content-length' in self.headers: - return int(self.headers.get('content-length')) - return len(self.body) - - def status_line(self): - sline = "HTTP/{proto_major}.{proto_minor} {status_code} {reason}".format( - proto_major=self.proto_major, proto_minor=self.proto_minor, - status_code=self.status_code, reason=self.reason).encode() - return sline - - def headers_section(self): - message = self.status_line() + b"\r\n" - for k, v in self.headers.pairs(): - message += "{}: {}\r\n".format(k, v).encode() - return message - - def full_message(self): - message = self.headers_section() - message += b"\r\n" - message += self.body - return message - - def cookies(self): - try: - cookie = hcookies.BaseCookie() - for _, v in self.headers.pairs('set-cookie'): - cookie.load(v) - return cookie - except Exception as e: - return hcookies.BaseCookie() - - def cookie_iter(self): - c = self.cookies() - for k in c: - yield k, c[k].value - - def set_cookie(self, key, val): - c = self.cookies() - c[key] = val - self.set_cookies(c) - - def del_cookie(self, key): - c = self.cookies() - del c[key] - self.set_cookies(c) - - def set_cookies(self, c): - self.headers.delete("set-cookie") - if isinstance(c, hcookies.BaseCookie): - cookies = c - else: - cookies = hcookies.BaseCookie() - for k, v in c.items(): - cookies[k] = v - for _, m in c.items(): - self.headers.add("Set-Cookie", m.OutputString()) - - def copy(self): - return HTTPResponse( - status_code=self.status_code, - reason=self.reason, - proto_major=self.proto_major, - proto_minor=self.proto_minor, - headers=self.headers.headers, - body=self.body, - headers_only=self.headers_only, - ) - -class WSMessage: - def __init__(self, is_binary=True, message=bytes(), to_server=True, - timestamp=None, db_id=""): - self.is_binary = is_binary - self.message = message - self.to_server = to_server - self.timestamp = timestamp or datetime.datetime(1970, 1, 1) - - self.unmangled = None - self.db_id = db_id - - def copy(self): - return WSMessage( - is_binary=self.is_binary, - message=self.message, - to_server=self.to_server, - ) - -ScopeResult = namedtuple("ScopeResult", ["is_custom", "filter"]) -ListenerResult = namedtuple("ListenerResult", ["lid", "addr"]) -GenPemCertsResult = namedtuple("GenPemCertsResult", ["key_pem", "cert_pem"]) -SavedQuery = namedtuple("SavedQuery", ["name", "query"]) -SavedStorage = namedtuple("SavedStorage", ["storage_id", "description"]) - -def messagingFunction(func): - def f(self, *args, **kwargs): - if self.is_interactive: - raise MessageError("cannot be called while other message is interactive") - if self.closed: - raise MessageError("connection is closed") - return func(self, *args, **kwargs) - return f - -class ProxyConnection: - next_id = 1 - def __init__(self, kind="", addr=""): - self.connid = ProxyConnection.next_id - ProxyConnection.next_id += 1 - self.sbuf = None - self.buf = bytes() - self.parent_client = None - self.debug = False - self.is_interactive = False - self.closed = True - self.sock_lock_read = threading.Lock() - self.sock_lock_write = threading.Lock() - self.kind = None - self.addr = None - - if kind.lower() == "tcp": - tcpaddr, port = addr.rsplit(":", 1) - self.connect_tcp(tcpaddr, int(port)) - elif kind.lower() == "unix": - self.connect_unix(addr) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def connect_tcp(self, addr, port): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((addr, port)) - self.sbuf = SockBuffer(s) - self.closed = False - self.kind = "tcp" - self.addr = "{}:{}".format(addr, port) - - def connect_unix(self, addr): - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - s.connect(addr) - self.sbuf = SockBuffer(s) - self.closed = False - self.kind = "unix" - self.addr = addr - - @property - def maddr(self): - if self.kind is not None: - return "{}:{}".format(self.kind, self.addr) - else: - return None - - def close(self): - self.sbuf.close() - if self.parent_client is not None: - self.parent_client.conns.remove(self) - self.closed = True - - def read_message(self): - with self.sock_lock_read: - l = self.sbuf.readline() - if self.debug: - print("<({}) {}".format(self.connid, l)) - j = json.loads(l) - if "Success" in j and j["Success"] == False: - if "Reason" in j: - raise MessageError(j["Reason"]) - raise MessageError("unknown error") - return j - - def submit_command(self, cmd): - with self.sock_lock_write: - ln = json.dumps(cmd).encode()+b"\n" - if self.debug: - print(">({}) {} ".format(self.connid, ln.decode())) - self.sbuf.send(ln) - - def reqrsp_cmd(self, cmd): - self.submit_command(cmd) - ret = self.read_message() - if ret is None: - raise Exception() - return ret - - ########### - ## Commands - - @messagingFunction - def ping(self): - cmd = {"Command": "Ping"} - result = self.reqrsp_cmd(cmd) - return result["Ping"] - - @messagingFunction - def submit(self, req, storage=None): - cmd = { - "Command": "Submit", - "Request": encode_req(req), - "Storage": 0, - } - if storage is not None: - cmd["Storage"] = storage - result = self.reqrsp_cmd(cmd) - if "SubmittedRequest" not in result: - raise MessageError("no request returned") - req = decode_req(result["SubmittedRequest"]) - req.storage_id = storage - return req - - @messagingFunction - def save_new(self, req, storage): - reqd = encode_req(req) - cmd = { - "Command": "SaveNew", - "Request": encode_req(req), - "Storage": storage, - } - result = self.reqrsp_cmd(cmd) - req.db_id = result["DbId"] - req.storage_id = storage - return result["DbId"] - - def _query_storage(self, q, storage, headers_only=False, max_results=0): - cmd = { - "Command": "StorageQuery", - "Query": q, - "HeadersOnly": headers_only, - "MaxResults": max_results, - "Storage": storage, - } - result = self.reqrsp_cmd(cmd) - reqs = [] - for reqd in result["Results"]: - req = decode_req(reqd, headers_only=headers_only) - req.storage_id = storage - reqs.append(req) - return reqs - - @messagingFunction - def query_storage(self, q, storage, max_results=0, headers_only=False): - return self._query_storage(q, storage, headers_only=headers_only, max_results=max_results) - - @messagingFunction - def req_by_id(self, reqid, storage, headers_only=False): - results = self._query_storage([[["dbid", "is", reqid]]], storage, - headers_only=headers_only, max_results=1) - if len(results) == 0: - raise MessageError("request with id {} does not exist".format(reqid)) - return results[0] - - @messagingFunction - def set_scope(self, filt): - cmd = { - "Command": "SetScope", - "Query": filt, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def get_scope(self): - cmd = { - "Command": "ViewScope", - } - result = self.reqrsp_cmd(cmd) - ret = ScopeResult(result["IsCustom"], result["Query"]) - return ret - - @messagingFunction - def add_tag(self, reqid, tag, storage): - cmd = { - "Command": "AddTag", - "ReqId": reqid, - "Tag": tag, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def remove_tag(self, reqid, tag, storage): - cmd = { - "Command": "RemoveTag", - "ReqId": reqid, - "Tag": tag, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def clear_tag(self, reqid, storage): - cmd = { - "Command": "ClearTag", - "ReqId": reqid, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def all_saved_queries(self, storage): - cmd = { - "Command": "AllSavedQueries", - "Storage": storage, - } - results = self.reqrsp_cmd(cmd) - queries = [] - for result in results["Queries"]: - queries.append(SavedQuery(name=result["Name"], query=result["Query"])) - return queries - - @messagingFunction - def save_query(self, name, filt, storage): - cmd = { - "Command": "SaveQuery", - "Name": name, - "Query": filt, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def load_query(self, name, storage): - cmd = { - "Command": "LoadQuery", - "Name": name, - "Storage": storage, - } - result = self.reqrsp_cmd(cmd) - return result["Query"] - - @messagingFunction - def delete_query(self, name, storage): - cmd = { - "Command": "DeleteQuery", - "Name": name, - "Storage": storage, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def add_listener(self, addr, port): - laddr = "{}:{}".format(addr, port) - cmd = { - "Command": "AddListener", - "Type": "tcp", - "Addr": laddr, - } - result = self.reqrsp_cmd(cmd) - lid = result["Id"] - return lid - - @messagingFunction - def remove_listener(self, lid): - cmd = { - "Command": "RemoveListener", - "Id": lid, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def get_listeners(self): - cmd = { - "Command": "GetListeners", - } - result = self.reqrsp_cmd(cmd) - results = [] - for r in result["Results"]: - results.append(r["Id"], r["Addr"]) - return results - - @messagingFunction - def load_certificates(self, pkey_file, cert_file): - cmd = { - "Command": "LoadCerts", - "KeyFile": pkey_file, - "CertificateFile": cert_file, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def set_certificates(self, pkey_pem, cert_pem): - cmd = { - "Command": "SetCerts", - "KeyPEMData": pkey_pem, - "CertificatePEMData": cert_pem, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def clear_certificates(self): - cmd = { - "Command": "ClearCerts", - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def generate_certificates(self, pkey_file, cert_file): - cmd = { - "Command": "GenCerts", - "KeyFile": pkey_file, - "CertFile": cert_file, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def generate_pem_certificates(self): - cmd = { - "Command": "GenPEMCerts", - } - result = self.reqrsp_cmd(cmd) - ret = GenPemCertsResult(result["KeyPEMData"], result["CertificatePEMData"]) - return ret - - @messagingFunction - def validate_query(self, query): - cmd = { - "Command": "ValidateQuery", - "Query": query, - } - try: - result = self.reqrsp_cmd(cmd) - except MessageError as e: - raise InvalidQuery(str(e)) - - @messagingFunction - def add_sqlite_storage(self, path, desc): - cmd = { - "Command": "AddSQLiteStorage", - "Path": path, - "Description": desc - } - result = self.reqrsp_cmd(cmd) - return result["StorageId"] - - @messagingFunction - def add_in_memory_storage(self, desc): - cmd = { - "Command": "AddInMemoryStorage", - "Description": desc - } - result = self.reqrsp_cmd(cmd) - return result["StorageId"] - - @messagingFunction - def close_storage(self, strage_id): - cmd = { - "Command": "CloseStorage", - "StorageId": storage_id, - } - result = self.reqrsp_cmd(cmd) - - @messagingFunction - def set_proxy_storage(self, storage_id): - cmd = { - "Command": "SetProxyStorage", - "StorageId": storage_id, - } - result = self.reqrsp_cmd(cmd) - - @messagingFunction - def list_storage(self): - cmd = { - "Command": "ListStorage", - } - result = self.reqrsp_cmd(cmd) - ret = [] - for ss in result["Storages"]: - ret.append(SavedStorage(ss["Id"], ss["Description"])) - return ret - - @messagingFunction - def set_proxy(self, use_proxy=False, proxy_host="", proxy_port=0, use_creds=False, - username="", password="", is_socks=False): - cmd = { - "Command": "SetProxy", - "UseProxy": use_proxy, - "ProxyHost": proxy_host, - "ProxyPort": proxy_port, - "ProxyIsSOCKS": is_socks, - "UseCredentials": use_creds, - "Username": username, - "Password": password, - } - self.reqrsp_cmd(cmd) - - @messagingFunction - def intercept(self, macro): - # Run an intercepting macro until closed - - from .util import log_error - # Start intercepting - self.is_interactive = True - cmd = { - "Command": "Intercept", - "InterceptRequests": macro.intercept_requests, - "InterceptResponses": macro.intercept_responses, - "InterceptWS": macro.intercept_ws, - } - try: - self.reqrsp_cmd(cmd) - except Exception as e: - self.is_interactive = False - raise e - - def run_macro(): - while True: - try: - msg = self.read_message() - except MessageError as e: - log_error(str(e)) - return - except SocketClosed: - return - - def mangle_and_respond(msg): - retCmd = None - if msg["Type"] == "httprequest": - req = decode_req(msg["Request"]) - newReq = macro.mangle_request(req) - - if newReq is None: - retCmd = { - "Id": msg["Id"], - "Dropped": True, - } - else: - newReq.unmangled = None - newReq.response = None - newReq.ws_messages = [] - - retCmd = { - "Id": msg["Id"], - "Dropped": False, - "Request": encode_req(newReq), - } - elif msg["Type"] == "httpresponse": - req = decode_req(msg["Request"]) - rsp = decode_rsp(msg["Response"]) - newRsp = macro.mangle_response(req, rsp) - - if newRsp is None: - retCmd = { - "Id": msg["Id"], - "Dropped": True, - } - else: - newRsp.unmangled = None - - retCmd = { - "Id": msg["Id"], - "Dropped": False, - "Response": encode_rsp(newRsp), - } - elif msg["Type"] == "wstoserver" or msg["Type"] == "wstoclient": - req = decode_req(msg["Request"]) - rsp = decode_rsp(msg["Response"]) - wsm = decode_ws(msg["WSMessage"]) - newWsm = macro.mangle_websocket(req, rsp, wsm) - - if newWsm is None: - retCmd = { - "Id": msg["Id"], - "Dropped": True, - } - else: - newWsm.unmangled = None - - retCmd = { - "Id": msg["Id"], - "Dropped": False, - "WSMessage": encode_ws(newWsm), - } - else: - raise Exception("Unknown message type: " + msg["Type"]) - if retCmd is not None: - try: - self.submit_command(retCmd) - except SocketClosed: - return - - mangle_thread = threading.Thread(target=mangle_and_respond, - args=(msg,)) - mangle_thread.start() - - self.int_thread = threading.Thread(target=run_macro) - self.int_thread.start() - - -ActiveStorage = namedtuple("ActiveStorage", ["type", "storage_id", "prefix"]) - -def _serialize_storage(stype, prefix): - return "{}|{}".format(stype, prefix) - -class ProxyClient: - def __init__(self, binary=None, debug=False, conn_addr=None): - self.binloc = binary - self.proxy_proc = None - self.ltype = None - self.laddr = None - self.debug = debug - self.conn_addr = conn_addr - - self.conns = set() - self.msg_conn = None # conn for single req/rsp messages - - self.context = RequestContext(self) - - self.storage_by_id = {} - self.storage_by_prefix = {} - self.proxy_storage = None - - self.reqrsp_methods = { - "submit_command", - #"reqrsp_cmd", - "ping", - #"submit", - #"save_new", - #"query_storage", - #"req_by_id", - "set_scope", - "get_scope", - # "add_tag", - # "remove_tag", - # "clear_tag", - "all_saved_queries", - "save_query", - "load_query", - "delete_query", - "add_listener", - "remove_listener", - "get_listeners", - "load_certificates", - "set_certificates", - "clear_certificates", - "generate_certificates", - "generate_pem_certificates", - "validate_query", - "list_storage", - # "add_sqlite_storage", - # "add_in_memory_storage", - # "close_storage", - # "set_proxy_storage", - "set_proxy" - } - - def __enter__(self): - if self.conn_addr is not None: - self.msg_connect(self.conn_addr) - else: - self.execute_binary(binary=self.binloc, debug=self.debug) - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def __getattr__(self, name): - if name in self.reqrsp_methods: - return getattr(self.msg_conn, name) - raise NotImplementedError(name) - - @property - def maddr(self): - if self.ltype is not None: - return "{}:{}".format(self.ltype, self.laddr) - else: - return None - - def execute_binary(self, binary=None, debug=False, listen_addr=None): - self.binloc = binary - args = [self.binloc] - if listen_addr is not None: - args += ["--msglisten", listen_addr] - else: - args += ["--msgauto"] - - if debug: - args += ["--dbg"] - self.proxy_proc = Popen(args, stdout=PIPE, stderr=PIPE) - - # Wait for it to start and make connection - listenstr = self.proxy_proc.stdout.readline().rstrip() - self.msg_connect(listenstr.decode()) - - def msg_connect(self, addr): - self.ltype, self.laddr = addr.split(":", 1) - self.msg_conn = self.new_conn() - self._get_storage() - - def close(self): - conns = list(self.conns) - for conn in conns: - conn.close() - if self.proxy_proc is not None: - self.proxy_proc.terminate() - - def new_conn(self): - conn = ProxyConnection(kind=self.ltype, addr=self.laddr) - conn.parent_client = self - conn.debug = self.debug - self.conns.add(conn) - return conn - - # functions involving storage - - def _add_storage(self, storage, prefix): - self.storage_by_prefix[prefix] = storage - self.storage_by_id[storage.storage_id] = storage - - def _clear_storage(self): - self.storage_by_prefix = {} - self.storage_by_id = {} - - def _get_storage(self): - self._clear_storage() - storages = self.list_storage() - for s in storages: - stype, prefix = s.description.split("|") - storage = ActiveStorage(stype, s.storage_id, prefix) - self._add_storage(storage, prefix) - - def parse_reqid(self, reqid): - if reqid[0].isalpha(): - prefix = reqid[0] - realid = reqid[1:] - else: - prefix = "" - realid = reqid - storage = self.storage_by_prefix[prefix] - return storage, realid - - def get_reqid(self, req): - storage = self.storage_by_id[req.storage_id] - return storage.prefix + req.db_id - - def storage_iter(self): - for _, s in self.storage_by_id.items(): - yield s - - def _stg_or_def(self, storage): - if storage is None: - return self.proxy_storage - return storage - - def in_context_requests(self, headers_only=False, max_results=0): - results = self.query_storage(self.context.query, - headers_only=headers_only, - max_results=max_results) - ret = results - if max_results > 0 and len(results) > max_results: - ret = results[:max_results] - return ret - - def in_context_requests_iter(self, headers_only=False, max_results=0): - results = self.query_storage(self.context.query, - headers_only=headers_only, - max_results=max_results) - ret = results - if max_results > 0 and len(results) > max_results: - ret = results[:max_results] - for reqh in ret: - req = self.req_by_id(reqh.db_id, storage_id=reqh.storage_id) - yield req - - def prefixed_reqid(self, req): - prefix = "" - if req.storage_id in self.storage_by_id: - s = self.storage_by_id[req.storage_id] - prefix = s.prefix - return "{}{}".format(prefix, req.db_id) - - # functions that don't just pass through to underlying conn - - def add_sqlite_storage(self, path, prefix): - desc = _serialize_storage("sqlite", prefix) - sid = self.msg_conn.add_sqlite_storage(path, desc) - s = ActiveStorage(type="sqlite", storage_id=sid, prefix=prefix) - self._add_storage(s, prefix) - return s - - def add_in_memory_storage(self, prefix): - desc = _serialize_storage("inmem", prefix) - sid = self.msg_conn.add_in_memory_storage(desc) - s = ActiveStorage(type="inmem", storage_id=sid, prefix=prefix) - self._add_storage(s, prefix) - return s - - def close_storage(self, storage_id): - s = self.storage_by_id[storage_id] - self.msg_conn.close_storage(s.storage_id) - del self.storage_by_id[s.storage_id] - del self.storage_by_prefix[s.storage_prefix] - - def set_proxy_storage(self, storage_id): - s = self.storage_by_id[storage_id] - self.msg_conn.set_proxy_storage(s.storage_id) - self.proxy_storage = storage_id - - def save_new(self, req, storage=None): - self.msg_conn.save_new(req, storage=self._stg_or_def(storage)) - - def submit(self, req, storage=None): - self.msg_conn.submit(req, storage=self._stg_or_def(storage)) - - def query_storage(self, q, max_results=0, headers_only=False, storage=None): - results = [] - if storage is None: - for s in self.storage_iter(): - results += self.msg_conn.query_storage(q, max_results=max_results, - headers_only=headers_only, - storage=s.storage_id) - else: - results += self.msg_conn.query_storage(q, max_results=max_results, - headers_only=headers_only, - storage=storage) - results.sort(key=lambda req: req.time_start) - results = [r for r in reversed(results)] - return results - - def req_by_id(self, reqid, storage_id=None, headers_only=False): - if storage_id is None: - storage, db_id = self.parse_reqid(reqid) - storage_id = storage.storage_id - else: - db_id = reqid - return self.msg_conn.req_by_id(db_id, headers_only=headers_only, - storage=storage_id) - - # for these and submit, might need storage stored on the request itself - def add_tag(self, reqid, tag, storage=None): - self.msg_conn.add_tag(reqid, tag, storage=self._stg_or_def(storage)) - - def remove_tag(self, reqid, tag, storage=None): - self.msg_conn.remove_tag(reqid, tag, storage=self._stg_or_def(storage)) - - def clear_tag(self, reqid, storage=None): - self.msg_conn.clear_tag(reqid, storage=self._stg_or_def(storage)) - - def all_saved_queries(self, storage=None): - self.msg_conn.all_saved_queries(storage=None) - - def save_query(self, name, filt, storage=None): - self.msg_conn.save_query(name, filt, storage=self._stg_or_def(storage)) - - def load_query(self, name, storage=None): - self.msg_conn.load_query(name, storage=self._stg_or_def(storage)) - - def delete_query(self, name, storage=None): - self.msg_conn.delete_query(name, storage=self._stg_or_def(storage)) - - -def decode_req(result, headers_only=False): - if "StartTime" in result and result["StartTime"] > 0: - time_start = time_from_nsecs(result["StartTime"]) - else: - time_start = None - - if "EndTime" in result and result["EndTime"] > 0: - time_end = time_from_nsecs(result["EndTime"]) - else: - time_end = None - - if "DbId" in result: - db_id = result["DbId"] - else: - db_id = "" - - if "Tags" in result: - tags = result["Tags"] - else: - tags = "" - - ret = HTTPRequest( - method=result["Method"], - path=result["Path"], - proto_major=result["ProtoMajor"], - proto_minor=result["ProtoMinor"], - headers=copy.deepcopy(result["Headers"]), - body=base64.b64decode(result["Body"]), - dest_host=result["DestHost"], - dest_port=result["DestPort"], - use_tls=result["UseTLS"], - time_start=time_start, - time_end=time_end, - tags=tags, - headers_only=headers_only, - db_id=db_id, - ) - - if "Unmangled" in result: - ret.unmangled = decode_req(result["Unmangled"], headers_only=headers_only) - if "Response" in result: - ret.response = decode_rsp(result["Response"], headers_only=headers_only) - if "WSMessages" in result: - for wsm in result["WSMessages"]: - ret.ws_messages.append(decode_ws(wsm)) - return ret - -def decode_rsp(result, headers_only=False): - ret = HTTPResponse( - status_code=result["StatusCode"], - reason=result["Reason"], - proto_major=result["ProtoMajor"], - proto_minor=result["ProtoMinor"], - headers=copy.deepcopy(result["Headers"]), - body=base64.b64decode(result["Body"]), - headers_only=headers_only, - ) - - if "Unmangled" in result: - ret.unmangled = decode_rsp(result["Unmangled"], headers_only=headers_only) - return ret - -def decode_ws(result): - timestamp = None - db_id = "" - - if "Timestamp" in result: - timestamp = time_from_nsecs(result["Timestamp"]) - if "DbId" in result: - db_id = result["DbId"] - - ret = WSMessage( - is_binary=result["IsBinary"], - message=base64.b64decode(result["Message"]), - to_server=result["ToServer"], - timestamp=timestamp, - db_id=db_id, - ) - - if "Unmangled" in result: - ret.unmangled = decode_ws(result["Unmangled"]) - - return ret - -def encode_req(req, int_rsp=False): - msg = { - "DestHost": req.dest_host, - "DestPort": req.dest_port, - "UseTLS": req.use_tls, - "Method": req.method, - "Path": req.url.geturl(), - "ProtoMajor": req.proto_major, - "ProtoMinor": req.proto_major, - "Headers": req.headers.dict(), - "Body": base64.b64encode(copy.copy(req.body)).decode(), - } - - if not int_rsp: - msg["StartTime"] = time_to_nsecs(req.time_start) - msg["EndTime"] = time_to_nsecs(req.time_end) - if req.unmangled is not None: - msg["Unmangled"] = encode_req(req.unmangled) - if req.response is not None: - msg["Response"] = encode_rsp(req.response) - msg["WSMessages"] = [] - for wsm in req.ws_messages: - msg["WSMessages"].append(encode_ws(wsm)) - return msg - -def encode_rsp(rsp, int_rsp=False): - msg = { - "ProtoMajor": rsp.proto_major, - "ProtoMinor": rsp.proto_minor, - "StatusCode": rsp.status_code, - "Reason": rsp.reason, - "Headers": rsp.headers.dict(), - "Body": base64.b64encode(copy.copy(rsp.body)).decode(), - } - - if not int_rsp: - if rsp.unmangled is not None: - msg["Unmangled"] = encode_rsp(rsp.unmangled) - return msg - -def encode_ws(ws, int_rsp=False): - msg = { - "Message": base64.b64encode(ws.message).decode(), - "IsBinary": ws.is_binary, - "toServer": ws.to_server, - } - if not int_rsp: - if ws.unmangled is not None: - msg["Unmangled"] = encode_ws(ws.unmangled) - msg["Timestamp"] = time_to_nsecs(ws.timestamp) - msg["DbId"] = ws.db_id - return msg - -def time_from_nsecs(nsecs): - secs = nsecs/1000000000 - t = datetime.datetime.utcfromtimestamp(secs) - return t - -def time_to_nsecs(t): - if t is None: - return None - secs = (t-datetime.datetime(1970,1,1)).total_seconds() - return int(math.floor(secs * 1000000000)) - -RequestStatusLine = namedtuple("RequestStatusLine", ["method", "path", "proto_major", "proto_minor"]) -ResponseStatusLine = namedtuple("ResponseStatusLine", ["proto_major", "proto_minor", "status_code", "reason"]) - -def parse_req_sline(sline): - if len(sline.split(b' ')) == 3: - verb, path, version = sline.split(b' ') - elif len(parts) == 2: - verb, version = parts.split(b' ') - path = b'' - else: - raise ParseError("malformed statusline") - raw_version = version[5:] # strip HTTP/ - pmajor, pminor = raw_version.split(b'.', 1) - return RequestStatusLine(verb.decode(), path.decode(), int(pmajor), int(pminor)) - -def parse_rsp_sline(sline): - if len(sline.split(b' ')) > 2: - version, status_code, reason = sline.split(b' ', 2) - else: - version, status_code = sline.split(b' ', 1) - reason = '' - raw_version = version[5:] # strip HTTP/ - pmajor, pminor = raw_version.split(b'.', 1) - return ResponseStatusLine(int(pmajor), int(pminor), int(status_code), reason.decode()) - -def _parse_message(bs, sline_parser): - header_env, body = re.split(b"\r?\n\r?\n", bs, 1) - status_line, header_bytes = re.split(b"\r?\n", header_env, 1) - h = Headers() - for l in re.split(b"\r?\n", header_bytes): - k, v = l.split(b": ", 1) - if k.lower != 'content-length': - h.add(k.decode(), v.decode()) - h.add("Content-Length", str(len(body))) - return (sline_parser(status_line), h, body) - -def parse_request(bs, dest_host='', dest_port=80, use_tls=False): - req_sline, headers, body = _parse_message(bs, parse_req_sline) - req = HTTPRequest( - method=req_sline.method, - path=req_sline.path, - proto_major=req_sline.proto_major, - proto_minor=req_sline.proto_minor, - headers=headers.dict(), - body=body, - dest_host=dest_host, - dest_port=dest_port, - use_tls=use_tls, - ) - return req - -def parse_response(bs): - rsp_sline, headers, body = _parse_message(bs, parse_rsp_sline) - rsp = HTTPResponse( - status_code=rsp_sline.status_code, - reason=rsp_sline.reason, - proto_major=rsp_sline.proto_major, - proto_minor=rsp_sline.proto_minor, - headers=headers.dict(), - body=body, - ) - return rsp diff --git a/python/puppy/puppyproxy/pup.py b/python/puppy/puppyproxy/pup.py deleted file mode 100644 index 67f66ca..0000000 --- a/python/puppy/puppyproxy/pup.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import sys -import time -import os - -from .proxy import HTTPRequest, ProxyClient, MessageError -from .console import interface_loop -from .config import ProxyConfig -from .util import confirm - -def fmt_time(t): - timestr = strftime("%Y-%m-%d %H:%M:%S.%f", t) - return timestr - -def print_msg(msg, title): - print("-"*10 + " " + title + " " + "-"*10) - print(msg.full_message().decode()) - -def print_rsp(rsp): - print_msg(rsp, "RESPONSE") - if rsp.unmangled: - print_msg(rsp, "UNMANGLED RESPONSE") - -def print_ws(ws): - print("ToServer=%s, IsBinary=%s") - print(ws.message) - -def print_req(req): - print_msg(req, "REQUEST") - if req.unmangled: - print_msg(req, "UNMANGLED REQUEST") - if req.response: - print_rsp(req.response) - -def generate_certificates(client, path): - try: - os.makedirs(path, 0o755) - except os.error as e: - if not os.path.isdir(path): - raise e - pkey_file = os.path.join(path, 'server.key') - cert_file = os.path.join(path, 'server.pem') - client.generate_certificates(pkey_file, cert_file) - -def load_certificates(client, path): - client.load_certificates(os.path.join(path, "server.key"), - os.path.join(path, "server.pem")) - -def main(): - parser = argparse.ArgumentParser(description="Puppy client") - parser.add_argument("--binary", nargs=1, help="location of the backend binary") - parser.add_argument("--attach", nargs=1, help="attach to an already running backend") - parser.add_argument("--dbgattach", nargs=1, help="attach to an already running backend and also perform setup") - parser.add_argument('--debug', help='run in debug mode', action='store_true') - parser.add_argument('--lite', help='run in lite mode', action='store_true') - args = parser.parse_args() - - if args.binary is not None and args.attach is not None: - print("Cannot provide both a binary location and an address to connect to") - exit(1) - - if args.binary is not None: - binloc = args.binary[0] - msg_addr = None - elif args.attach is not None or args.dbgattach: - binloc = None - if args.attach is not None: - msg_addr = args.attach[0] - if args.dbgattach is not None: - msg_addr = args.dbgattach[0] - else: - msg_addr = None - try: - gopath = os.environ["GOPATH"] - binloc = os.path.join(gopath, "bin", "puppy") - except: - print("Could not find puppy binary in GOPATH. Please ensure that it has been compiled, or pass in the binary location from the command line") - exit(1) - data_dir = os.path.join(os.path.expanduser('~'), '.puppy') - config = ProxyConfig() - if not args.lite: - config.load("./config.json") - cert_dir = os.path.join(data_dir, "certs") - - with ProxyClient(binary=binloc, conn_addr=msg_addr, debug=args.debug) as client: - try: - load_certificates(client, cert_dir) - except MessageError as e: - print(str(e)) - if(confirm("Would you like to generate the certificates now?", "y")): - generate_certificates(client, cert_dir) - print("Certificates generated to {}".format(cert_dir)) - print("Be sure to add {} to your trusted CAs in your browser!".format(os.path.join(cert_dir, "server.pem"))) - load_certificates(client, cert_dir) - else: - print("Can not run proxy without SSL certificates") - exit(1) - try: - # Only try and listen/set default storage if we're not attaching - if args.attach is None: - if args.lite: - storage = client.add_in_memory_storage("") - else: - storage = client.add_sqlite_storage("./data.db", "") - - client.disk_storage = storage - client.inmem_storage = client.add_in_memory_storage("m") - client.set_proxy_storage(storage.storage_id) - - for iface, port in config.listeners: - try: - client.add_listener(iface, port) - except MessageError as e: - print(str(e)) - - # Set upstream proxy - if config.use_proxy: - client.set_proxy(config.use_proxy, - config.proxy_host, - config.proxy_port, - config.is_socks_proxy) - interface_loop(client) - except MessageError as e: - print(str(e)) - -if __name__ == "__main__": - main() - -def start(): - main() diff --git a/python/puppy/puppyproxy/templates/macro.py.tmpl b/python/puppy/puppyproxy/templates/macro.py.tmpl deleted file mode 100644 index b6f0dee..0000000 --- a/python/puppy/puppyproxy/templates/macro.py.tmpl +++ /dev/null @@ -1,22 +0,0 @@ -{% include 'macroheader.py.tmpl' %} -{% if req_lines %} -########### -## Requests -# It's suggested that you call .copy() on these and then edit attributes -# as needed to create modified requests -## -{% for lines, params in zip(req_lines, req_params) %} -req{{ loop.index }} = parse_request(({% for line in lines %} - {{ line }}{% endfor %} -), {{ params }}) -{% endfor %}{% endif %} - -def run_macro(client, args): - # Example: - """ - req = req1.copy() # Copy req1 - client.submit(req) # Submit the request to get a response - print(req.response.full_message()) # print the response - client.save_new(req) # save the request to the data file - """ - pass diff --git a/python/puppy/puppyproxy/templates/macroheader.py.tmpl b/python/puppy/puppyproxy/templates/macroheader.py.tmpl deleted file mode 100644 index db21b03..0000000 --- a/python/puppy/puppyproxy/templates/macroheader.py.tmpl +++ /dev/null @@ -1 +0,0 @@ -from puppyproxy.proxy import parse_request, parse_response \ No newline at end of file diff --git a/python/puppy/puppyproxy/util.py b/python/puppy/puppyproxy/util.py deleted file mode 100644 index 35ec28b..0000000 --- a/python/puppy/puppyproxy/util.py +++ /dev/null @@ -1,311 +0,0 @@ -import sys -import string -import time -import datetime -import base64 -from pygments.formatters import TerminalFormatter -from pygments.lexers import get_lexer_for_mimetype, HttpLexer -from pygments import highlight -from .colors import Colors, Styles, verb_color, scode_color, path_formatter, color_string - - -def str_hash_code(s): - h = 0 - n = len(s)-1 - for c in s.encode(): - h += c*31**n - n -= 1 - return h - -def printable_data(data, colors=True): - """ - Return ``data``, but replaces unprintable characters with periods. - - :param data: The data to make printable - :type data: String - :rtype: String - """ - chars = [] - colored = False - for c in data: - if chr(c) in string.printable: - if colored and colors: - chars.append(Colors.ENDC) - colored = False - chars.append(chr(c)) - else: - if (not colored) and colors: - chars.append(Styles.UNPRINTABLE_DATA) - colored = True - chars.append('.') - if colors: - chars.append(Colors.ENDC) - return ''.join(chars) - -def remove_color(s): - ansi_escape = re.compile(r'\x1b[^m]*m') - return ansi_escape.sub('', s) - -def hexdump(src, length=16): - FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.' for x in range(256)]) - lines = [] - for c in range(0, len(src), length): - chars = src[c:c+length] - hex = ' '.join(["%02x" % x for x in chars]) - printable = ''.join(["%s" % ((x <= 127 and FILTER[x]) or Styles.UNPRINTABLE_DATA+'.'+Colors.ENDC) for x in chars]) - lines.append("%04x %-*s %s\n" % (c, length*3, hex, printable)) - return ''.join(lines) - -def maybe_hexdump(s): - if any(chr(c) not in string.printable for c in s): - return hexdump(s) - return s - -def print_table(coldata, rows): - """ - Print a table. - Coldata: List of dicts with info on how to print the columns. - ``name`` is the heading to give column, - ``width (optional)`` maximum width before truncating. 0 for unlimited. - - Rows: List of tuples with the data to print - """ - - # Get the width of each column - widths = [] - headers = [] - for data in coldata: - if 'name' in data: - headers.append(data['name']) - else: - headers.append('') - empty_headers = True - for h in headers: - if h != '': - empty_headers = False - if not empty_headers: - rows = [headers] + rows - - for i in range(len(coldata)): - col = coldata[i] - if 'width' in col and col['width'] > 0: - maxwidth = col['width'] - else: - maxwidth = 0 - colwidth = 0 - for row in rows: - printdata = row[i] - if isinstance(printdata, dict): - collen = len(str(printdata['data'])) - else: - collen = len(str(printdata)) - if collen > colwidth: - colwidth = collen - if maxwidth > 0 and colwidth > maxwidth: - widths.append(maxwidth) - else: - widths.append(colwidth) - - # Print rows - padding = 2 - is_heading = not empty_headers - for row in rows: - if is_heading: - sys.stdout.write(Styles.TABLE_HEADER) - for (col, width) in zip(row, widths): - if isinstance(col, dict): - printstr = str(col['data']) - if 'color' in col: - colors = col['color'] - formatter = None - elif 'formatter' in col: - colors = None - formatter = col['formatter'] - else: - colors = None - formatter = None - else: - printstr = str(col) - colors = None - formatter = None - if len(printstr) > width: - trunc_printstr=printstr[:width] - trunc_printstr=trunc_printstr[:-3]+'...' - else: - trunc_printstr=printstr - if colors is not None: - sys.stdout.write(colors) - sys.stdout.write(trunc_printstr) - sys.stdout.write(Colors.ENDC) - elif formatter is not None: - toprint = formatter(printstr, width) - sys.stdout.write(toprint) - else: - sys.stdout.write(trunc_printstr) - sys.stdout.write(' '*(width-len(printstr))) - sys.stdout.write(' '*padding) - if is_heading: - sys.stdout.write(Colors.ENDC) - is_heading = False - sys.stdout.write('\n') - sys.stdout.flush() - -def print_requests(requests, client=None): - """ - Takes in a list of requests and prints a table with data on each of the - requests. It's the same table that's used by ``ls``. - """ - rows = [] - for req in requests: - rows.append(get_req_data_row(req, client=client)) - print_request_rows(rows) - -def print_request_rows(request_rows): - """ - Takes in a list of request rows generated from :func:`pappyproxy.console.get_req_data_row` - and prints a table with data on each of the - requests. Used instead of :func:`pappyproxy.console.print_requests` if you - can't count on storing all the requests in memory at once. - """ - # Print a table with info on all the requests in the list - cols = [ - {'name':'ID'}, - {'name':'Verb'}, - {'name': 'Host'}, - {'name':'Path', 'width':40}, - {'name':'S-Code', 'width':16}, - {'name':'Req Len'}, - {'name':'Rsp Len'}, - {'name':'Time'}, - {'name':'Mngl'}, - ] - print_rows = [] - for row in request_rows: - (reqid, verb, host, path, scode, qlen, slen, time, mngl) = row - - verb = {'data':verb, 'color':verb_color(verb)} - scode = {'data':scode, 'color':scode_color(scode)} - host = {'data':host, 'color':color_string(host, color_only=True)} - path = {'data':path, 'formatter':path_formatter} - - print_rows.append((reqid, verb, host, path, scode, qlen, slen, time, mngl)) - print_table(cols, print_rows) - -def get_req_data_row(request, client=None): - """ - Get the row data for a request to be printed. - """ - if client is not None: - rid = client.prefixed_reqid(request) - else: - rid = request.db_id - method = request.method - host = request.dest_host - path = request.url.geturl() - reqlen = request.content_length - rsplen = 'N/A' - mangle_str = '--' - - if request.unmangled: - mangle_str = 'q' - - if request.response: - response_code = str(request.response.status_code) + \ - ' ' + request.response.reason - rsplen = request.response.content_length - if request.response.unmangled: - if mangle_str == '--': - mangle_str = 's' - else: - mangle_str += '/s' - else: - response_code = '' - - time_str = '--' - if request.time_start and request.time_end: - time_delt = request.time_end - request.time_start - time_str = "%.2f" % time_delt.total_seconds() - - return [rid, method, host, path, response_code, - reqlen, rsplen, time_str, mangle_str] - -def confirm(message, default='n'): - """ - A helper function to get confirmation from the user. It prints ``message`` - then asks the user to answer yes or no. Returns True if the user answers - yes, otherwise returns False. - """ - if 'n' in default.lower(): - default = False - else: - default = True - - print(message) - if default: - answer = input('(Y/n) ') - else: - answer = input('(y/N) ') - - - if not answer: - return default - - if answer[0].lower() == 'y': - return True - else: - return False - -# Taken from http://stackoverflow.com/questions/4770297/python-convert-utc-datetime-string-to-local-datetime -def utc2local(utc): - epoch = time.mktime(utc.timetuple()) - offset = datetime.datetime.fromtimestamp(epoch) - datetime.datetime.utcfromtimestamp(epoch) - return utc + offset - -def datetime_string(dt): - dtobj = utc2local(dt) - time_made_str = dtobj.strftime('%a, %b %d, %Y, %I:%M:%S.%f %p') - return time_made_str - -def copy_to_clipboard(text): - from .clip import copy - copy(text) - -def clipboard_contents(): - from .clip import paste - return paste() - -def encode_basic_auth(username, password): - decoded = '%s:%s' % (username, password) - encoded = base64.b64encode(decoded.encode()) - header = 'Basic %s' % encoded.decode() - return header - -def parse_basic_auth(header): - """ - Parse a raw basic auth header and return (username, password) - """ - _, creds = header.split(' ', 1) - decoded = base64.b64decode(creds) - username, password = decoded.split(':', 1) - return (username, password) - -def print_query(query): - for p in query: - fstrs = [] - for f in p: - fstrs.append(' '.join(f)) - - print((Colors.BLUE+' OR '+Colors.ENDC).join(fstrs)) - -def log_error(msg): - print(msg) - -def autocomplete_startswith(text, lst, allow_spaces=False): - ret = None - if not text: - ret = lst[:] - else: - ret = [n for n in lst if n.startswith(text)] - if not allow_spaces: - ret = [s for s in ret if ' ' not in s] - return ret diff --git a/python/puppy/setup.py b/python/puppy/setup.py deleted file mode 100755 index 60c4af8..0000000 --- a/python/puppy/setup.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -import pkgutil -#import pappyproxy -from setuptools import setup, find_packages - -VERSION = "0.0.1" - -setup(name='puppyproxy', - version=VERSION, - description='The Puppy Intercepting Proxy', - author='Rob Glew', - author_email='rglew56@gmail.com', - #url='https://www.github.com/roglew/puppy-proxy', - packages=['puppyproxy'], - include_package_data = True, - license='MIT', - entry_points = { - 'console_scripts':['puppy = puppyproxy.pup:start'], - }, - #long_description=open('docs/source/overview.rst').read(), - long_description="The Puppy Proxy", - keywords='http proxy hacking 1337hax pwnurmum', - #download_url='https://github.com/roglew/pappy-proxy/archive/%s.tar.gz'%VERSION, - install_requires=[ - 'cmd2>=0.6.8', - 'Jinja2>=2.8', - 'pygments>=2.0.2', - ], - classifiers=[ - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Operating System :: MacOS', - 'Operating System :: POSIX :: Linux', - 'Development Status :: 2 - Pre-Alpha', - 'Programming Language :: Python :: 3.6', - 'License :: OSI Approved :: MIT License', - 'Topic :: Security', - ] -) diff --git a/schema.go b/schema.go index 3b2f6a5..4b12965 100644 --- a/schema.go +++ b/schema.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "database/sql" diff --git a/search.go b/search.go index aff6b21..518a261 100644 --- a/search.go +++ b/search.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "errors" @@ -14,8 +14,8 @@ import ( type SearchField int type StrComparer int -type StrFieldGetter func(req *ProxyRequest) ([]string, error) -type KvFieldGetter func(req *ProxyRequest) ([]*PairValue, error) +type strFieldGetter func(req *ProxyRequest) ([]string, error) +type kvFieldGetter func(req *ProxyRequest) ([]*PairValue, error) type RequestChecker func(req *ProxyRequest) bool @@ -37,6 +37,7 @@ const ( FieldPath FieldURL FieldStatusCode + FieldTag FieldBothParam FieldURLParam @@ -44,7 +45,6 @@ const ( FieldResponseCookie FieldRequestCookie FieldBothCookie - FieldTag FieldAfter FieldBefore @@ -72,13 +72,17 @@ type PairValue struct { value string } -type QueryPhrase [][]interface{} // A list of queries. Will match if any queries match the request -type MessageQuery []QueryPhrase // A list of phrases. Will match if all the phrases match the request +// A list of queries. Will match if any queries match the request +type QueryPhrase [][]interface{} +// A list of phrases. Will match if all the phrases match the request +type MessageQuery []QueryPhrase +// A list of queries in string form. Will match if any queries match the request type StrQueryPhrase [][]string +// A list of phrases in string form. Will match if all the phrases match the request type StrMessageQuery []StrQueryPhrase -// Return a function that returns whether a request matches the given condition +// Return a function that returns whether a request matches the given conditions func NewRequestChecker(args ...interface{}) (RequestChecker, error) { // Generates a request checker from the given search arguments if len(args) == 0 { @@ -94,7 +98,7 @@ func NewRequestChecker(args ...interface{}) (RequestChecker, error) { // Normal string fields case FieldAll, FieldRequestBody, FieldResponseBody, FieldAllBody, FieldWSMessage, FieldMethod, FieldHost, FieldPath, FieldStatusCode, FieldTag, FieldId: - getter, err := CreateStrFieldGetter(field) + getter, err := createstrFieldGetter(field) if err != nil { return nil, fmt.Errorf("error performing search: %s", err.Error()) } @@ -108,11 +112,11 @@ func NewRequestChecker(args ...interface{}) (RequestChecker, error) { return nil, errors.New("comparer must be a StrComparer") } - return GenStrFieldChecker(getter, comparer, args[2]) + return genStrFieldChecker(getter, comparer, args[2]) // Normal key/value fields case FieldRequestHeaders, FieldResponseHeaders, FieldBothHeaders, FieldBothParam, FieldURLParam, FieldPostParam, FieldResponseCookie, FieldRequestCookie, FieldBothCookie: - getter, err := CreateKvPairGetter(field) + getter, err := createKvPairGetter(field) if err != nil { return nil, fmt.Errorf("error performing search: %s", err.Error()) } @@ -124,7 +128,7 @@ func NewRequestChecker(args ...interface{}) (RequestChecker, error) { return nil, errors.New("comparer must be a StrComparer") } - // Create a StrFieldGetter out of our key/value getter + // Create a strFieldGetter out of our key/value getter strgetter := func(req *ProxyRequest) ([]string, error) { pairs, err := getter(req) if err != nil { @@ -134,7 +138,7 @@ func NewRequestChecker(args ...interface{}) (RequestChecker, error) { } // return a str field checker using our new str getter - return GenStrFieldChecker(strgetter, comparer, args[2]) + return genStrFieldChecker(strgetter, comparer, args[2]) } else if len(args) == 5 { // Get comparer and value out of function arguments comparer1, ok := args[1].(StrComparer) @@ -158,7 +162,7 @@ func NewRequestChecker(args ...interface{}) (RequestChecker, error) { } // Create a checker out of our getter, comparers, and vals - return GenKvFieldChecker(getter, comparer1, val1, comparer2, val2) + return genKvFieldChecker(getter, comparer1, val1, comparer2, val2) } else { return nil, errors.New("invalid number of arguments for a key/value search") } @@ -225,9 +229,7 @@ func NewRequestChecker(args ...interface{}) (RequestChecker, error) { } } -func CreateStrFieldGetter(field SearchField) (StrFieldGetter, error) { - // Returns a function to pull the relevant strings out of the request - +func createstrFieldGetter(field SearchField) (strFieldGetter, error) { switch field { case FieldAll: return func(req *ProxyRequest) ([]string, error) { @@ -310,6 +312,10 @@ func CreateStrFieldGetter(field SearchField) (StrFieldGetter, error) { } return strs, nil }, nil + case FieldTag: + return func(req *ProxyRequest) ([]string, error) { + return req.Tags(), nil + }, nil case FieldId: return func(req *ProxyRequest) ([]string, error) { strs := make([]string, 1) @@ -321,8 +327,7 @@ func CreateStrFieldGetter(field SearchField) (StrFieldGetter, error) { } } -func GenStrChecker(cmp StrComparer, argval interface{}) (func(str string) bool, error) { - // Create a function to check if a string matches a value using the given comparer +func genStrChecker(cmp StrComparer, argval interface{}) (func(str string) bool, error) { switch cmp { case StrContains: val, ok := argval.(string) @@ -396,10 +401,9 @@ func GenStrChecker(cmp StrComparer, argval interface{}) (func(str string) bool, } } -func GenStrFieldChecker(strGetter StrFieldGetter, cmp StrComparer, val interface{}) (RequestChecker, error) { - // Generates a request checker from a string getter, a comparer, and a value +func genStrFieldChecker(strGetter strFieldGetter, cmp StrComparer, val interface{}) (RequestChecker, error) { getter := strGetter - comparer, err := GenStrChecker(cmp, val) + comparer, err := genStrChecker(cmp, val) if err != nil { return nil, err } @@ -462,8 +466,7 @@ func pairsToStrings(pairs []*PairValue) []string { return strs } -func CreateKvPairGetter(field SearchField) (KvFieldGetter, error) { - // Returns a function to pull the relevant pairs out of the request +func createKvPairGetter(field SearchField) (kvFieldGetter, error) { switch field { case FieldRequestHeaders: return func(req *ProxyRequest) ([]*PairValue, error) { @@ -535,15 +538,15 @@ func CreateKvPairGetter(field SearchField) (KvFieldGetter, error) { } } -func GenKvFieldChecker(kvGetter KvFieldGetter, cmp1 StrComparer, val1 string, +func genKvFieldChecker(kvGetter kvFieldGetter, cmp1 StrComparer, val1 string, cmp2 StrComparer, val2 string) (RequestChecker, error) { getter := kvGetter - cmpfunc1, err := GenStrChecker(cmp1, val1) + cmpfunc1, err := genStrChecker(cmp1, val1) if err != nil { return nil, err } - cmpfunc2, err := GenStrChecker(cmp2, val2) + cmpfunc2, err := genStrChecker(cmp2, val2) if err != nil { return nil, err } @@ -563,7 +566,7 @@ func GenKvFieldChecker(kvGetter KvFieldGetter, cmp1 StrComparer, val1 string, }, nil } -func CheckerFromPhrase(phrase QueryPhrase) (RequestChecker, error) { +func checkerFromPhrase(phrase QueryPhrase) (RequestChecker, error) { checkers := make([]RequestChecker, len(phrase)) for i, args := range phrase { newChecker, err := NewRequestChecker(args...) @@ -585,10 +588,11 @@ func CheckerFromPhrase(phrase QueryPhrase) (RequestChecker, error) { return ret, nil } +// Creates a RequestChecker from a MessageQuery func CheckerFromMessageQuery(query MessageQuery) (RequestChecker, error) { checkers := make([]RequestChecker, len(query)) for i, phrase := range query { - newChecker, err := CheckerFromPhrase(phrase) + newChecker, err := checkerFromPhrase(phrase) if err != nil { return nil, fmt.Errorf("error with phrase %d: %s", i, err.Error()) } @@ -611,7 +615,7 @@ func CheckerFromMessageQuery(query MessageQuery) (RequestChecker, error) { StringSearch conversions */ -func FieldGoToString(field SearchField) (string, error) { +func fieldGoToString(field SearchField) (string, error) { switch field { case FieldAll: return "all", nil @@ -668,7 +672,7 @@ func FieldGoToString(field SearchField) (string, error) { } } -func FieldStrToGo(field string) (SearchField, error) { +func fieldStrToGo(field string) (SearchField, error) { switch strings.ToLower(field) { case "all": return FieldAll, nil @@ -725,7 +729,8 @@ func FieldStrToGo(field string) (SearchField, error) { } } -func CmpValGoToStr(comparer StrComparer, val interface{}) (string, string, error) { +// Converts a StrComparer and a value into a comparer and value that can be used in string queries +func cmpValGoToStr(comparer StrComparer, val interface{}) (string, string, error) { var cmpStr string switch comparer { case StrIs: @@ -775,7 +780,7 @@ func CmpValGoToStr(comparer StrComparer, val interface{}) (string, string, error } } -func CmpValStrToGo(strArgs []string) (StrComparer, interface{}, error) { +func cmpValStrToGo(strArgs []string) (StrComparer, interface{}, error) { if len(strArgs) != 2 { return 0, "", fmt.Errorf("parsing a comparer/val requires one comparer and one value. Got %d arguments.", len(strArgs)) } @@ -817,7 +822,7 @@ func CheckArgsStrToGo(strArgs []string) ([]interface{}, error) { } // Parse the field - field, err := FieldStrToGo(strArgs[0]) + field, err := fieldStrToGo(strArgs[0]) if err != nil { return nil, err } @@ -832,7 +837,7 @@ func CheckArgsStrToGo(strArgs []string) ([]interface{}, error) { return nil, errors.New("string field searches require one comparer and one value") } - cmp, val, err := CmpValStrToGo(remaining) + cmp, val, err := cmpValStrToGo(remaining) if err != nil { return nil, err } @@ -841,21 +846,21 @@ func CheckArgsStrToGo(strArgs []string) ([]interface{}, error) { // Normal key/value fields case FieldRequestHeaders, FieldResponseHeaders, FieldBothHeaders, FieldBothParam, FieldURLParam, FieldPostParam, FieldResponseCookie, FieldRequestCookie, FieldBothCookie: if len(remaining) == 2 { - cmp, val, err := CmpValStrToGo(remaining) + cmp, val, err := cmpValStrToGo(remaining) if err != nil { return nil, err } args = append(args, cmp) args = append(args, val) } else if len(remaining) == 4 { - cmp, val, err := CmpValStrToGo(remaining[0:2]) + cmp, val, err := cmpValStrToGo(remaining[0:2]) if err != nil { return nil, err } args = append(args, cmp) args = append(args, val) - cmp, val, err = CmpValStrToGo(remaining[2:4]) + cmp, val, err = cmpValStrToGo(remaining[2:4]) if err != nil { return nil, err } @@ -918,7 +923,7 @@ func CheckArgsGoToStr(args []interface{}) ([]string, error) { return nil, errors.New("first argument is not a field") } - strField, err := FieldGoToString(field) + strField, err := fieldGoToString(field) if err != nil { return nil, err } @@ -935,7 +940,7 @@ func CheckArgsGoToStr(args []interface{}) ([]string, error) { return nil, errors.New("comparer must be a StrComparer") } - cmpStr, valStr, err := CmpValGoToStr(comparer, args[2]) + cmpStr, valStr, err := cmpValGoToStr(comparer, args[2]) if err != nil { return nil, err } @@ -950,7 +955,7 @@ func CheckArgsGoToStr(args []interface{}) ([]string, error) { return nil, errors.New("comparer must be a StrComparer") } - cmpStr, valStr, err := CmpValGoToStr(comparer, args[2]) + cmpStr, valStr, err := cmpValGoToStr(comparer, args[2]) if err != nil { return nil, err } @@ -964,7 +969,7 @@ func CheckArgsGoToStr(args []interface{}) ([]string, error) { return nil, errors.New("comparer1 must be a StrComparer") } - cmpStr1, valStr1, err := CmpValGoToStr(comparer1, args[2]) + cmpStr1, valStr1, err := cmpValGoToStr(comparer1, args[2]) if err != nil { return nil, err } @@ -976,7 +981,7 @@ func CheckArgsGoToStr(args []interface{}) ([]string, error) { return nil, errors.New("comparer2 must be a StrComparer") } - cmpStr2, valStr2, err := CmpValGoToStr(comparer2, args[2]) + cmpStr2, valStr2, err := cmpValGoToStr(comparer2, args[2]) if err != nil { return nil, err } @@ -1034,7 +1039,7 @@ func CheckArgsGoToStr(args []interface{}) ([]string, error) { } } -func StrPhraseToGoPhrase(phrase StrQueryPhrase) (QueryPhrase, error) { +func strPhraseToGoPhrase(phrase StrQueryPhrase) (QueryPhrase, error) { goPhrase := make(QueryPhrase, len(phrase)) for i, strArgs := range phrase { var err error @@ -1046,7 +1051,7 @@ func StrPhraseToGoPhrase(phrase StrQueryPhrase) (QueryPhrase, error) { return goPhrase, nil } -func GoPhraseToStrPhrase(phrase QueryPhrase) (StrQueryPhrase, error) { +func goPhraseToStrPhrase(phrase QueryPhrase) (StrQueryPhrase, error) { strPhrase := make(StrQueryPhrase, len(phrase)) for i, goArgs := range phrase { var err error @@ -1058,11 +1063,12 @@ func GoPhraseToStrPhrase(phrase QueryPhrase) (StrQueryPhrase, error) { return strPhrase, nil } -func StrQueryToGoQuery(query StrMessageQuery) (MessageQuery, error) { +// Converts a StrMessageQuery into a MessageQuery +func StrQueryToMsgQuery(query StrMessageQuery) (MessageQuery, error) { goQuery := make(MessageQuery, len(query)) for i, phrase := range query { var err error - goQuery[i], err = StrPhraseToGoPhrase(phrase) + goQuery[i], err = strPhraseToGoPhrase(phrase) if err != nil { return nil, fmt.Errorf("Error with phrase %d: %s", i, err.Error()) } @@ -1070,11 +1076,12 @@ func StrQueryToGoQuery(query StrMessageQuery) (MessageQuery, error) { return goQuery, nil } -func GoQueryToStrQuery(query MessageQuery) (StrMessageQuery, error) { +// Converts a MessageQuery into a StrMessageQuery +func MsgQueryToStrQuery(query MessageQuery) (StrMessageQuery, error) { strQuery := make(StrMessageQuery, len(query)) for i, phrase := range query { var err error - strQuery[i], err = GoPhraseToStrPhrase(phrase) + strQuery[i], err = goPhraseToStrPhrase(phrase) if err != nil { return nil, fmt.Errorf("Error with phrase %d: %s", i, err.Error()) } diff --git a/search_test.go b/search_test.go index 57b9e33..6183c27 100644 --- a/search_test.go +++ b/search_test.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "runtime" diff --git a/signer.go b/signer.go index b5e5be9..2b0431a 100644 --- a/signer.go +++ b/signer.go @@ -1,4 +1,4 @@ -package main +package puppy /* Copyright (c) 2012 Elazar Leibovich. All rights reserved. @@ -56,14 +56,14 @@ import ( counterecryptor.go */ -type CounterEncryptorRand struct { +type counterEncryptorRand struct { cipher cipher.Block counter []byte rand []byte ix int } -func NewCounterEncryptorRandFromKey(key interface{}, seed []byte) (r CounterEncryptorRand, err error) { +func newCounterEncryptorRandFromKey(key interface{}, seed []byte) (r counterEncryptorRand, err error) { var keyBytes []byte switch key := key.(type) { case *rsa.PrivateKey: @@ -85,14 +85,14 @@ func NewCounterEncryptorRandFromKey(key interface{}, seed []byte) (r CounterEncr return } -func (c *CounterEncryptorRand) Seed(b []byte) { +func (c *counterEncryptorRand) Seed(b []byte) { if len(b) != len(c.counter) { panic("SetCounter: wrong counter size") } copy(c.counter, b) } -func (c *CounterEncryptorRand) refill() { +func (c *counterEncryptorRand) refill() { c.cipher.Encrypt(c.rand, c.counter) for i := 0; i < len(c.counter); i++ { if c.counter[i]++; c.counter[i] != 0 { @@ -102,7 +102,7 @@ func (c *CounterEncryptorRand) refill() { c.ix = 0 } -func (c *CounterEncryptorRand) Read(b []byte) (n int, err error) { +func (c *counterEncryptorRand) Read(b []byte) (n int, err error) { if c.ix == len(c.rand) { c.refill() } @@ -137,7 +137,7 @@ func hashSortedBigInt(lst []string) *big.Int { var goproxySignerVersion = ":goroxy1" -func SignHost(ca tls.Certificate, hosts []string) (cert tls.Certificate, err error) { +func signHost(ca tls.Certificate, hosts []string) (cert tls.Certificate, err error) { var x509ca *x509.Certificate // Use the provided ca and not the global GoproxyCa for certificate generation. @@ -173,8 +173,8 @@ func SignHost(ca tls.Certificate, hosts []string) (cert tls.Certificate, err err template.DNSNames = append(template.DNSNames, h) } } - var csprng CounterEncryptorRand - if csprng, err = NewCounterEncryptorRandFromKey(ca.PrivateKey, hash); err != nil { + var csprng counterEncryptorRand + if csprng, err = newCounterEncryptorRandFromKey(ca.PrivateKey, hash); err != nil { return } var certpriv *rsa.PrivateKey diff --git a/sqlitestorage.go b/sqlitestorage.go index 6b42cc1..6b3af2e 100644 --- a/sqlitestorage.go +++ b/sqlitestorage.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "database/sql" @@ -15,9 +15,9 @@ import ( _ "github.com/mattn/go-sqlite3" ) -var REQUEST_SELECT string = "SELECT id, full_request, response_id, unmangled_id, port, is_ssl, host, start_datetime, end_datetime FROM requests" -var RESPONSE_SELECT string = "SELECT id, full_response, unmangled_id FROM responses" -var WS_SELECT string = "SELECT id, parent_request, unmangled_id, is_binary, direction, time_sent, contents FROM websocket_messages" +var request_select string = "SELECT id, full_request, response_id, unmangled_id, port, is_ssl, host, start_datetime, end_datetime FROM requests" +var response_select string = "SELECT id, full_response, unmangled_id FROM responses" +var ws_select string = "SELECT id, parent_request, unmangled_id, is_binary, direction, time_sent, contents FROM websocket_messages" var inmemIdCounter = IdCounter() @@ -223,7 +223,6 @@ func rspFromRow(tx *sql.Tx, ms *SQLiteStorage, id sql.NullInt64, db_full_respons rsp.DbId = strconv.FormatInt(id.Int64, 10) if db_unmangled_id.Valid { - MainLogger.Println(db_unmangled_id.Int64) unmangledRsp, err := ms.loadResponse(tx, strconv.FormatInt(db_unmangled_id.Int64, 10)) if err != nil { return nil, fmt.Errorf("unable to load unmangled response for rspid=%d: %s", id.Int64, err.Error()) @@ -562,7 +561,7 @@ func (ms *SQLiteStorage) loadRequest(tx *sql.Tx, reqid string) (*ProxyRequest, e // SELECT // id, full_request, response_id, unmangled_id, port, is_ssl, host, start_datetime, end_datetime // FROM requests WHERE id=?`, dbId).Scan( - err = tx.QueryRow(REQUEST_SELECT+" WHERE id=?", dbId).Scan( + err = tx.QueryRow(request_select+" WHERE id=?", dbId).Scan( &db_id, &db_full_request, &db_response_id, @@ -844,7 +843,7 @@ func (ms *SQLiteStorage) loadResponse(tx *sql.Tx, rspid string) (*ProxyResponse, var db_full_response []byte var db_unmangled_id sql.NullInt64 - err = tx.QueryRow(RESPONSE_SELECT+" WHERE id=?", dbId).Scan( + err = tx.QueryRow(response_select+" WHERE id=?", dbId).Scan( &db_id, &db_full_response, &db_unmangled_id, @@ -1143,7 +1142,7 @@ func (ms *SQLiteStorage) loadWSMessage(tx *sql.Tx, wsmid string) (*ProxyWSMessag var db_time_sent sql.NullInt64 var db_contents []byte - err = tx.QueryRow(WS_SELECT+" WHERE id=?", dbId).Scan( + err = tx.QueryRow(ws_select+" WHERE id=?", dbId).Scan( &db_id, &db_parent_request, &db_unmangled_id, @@ -1314,7 +1313,7 @@ func (ms *SQLiteStorage) requestKeys(tx *sql.Tx) ([]string, error) { } func (ms *SQLiteStorage) reqSearchHelper(tx *sql.Tx, limit int64, checker RequestChecker, sqlTail string) ([]*ProxyRequest, error) { - rows, err := tx.Query(REQUEST_SELECT + sqlTail + " ORDER BY start_datetime DESC;") + rows, err := tx.Query(request_select + sqlTail + " ORDER BY start_datetime DESC;") if err != nil { return nil, errors.New("error with sql query: " + err.Error()) } @@ -1450,7 +1449,7 @@ func (ms *SQLiteStorage) SaveQuery(name string, query MessageQuery) error { } func (ms *SQLiteStorage) saveQuery(tx *sql.Tx, name string, query MessageQuery) error { - strQuery, err := GoQueryToStrQuery(query) + strQuery, err := MsgQueryToStrQuery(query) if err != nil { return fmt.Errorf("error creating string version of query: %s", err.Error()) } @@ -1516,7 +1515,7 @@ func (ms *SQLiteStorage) loadQuery(tx *sql.Tx, name string) (MessageQuery, error return nil, err } - retQuery, err := StrQueryToGoQuery(strRetQuery) + retQuery, err := StrQueryToMsgQuery(strRetQuery) if err != nil { return nil, err } @@ -1590,7 +1589,7 @@ func (ms *SQLiteStorage) allSavedQueries(tx *sql.Tx) ([]*SavedQuery, error) { var strQuery StrMessageQuery err = json.Unmarshal([]byte(queryStr.String), &strQuery) - goQuery, err := StrQueryToGoQuery(strQuery) + goQuery, err := StrQueryToMsgQuery(strQuery) if err != nil { return nil, err } diff --git a/sqlitestorage_test.go b/sqlitestorage_test.go index f5f6e49..77f88d1 100644 --- a/sqlitestorage_test.go +++ b/sqlitestorage_test.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "runtime" diff --git a/storage.go b/storage.go index 683c2b9..d2048a6 100644 --- a/storage.go +++ b/storage.go @@ -1,13 +1,12 @@ -package main +package puppy import ( "errors" "fmt" ) +// An interface that represents something that can be used to store data from the proxy type MessageStorage interface { - // NOTE: Load func responsible for loading dependent messages, delte function responsible for deleting dependent messages - // if it takes an ID, the storage is responsible for dependent messages // Close the storage Close() @@ -18,8 +17,9 @@ type MessageStorage interface { SaveNewRequest(req *ProxyRequest) error // Load a request given a unique id LoadRequest(reqid string) (*ProxyRequest, error) + // Load the unmangled version of a request given a unique id LoadUnmangledRequest(reqid string) (*ProxyRequest, error) - // Delete a request + // Delete a request given a unique id DeleteRequest(reqid string) error // Update an existing response in the storage. Requires that it has already been saved @@ -28,21 +28,23 @@ type MessageStorage interface { SaveNewResponse(rsp *ProxyResponse) error // Load a response given a unique id LoadResponse(rspid string) (*ProxyResponse, error) + // Load the unmangled version of a response given a unique id LoadUnmangledResponse(rspid string) (*ProxyResponse, error) - // Delete a response + // Delete a response given a unique id DeleteResponse(rspid string) error // Update an existing websocket message in the storage. Requires that it has already been saved UpdateWSMessage(req *ProxyRequest, wsm *ProxyWSMessage) error // Save a new instance of the websocket message in the storage regardless of if it has already been saved SaveNewWSMessage(req *ProxyRequest, wsm *ProxyWSMessage) error - // Load a websocket given a unique id + // Load a websocket message given a unique id LoadWSMessage(wsmid string) (*ProxyWSMessage, error) + // Load the unmangled version of a websocket message given a unique id LoadUnmangledWSMessage(wsmid string) (*ProxyWSMessage, error) - // Delete a WSMessage + // Delete a websocket message given a unique id DeleteWSMessage(wsmid string) error - // Get list of all the request keys + // Get list of the keys for all of the stored requests RequestKeys() ([]string, error) // A function to perform a search of requests in the storage. Same arguments as NewRequestChecker @@ -51,56 +53,30 @@ type MessageStorage interface { // A function to naively check every function in storage with the given function and return the ones that match CheckRequests(limit int64, checker RequestChecker) ([]*ProxyRequest, error) - // Same as Search() but returns the IDs of the requests instead - // If Search() starts causing memory errors and I can't assume all the matching requests will fit in memory, I'll implement this or something - //SearchIDs(args ...interface{}) ([]string, error) - - // Query functions + // Return a list of all the queries stored in the MessageStorage AllSavedQueries() ([]*SavedQuery, error) + // Save a query in the storage with a given name. If the name is already in storage, it should be overwritten SaveQuery(name string, query MessageQuery) error + // Load a query by name from the storage LoadQuery(name string) (MessageQuery, error) + // Delete a query by name from the storage DeleteQuery(name string) error } +// An error to be returned if a query is not supported const QueryNotSupported = ConstErr("custom query not supported") -type ReqSort []*ProxyRequest - +// A type representing a search query that is stored in a MessageStorage type SavedQuery struct { Name string Query MessageQuery } -func (reql ReqSort) Len() int { - return len(reql) -} - -func (reql ReqSort) Swap(i int, j int) { - reql[i], reql[j] = reql[j], reql[i] -} - -func (reql ReqSort) Less(i int, j int) bool { - return reql[j].StartDatetime.After(reql[i].StartDatetime) -} - -type WSSort []*ProxyWSMessage - -func (wsml WSSort) Len() int { - return len(wsml) -} - -func (wsml WSSort) Swap(i int, j int) { - wsml[i], wsml[j] = wsml[j], wsml[i] -} - -func (wsml WSSort) Less(i int, j int) bool { - return wsml[j].Timestamp.After(wsml[i].Timestamp) -} - /* General storage functions */ +// Save a new request and new versions of all its dependant messages (response, websocket messages, and unmangled versions of everything). func SaveNewRequest(ms MessageStorage, req *ProxyRequest) error { if req.ServerResponse != nil { if err := SaveNewResponse(ms, req.ServerResponse); err != nil { @@ -130,6 +106,7 @@ func SaveNewRequest(ms MessageStorage, req *ProxyRequest) error { return nil } +// Update a request and all its dependent messages. If the request has a DbId it will be updated, otherwise it will be inserted into the database and have its DbId updated. Same for all dependent messages func UpdateRequest(ms MessageStorage, req *ProxyRequest) error { if req.ServerResponse != nil { if err := UpdateResponse(ms, req.ServerResponse); err != nil { @@ -165,6 +142,7 @@ func UpdateRequest(ms MessageStorage, req *ProxyRequest) error { return nil } +// Save a new response/unmangled response to the message storage regardless of the existence of a DbId func SaveNewResponse(ms MessageStorage, rsp *ProxyResponse) error { if rsp.Unmangled != nil { if rsp.DbId != "" && rsp.DbId == rsp.Unmangled.DbId { @@ -178,6 +156,7 @@ func SaveNewResponse(ms MessageStorage, rsp *ProxyResponse) error { return ms.SaveNewResponse(rsp) } +// Update a response and its unmangled version in the database. If it has a DbId, it will be updated, otherwise a new version will be saved in the database func UpdateResponse(ms MessageStorage, rsp *ProxyResponse) error { if rsp.Unmangled != nil { if rsp.DbId != "" && rsp.DbId == rsp.Unmangled.DbId { @@ -195,6 +174,7 @@ func UpdateResponse(ms MessageStorage, rsp *ProxyResponse) error { } } +// Save a new websocket emssage/unmangled version to the message storage regardless of the existence of a DbId func SaveNewWSMessage(ms MessageStorage, req *ProxyRequest, wsm *ProxyWSMessage) error { if wsm.Unmangled != nil { if wsm.DbId != "" && wsm.DbId == wsm.Unmangled.DbId { @@ -208,6 +188,7 @@ func SaveNewWSMessage(ms MessageStorage, req *ProxyRequest, wsm *ProxyWSMessage) return ms.SaveNewWSMessage(req, wsm) } +// Update a websocket message and its unmangled version in the database. If it has a DbId, it will be updated, otherwise a new version will be saved in the database func UpdateWSMessage(ms MessageStorage, req *ProxyRequest, wsm *ProxyWSMessage) error { if wsm.Unmangled != nil { if wsm.DbId != "" && wsm.Unmangled.DbId == wsm.DbId { diff --git a/testutil.go b/testutil.go index eb13167..31291cd 100644 --- a/testutil.go +++ b/testutil.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "runtime" diff --git a/util.go b/util.go index 4ce3890..2b8c7e4 100644 --- a/util.go +++ b/util.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "io/ioutil" @@ -6,6 +6,7 @@ import ( "sync" ) +// A type that can be used to create specific error types. ie `const QueryNotSupported = ConstErr("custom query not supported")` type ConstErr string func (e ConstErr) Error() string { return string(e) } @@ -31,3 +32,33 @@ func IdCounter() func() int { func NullLogger() *log.Logger { return log.New(ioutil.Discard, "", log.Lshortfile) } + +// A helper type to sort requests by submission time: ie sort.Sort(ReqSort(reqs)) +type ReqSort []*ProxyRequest + +func (reql ReqSort) Len() int { + return len(reql) +} + +func (reql ReqSort) Swap(i int, j int) { + reql[i], reql[j] = reql[j], reql[i] +} + +func (reql ReqSort) Less(i int, j int) bool { + return reql[j].StartDatetime.After(reql[i].StartDatetime) +} + +// A helper type to sort websocket messages by timestamp: ie sort.Sort(WSSort(req.WSMessages)) +type WSSort []*ProxyWSMessage + +func (wsml WSSort) Len() int { + return len(wsml) +} + +func (wsml WSSort) Swap(i int, j int) { + wsml[i], wsml[j] = wsml[j], wsml[i] +} + +func (wsml WSSort) Less(i int, j int) bool { + return wsml[j].Timestamp.After(wsml[i].Timestamp) +} diff --git a/webui.go b/webui.go index 544bb0f..2253de2 100644 --- a/webui.go +++ b/webui.go @@ -1,4 +1,4 @@ -package main +package puppy import ( "encoding/pem" @@ -7,163 +7,164 @@ import ( "strings" ) -// Page template -var MASTER_SRC string = ` - - -{{block "title" .}}Puppy Proxy{{end}} -{{block "head" .}}{{end}} - - -{{block "body" .}}{{end}} - - -` -var MASTER_TPL *template.Template - -// Page sources -var HOME_SRC string = ` -{{define "title"}}Puppy Home{{end}} -{{define "body"}} -

Welcome to Puppy

-

-{{end}} -` -var HOME_TPL *template.Template - -var CERTS_SRC string = ` -{{define "title"}}CA Certificate{{end}} -{{define "body"}} -

Downlad this CA cert and add it to your browser to intercept HTTPS messages

-

Download

-{{end}} -` -var CERTS_TPL *template.Template - -var RSPVIEW_SRC string = ` -{{define "title"}}Response Viewer{{end}} -{{define "head"}} - -{{end}} -{{define "body"}} -

Enter a response ID below to view it in the browser

- -{{end}} -` -var RSPVIEW_TPL *template.Template - -func init() { + +func responseHeaders(w http.ResponseWriter) { + w.Header().Set("Connection", "close") + w.Header().Set("Cache-control", "no-cache") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Cache-control", "no-store") + w.Header().Set("X-Frame-Options", "DENY") +} + +// Generate a proxy-compatible web handler that allows users to download certificates and view responses stored in the storage used by the proxyin the browser +func CreateWebUIHandler() ProxyWebUIHandler { + var masterSrc string = ` + + + {{block "title" .}}Puppy Proxy{{end}} + {{block "head" .}}{{end}} + + + {{block "body" .}}{{end}} + + + ` + var masterTpl *template.Template + + var homeSrc string = ` + {{define "title"}}Puppy Home{{end}} + {{define "body"}} +

Welcome to Puppy

+

+ {{end}} + ` + var homeTpl *template.Template + + var certsSrc string = ` + {{define "title"}}CA Certificate{{end}} + {{define "body"}} +

Downlad this CA cert and add it to your browser to intercept HTTPS messages

+

Download

+ {{end}} + ` + var certsTpl *template.Template + + var rspviewSrc string = ` + {{define "title"}}Response Viewer{{end}} + {{define "head"}} + + {{end}} + {{define "body"}} +

Enter a response ID below to view it in the browser

+ + {{end}} + ` + var rspviewTpl *template.Template + var err error - MASTER_TPL, err = template.New("master").Parse(MASTER_SRC) + masterTpl, err = template.New("master").Parse(masterSrc) if err != nil { panic(err) } - HOME_TPL, err = template.Must(MASTER_TPL.Clone()).Parse(HOME_SRC) + homeTpl, err = template.Must(masterTpl.Clone()).Parse(homeSrc) if err != nil { panic(err) } - CERTS_TPL, err = template.Must(MASTER_TPL.Clone()).Parse(CERTS_SRC) + certsTpl, err = template.Must(masterTpl.Clone()).Parse(certsSrc) if err != nil { panic(err) } - RSPVIEW_TPL, err = template.Must(MASTER_TPL.Clone()).Parse(RSPVIEW_SRC) + rspviewTpl, err = template.Must(masterTpl.Clone()).Parse(rspviewSrc) if err != nil { panic(err) } -} -func responseHeaders(w http.ResponseWriter) { - w.Header().Set("Connection", "close") - w.Header().Set("Cache-control", "no-cache") - w.Header().Set("Pragma", "no-cache") - w.Header().Set("Cache-control", "no-store") - w.Header().Set("X-Frame-Options", "DENY") -} - -func WebUIHandler(w http.ResponseWriter, r *http.Request, iproxy *InterceptingProxy) { - responseHeaders(w) - parts := strings.Split(r.URL.Path, "/") - switch parts[1] { - case "": - WebUIRootHandler(w, r, iproxy) - case "certs": - WebUICertsHandler(w, r, iproxy, parts[2:]) - case "rsp": - WebUIRspHandler(w, r, iproxy, parts[2:]) + var WebUIRootHandler = func(w http.ResponseWriter, r *http.Request, iproxy *InterceptingProxy) { + err := homeTpl.Execute(w, nil) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } } -} -func WebUIRootHandler(w http.ResponseWriter, r *http.Request, iproxy *InterceptingProxy) { - err := HOME_TPL.Execute(w, nil) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} + var WebUICertsHandler = func(w http.ResponseWriter, r *http.Request, iproxy *InterceptingProxy, path []string) { + if len(path) > 0 && path[0] == "download" { + cert := iproxy.GetCACertificate() + if cert == nil { + w.Write([]byte("no active certs to download")) + return + } -func WebUICertsHandler(w http.ResponseWriter, r *http.Request, iproxy *InterceptingProxy, path []string) { - if len(path) > 0 && path[0] == "download" { - cert := iproxy.GetCACertificate() - if cert == nil { - w.Write([]byte("no active certs to download")) + pemData := pem.EncodeToMemory( + &pem.Block{ + Type: "CERTIFICATE", + Bytes: cert.Certificate[0], + }, + ) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", "attachment; filename=\"cert.pem\"") + w.Write(pemData) + return + } + err := certsTpl.Execute(w, nil) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) return } - - pemData := pem.EncodeToMemory( - &pem.Block{ - Type: "CERTIFICATE", - Bytes: cert.Certificate[0], - }, - ) - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Disposition", "attachment; filename=\"cert.pem\"") - w.Write(pemData) - return - } - err := CERTS_TPL.Execute(w, nil) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return } -} -func viewResponseHeaders(w http.ResponseWriter) { - w.Header().Del("Cookie") -} + var viewResponseHeaders = func(w http.ResponseWriter) { + w.Header().Del("Cookie") + } -func WebUIRspHandler(w http.ResponseWriter, r *http.Request, iproxy *InterceptingProxy, path []string) { - if len(path) > 0 { - reqid := path[0] - ms := iproxy.GetProxyStorage() - req, err := ms.LoadRequest(reqid) + var WebUIRspHandler = func(w http.ResponseWriter, r *http.Request, iproxy *InterceptingProxy, path []string) { + if len(path) > 0 { + reqid := path[0] + ms := iproxy.GetProxyStorage() + req, err := ms.LoadRequest(reqid) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + rsp := req.ServerResponse + for k, v := range rsp.Header { + for _, vv := range v { + w.Header().Add(k, vv) + } + } + viewResponseHeaders(w) + w.WriteHeader(rsp.StatusCode) + w.Write(rsp.BodyBytes()) + return + } + err := rspviewTpl.Execute(w, nil) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - rsp := req.ServerResponse - for k, v := range rsp.Header { - for _, vv := range v { - w.Header().Add(k, vv) - } - } - viewResponseHeaders(w) - w.WriteHeader(rsp.StatusCode) - w.Write(rsp.BodyBytes()) - return } - err := RSPVIEW_TPL.Execute(w, nil) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + + return func(w http.ResponseWriter, r *http.Request, iproxy *InterceptingProxy) { + responseHeaders(w) + parts := strings.Split(r.URL.Path, "/") + switch parts[1] { + case "": + WebUIRootHandler(w, r, iproxy) + case "certs": + WebUICertsHandler(w, r, iproxy, parts[2:]) + case "rsp": + WebUIRspHandler(w, r, iproxy, parts[2:]) + } } + }