Now working for simple, non-encrypted requests

making requests to something like example.com over a non-encrypted
connection now works. Binary files are unlikely to work at the moment
although I haven't tried. Also, non-encrypted doesn't work.

I have also changed a little about how tests work. Requests tests now
display much better.

Makefile

@@ -6,7 +6,7 @@ TESTFILES   = $(wildcard tests/*.c)
 TESTOUT = $(TESTFILES:.c=)
 OUT      = yaip
 CFLAGS   = -Wall -g
-LDLIBS   = -lsqlite3
+LDLIBS   = -lsqlite3 -lm
 CC = gcc
 
 
@@ -24,8 +24,8 @@ $(OUT): $(OBJFILES)
 %.o: %.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
-tests/%.test: tests/%.test.c tests/munit/munit.c
+tests/%.test: tests/%.test.c tests/munit/munit.c $(filter-out src/main.o, $(OBJFILES))
-	$(CC) $? -o $@
+	$(CC) -o $@ $^ $(LDLIBS)
 
 
 test-%: tests/%.test

src/proxy.c

@@ -1,12 +1,50 @@
 #include "proxy.h"
 
-void proxy_startListener(unsigned int port){
+Response *upstreamGetResponse(Request *request){
+	//Here we pretend to be a client
 
+	int client_fd = 0;
+	struct sockaddr_in address;
+	memset( &address, 0, sizeof(address) );
+	struct hostent *host = gethostbyname(request->host);
+	Response *rsp = NULL;
+
+	if ((client_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0){
+		perror("socket failed");
+		return NULL;
+	}
+
+
+	address.sin_family = AF_INET;
+	address.sin_port = htons( 80 );
+	// We want the request to go out to whatever the host was resolved to
+	memcpy( &address.sin_addr, host->h_addr_list[0], host->h_length );
+
+	if((connect(client_fd, (struct sockaddr *)&address, sizeof(address)))<0)  {  
+		perror("connect failed");
+		return NULL;
+	}
+
+	char *toSend = requestToString(request);
+
+	if ( write( client_fd, toSend, strlen(toSend) ) != strlen(toSend) ){
+		perror( "Write Error" );
+		return NULL;
+	}
+
+	rsp = newResponseFromSocket( client_fd );
+
+	return rsp;
+
+}
+
+void proxy_startListener(unsigned int port){
+	//we need to act as an http server
 	int server_fd, new_socket;
     struct sockaddr_in address;
-    int opt = 1;
+	memset( &address, 0, sizeof(address) );
     int addrlen = sizeof(address);
-	char *response;
+	Response *response;
 
     // Creating socket file descriptor
     if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) {
@@ -30,25 +68,44 @@ void proxy_startListener(unsigned int port){
     }
 
 	while ( true ){
-		printf("Listening\n");
+		printf("Listening on port %i\n", port);
 		if ((new_socket = accept(server_fd, (struct sockaddr *)&address, 
 				(socklen_t*)&addrlen))<0) {
 			perror("accept");
 			return ;
 		}
 
+		//I think eventually I'd like a different thread here for each request
+		//Not sure how to do that yet though so I'll keep everything on the main
+		//thread
+
 		Request *request = newRequestFromSocket(new_socket);
 
+		//If this is an https request - this is the first part
+		if ( strcmp( request->method, "CONNECT" ) == 0 ){
+			printf("\n\n%s\n\n", requestToString( request ));
+		}
+
+		// If the host is not defined, then it is not a proxy request
+		// Note that the host here is where the request should be sent, not
+		// necesarily the hosts header
 		if ( strcmp( request->host, "" ) == 0 ){
 			response = webserverGetResponse(request);
 		} else {
-			printf("This is a proxy request\n");
+			response = upstreamGetResponse(request);
 		}
 
-		send(new_socket , response, strlen(response) , 0 );
+		char *responseStr = responseToString( response );
 
+		// I'm also not convinced that strlen is the best function to use here
+		// When we get to dealing with binary requests / responses, they may
+		// well have null characters in them
+		send(new_socket , responseStr, strlen(responseStr) , 0 );
 
-		//close(new_socket);
+		close(new_socket);
+		freeRequest( request );
+		freeResponse( response );
+		free(responseStr);
 	}
 
 }

src/proxy.h

@@ -6,6 +6,8 @@
 #include <sys/socket.h>
 #include <stdlib.h>
 #include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netdb.h>
 #include <string.h>
 #include <stdbool.h>
 
@@ -13,5 +15,6 @@
 #include "webserver.h"
 
 void proxy_startListener( unsigned int port );
+Response *upstreamGetResponse(Request *request);
 
 #endif /* ifndef PROXY_H */

src/request.c

@@ -10,12 +10,13 @@ Request* newRequest(){
 
 void requestFirstLine( Request *req, char line[] ){
 	char method[20] = {'\0'}; //Get, post, etc.
-	char *url = malloc(sizeof(char) *2048); // This may contain the method, the path, the domain and so on
+	// This may contain the method, the path, the domain and so on
+	char *url = malloc(sizeof(char) *2048);
 	memset(url, '\0', sizeof(char) * 2048);
-	char *currentPos;
+	char *currentPos = url;
 	float version = 0;
-	char protocol[6] = {'\0'};
 	char host[254] = {'\0'};
+	int port = -1; // 0 is a valid port number, -1 isn't
 	//https://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers
 	char path[2000] = {'\0'};
 
@@ -24,33 +25,63 @@ void requestFirstLine( Request *req, char line[] ){
 	req->version = version;
 	
 	//We've pulled out the easy bits. Now to go through the url and pull out what we need
-	currentPos = url;
+	int protEnd = strpos(url, "://" );
-	sscanf( currentPos, "%5[^:/]", protocol );
+	if ( protEnd > -1 ){
-	if ( strlen( protocol ) > 0 ){
+		req->protocol = strndup( url, protEnd );
-		currentPos = currentPos + strlen(protocol) + 3;
+		currentPos += protEnd + 3;
+	} else {
+		req->protocol = "";
 	}
 
+
 	sscanf( currentPos, "%253[^:/]", host );
 	if ( strlen( host ) > 0 ){
 		currentPos = currentPos + strlen(host);
+		req->host = strdup(host);
+	} else {
+		req->host = "";
+	}
+
+
+	if ( currentPos[0] == ':' ){
+		currentPos++;
+		sscanf( currentPos, "%i", &port );
+		req->port = port;
+		currentPos += (int)floor( log10( port ) ) + 1;
+		
+	} else {
+		req->port = -1;
 	}
 		
 
 	sscanf( currentPos, "%2000[^? ]", path );
 	if ( strlen( path ) > 0 ){
-		currentPos = currentPos + strlen(path);
+		currentPos += strlen(path);
+		req->path = strdup(path);
+	} else {
+		req->path = "";
 	}
 
-	req->protocol = malloc(sizeof(char) * strlen(protocol));
+	if ( strlen( currentPos ) > 0 ){
-	strcpy(req->protocol, protocol);
+		req->queryString = strdup( currentPos );
-	req->host = malloc(sizeof(char) * strlen(host));
+	} else {
-	strcpy(req->host, host);
+		req->queryString = "";
-	req->path = malloc(sizeof(char) * strlen(path));
+	}
-	strcpy(req->path, path);
 
-	//The query string is anything left
+	//We try and work out port and protocol if we don't have them
-	req->queryString = malloc(sizeof(char) * strlen(currentPos));
+	if ( req->port == -1){
-	strcpy(req->queryString, currentPos);
+		if ( strcmp( "https", req->protocol ) == 0 )
+			req->port = 443;
+		else
+			req->port = 80;
+	}
+
+	if ( strlen(req->protocol) == 0 ){
+		if ( req->port == 443 )
+			req->protocol = "https";
+		else
+			req->protocol = "http";
+	}
 
 
 	free(url);
@@ -70,30 +101,38 @@ Request* newRequestFromSocket(int socket){
 	//from the body (if there is a body)
 	valread = fdReadLine( socket, line, 1024);
 	while ( valread > 2 ){
-		printf("%s",line );
 		requestAddHeader( req, line );
-
 		//I believe at this point all the headers are done.
 		valread = fdReadLine( socket , line, 1024);
 	}
+
+	//TODO: make requests work with a body
+	//contentLength = getHeader( req->headers, "content-length" );
+
+	//if ( contentLength != NULL ){
+	//	printf( "Content length is %i\n", atoi(contentLength->value) );
+	//}
+	
 	return req;
 }
 
-char* requestToString( Request *req, bool proxy){
+char* requestToString( Request *req){
 	unsigned int fullLength = strlen(req->method) + 1 + sizeof( req->path ) +
+		//11 = [space]http/1.1\r\n
 		sizeof( req->queryString ) + 11 + headerListCharLength( req->headers );
-	if ( proxy )
+
-		fullLength += sizeof(req->method) + 3 + sizeof(req->host);
+	if ( strcmp( req->method, "CONNECT" ) == 0 )
+		fullLength += sizeof(req->host);
 
 
 	char retString[fullLength];
+	memset( retString, '\0', fullLength );
 
-	if (proxy)
+	if ( strcmp( req->method, "CONNECT" ) == 0 )
-		sprintf(retString, "%s %s://%s%s%s HTTP/%.1f\r\n%s", req->method,
+		sprintf(retString, "%s %s HTTP/%.1f\r\n%s\r\n", req->method,
-			req->method, req->host, req->path, req->queryString, req->version,
+			req->host, req->version, headersToString(req->headers));
-			headersToString(req->headers));
 	else
-		sprintf(retString, "%s %s%s HTTP/%.1f\r\n%s", req->method,
+		sprintf(retString, "%s %s%s HTTP/%.1f\r\n%s\r\n", req->method,
 			req->path, req->queryString, req->version,
 			headersToString(req->headers));
 	//	rsp->statusMessage,headersToString(rsp->headers),rsp->body);
@@ -111,5 +150,15 @@ void requestAddHeader( Request *req, char header[] ){
 }
 
 
+void freeRequest( Request *req ){
+	free(req->method);
+	free(req->protocol);
+	free(req->host);
+	free(req->path);
+	freeHeaderList( req->headers );
+	free(req->queryString);
+	free(req->body);
+	free(req);
+}
 
 

src/request.h

@@ -6,9 +6,12 @@
 #include <stdbool.h>
 #include <string.h>
 #include <netinet/in.h>
+#include <math.h>
 #include "readline.h"
 #include "requestresponse.h"
 
+#include "./util.h"
+
 
 
 /*
@@ -21,9 +24,10 @@ typedef struct {
 	char *protocol;
 	char *host;
 	char *path;
+	int port;
 	HeaderList *headers;
 	char *queryString;
-	char *body;
+	void *body;
 } Request;
 
 Request* newRequest();
@@ -32,12 +36,11 @@ Request* newRequestFromSocket(int socket);
 /*
 * requestToString
 * @prarm req the request to convert
-* @param proxy whether we want a proxy request (adds the host to the first line) 
 */
-char* requestToString( Request *req, bool proxy );
+char* requestToString( Request *req );
 void requestAddHeader( Request *req, char header[] );
-//void* requestAddHeader(Request *req, char line[]);
 
+void freeRequest( Request *req );
 
 
 #endif /* ifndef REQUEST_H */

src/requestresponse.c

@@ -4,42 +4,49 @@ Header* newHeader(char *str){
 	Header *header = malloc(sizeof(Header));
 	memset(header, 0, sizeof(Header));
 
-	int position = -1;
+
+	char *position = str;
+	char *end = str + strlen(str) - 1;
+
+	//Let's loose any trailing whitespace
+	while ( ( *end == '\n' || *end == '\r' ) && end > str ) end--;
+
+	while ( *position != ':' && position < end ) position++;
 
 
-	for ( unsigned int i = 0; i < strlen(str); i++ ){
+	if ( position == end ){
-		if ( str[i] == ':' ){
-			position = i;
-			break;
-		}
-	}
-
-	if ( position == -1 ){
 		printf("Header without colon. Not sure what to do\n");
 		return NULL;
 	}
 
 
-	// We want to allocate 1 more than the length so we can have a \0 at the end
+	header->name = malloc( sizeof(char) * ( position + 1 - str ) );
-	header->name = malloc( sizeof(char) * ( position + 1 ) );
+	memset(header->name, '\0', position + 1 - str );
-	memset(header->name, '\0', position+1);
+	strncpy( header->name, str, position - str );
-	strncpy( header->name, str, position );
 
-	for ( unsigned int i = position+1; i < strlen(str); i++ ){
+	//Skip over the colon
-		if ( str[i] == '\t' || str[i] == ' ' ) continue;
+	position++;
-		position = i;
+	//Skip over any trailing whitespace
-		break;
+	while ( *position == '\t' || *position == ' ' ) position++;
-	}
 
-
+	//Add 2 when allocating so that we have space for a \0 at the end
-	//Anything left is the value
+	header->value = malloc( sizeof(char) * ( end + 2 - position ) );
-	header->value = malloc( sizeof(char) * ( strlen(str) - position ) );
+	memset(header->value, '\0', ( end + 2 - position ) );
-	memset(header->value, '\0', ( strlen(str) - position ) );
+	strncpy( header->value, position, end + 1 - position );
-	strcpy( header->value, str + position );
 	
 	return header;
+
+
 }
 
+
+void freeHeader(Header *header){
+	free(header->name);
+	free(header->value);
+	free(header);
+}
+
+
 void addHeader(HeaderList *headers, char *str){
 	HeaderList *newListItem = malloc(sizeof(HeaderList));
 
@@ -53,6 +60,16 @@ void addHeader(HeaderList *headers, char *str){
 	last->next = newListItem;
 }
 
+void freeHeaderList(HeaderList *headers){
+	HeaderList *current;
+	while ( headers != NULL ){
+		current = headers;
+		freeHeader( current->header );
+		headers = headers->next;
+		free( current );
+	}
+}
+
 
 // Will return the first header in a header list with the name matching name
 // It matches name case insensitively as header names are case insensitive

src/requestresponse.h

@@ -28,9 +28,13 @@ struct HeaderList {
 //Creates a new header struct from a string like "Content-Length: 123"
 Header* newHeader(char *str);
 
+void freeHeader(Header *header);
+
 //Adds a header to the end of a header list
 void addHeader(HeaderList *headers, char *str);
 
+void freeHeaderList(HeaderList *headers);
+
 Header* getHeader(HeaderList *headers, char *name);
 
 unsigned int countHeaders(HeaderList *headers);

src/response.c

@@ -3,25 +3,38 @@
 Response* newResponse(){
 	Response *response = malloc( sizeof( Response ) );
 	memset(response, 0, sizeof(Response));
-	response->headers = malloc( sizeof( HeaderList ) );
+	response->headers = NULL;
-	response->headers->header = newHeader("Content-Length: 0");
+	response->statusMessage = NULL;
-	response->headers->next = NULL;
+	response->body = NULL;
-	addHeader(response->headers, "Content-Type: text/plain");
-	response->statusCode = 200;
-	response->statusMessage = "OK";
-	response->version = 1.1;
 	return response;
 }
 
+void responseBarebones(Response *rsp){
+	rsp->headers = malloc( sizeof( HeaderList ) );
+	rsp->headers->header = newHeader("Content-Length: 0");
+	rsp->headers->next = NULL;
+	addHeader(rsp->headers, "Content-Type: text/plain");
+	rsp->statusCode = 200;
+	rsp->statusMessage = "OK";
+	rsp->version = 1.1;
+}
+
 char* responseToString( Response *rsp ){
 	//               HTTP/x.x xxx      OK                       \r\n
 	int fullLength = 13 +           strlen(rsp->statusMessage) + 2 +
 	//      Headers                   \r\n\r\n
-		headerListCharLength(rsp->headers) + 4 +        strlen(rsp->body);
+		headerListCharLength(rsp->headers) + 4;
+
+	if ( rsp->body != NULL ) fullLength += strlen(rsp->body) + 2;
+
 	char retString[fullLength];
 
-	sprintf(retString, "HTTP/%.1f %3d %s\r\n%s\r\n%s", rsp->version, rsp->statusCode,
+
-		rsp->statusMessage,headersToString(rsp->headers),rsp->body);
+	sprintf(retString, "HTTP/%.1f %3d %s\r\n%s\r\n", rsp->version, rsp->statusCode,
+		rsp->statusMessage,headersToString(rsp->headers));
+
+	if ( rsp->body != NULL ) sprintf(retString + strlen(retString), "%s\r\n", (char *)rsp->body);
+
 	return strdup(retString);
 }
 
@@ -47,7 +60,65 @@ void responseAddHeader(Response *rsp, char header[]){
 
 }
 
+void responseFirstLine( Response *rsp, char line[] ){
+	// HTTP/1.1 200 OK
+	float version = 0;
+	int statusCode = 0;
+	char statusMessage[256] = {'\0'};
+
+	sscanf( line, "HTTP/%f %i %255[^\n]", &version, &statusCode, statusMessage );
+	rsp->version = version;
+	rsp->statusCode = statusCode;
+	rsp->statusMessage = strdup(statusMessage);
+}
+
 Response* newResponseFromSocket(int socket){
 	Response *rsp = newResponse();
+	int valread;
+	char line[1024] = {0};
+	Header *contentLength = NULL;
+	unsigned int bodySize = 0, bodyRead = 0;
+
+	valread = fdReadLine( socket, line, 1024);
+
+	responseFirstLine(rsp, line);
+
+	//a length of 2 will indicate an empty line which will split the headers
+	//from the body (if there is a body)
+	valread = fdReadLine( socket, line, 1024);
+	while ( valread > 2 ){
+		responseAddHeader( rsp, line );
+		valread = fdReadLine( socket , line, 1024);
+	}
+	//I believe at this point all the headers are done.
+	contentLength = getHeader( rsp->headers, "content-length" );
+
+	if ( contentLength != NULL ){
+		bodySize = atoi( contentLength->value );
+		if (bodySize > 0){
+			//When the body is not binary data, it is useful to treat it as a
+			//string. For this reason, I want the bite immediately after it to
+			//be \0
+			rsp->body = malloc( bodySize + 1 );
+			memset( rsp->body + bodySize + 1, '\0', 1 );
+			while ( bodyRead < bodySize ){
+				valread=read( socket, rsp->body + bodyRead, bodySize - bodyRead );
+				if (valread < 0){
+					perror("Unable to read");
+					return NULL;
+				}
+				bodyRead += valread;
+			}
+		}
+	}
+
+
 	return rsp;
 }
+
+void freeResponse( Response *rsp ){
+	free(rsp->statusMessage);
+	freeHeaderList( rsp->headers );
+	free(rsp->body);
+	free(rsp);
+}

src/response.h

@@ -20,15 +20,30 @@ typedef struct {
 	int statusCode;
 	char *statusMessage;
 	HeaderList *headers;
-	char *body;
+	void *body;
 	unsigned int headerLength;
 } Response;
 
 Response* newResponse();
+/*
+* creates the minium viable valid response
+*/
+void responseBarebones(Response *rsp);
 char *responseToString(Response *rsp);
+/* sets the body of a response to a string
+* @param rsp the response
+* @param string the string
+* @param updateContentLength whether the content-length header should be auto-updated
+* Note: if you want to set a binary body, use responseSetBodyRaw
+*/
 void responseSetBody(Response *rsp, char *string, bool updateContentLength);
+// TODO:
+//void responseSetBodyRaw(Response *rsp, void *body, size_t size, bool updateContentLength);
 void responseAddHeader(Response *rsp, char header[]);
+Response* newResponseFromSocket(int socket);
+void responseFirstLine( Response *req, char line[] );
 
+void freeResponse( Response *rsp );
 
 
 #endif /* ifndef REQUEST_H */

src/util.c

@@ -0,0 +1,7 @@
+#include "util.h"
+
+int strpos(char *haystack, char *needle) {
+	char *p = strstr(haystack, needle);
+	if ( p != NULL ) return p - haystack;
+	return -1;
+}

src/util.h

@@ -0,0 +1,9 @@
+#ifndef UTIL_H
+#define UTIL_H
+
+#include <stdio.h>
+#include <string.h>
+
+int strpos(char *haystack, char *needle);
+
+#endif

src/webserver.c

@@ -1,7 +1,8 @@
 #include "webserver.h"
 
-char* webserverGetResponse( Request *req ){
+Response* webserverGetResponse( Request *req ){
 	Response *rsp = newResponse();
+	responseBarebones(rsp);
 	responseSetBody(rsp, "Test", 1);
-	return responseToString(rsp);
+	return rsp;
 }

src/webserver.h

@@ -7,7 +7,7 @@
 #include "request.h"
 #include "response.h"
 
-char* webserverGetResponse( Request *req );
+Response* webserverGetResponse( Request *req );
 
 #endif /* ifndef WEBSERVER_H
 

tests/all.test.c

@@ -4,12 +4,12 @@
 #include "munit/munit.h"
 
 #include "config.test.c"
-#include "request.test.c"
-#include "response.test.c"
 #ifndef REQUESTRESPONSE_C
 #define REQUESTRESPONSE_C value
 #include "requestresponse.test.c"
 #endif /* ifndef REQUESTRESPONSE_C */
+#include "request.test.c"
+#include "response.test.c"
 #include <string.h>
 #include <stdio.h>
 

tests/request.test.c

@@ -1,31 +1,41 @@
 #ifndef REQUEST_TEST
 #define REQUEST_TEST
 
+
 #include "munit/munit.h"
-#include "../src/readline.c"
+
-#include "../src/request.c"
+#ifndef READLINE_C
+#define READLINE_C
+#include "../src/readline.h"
+#endif
+#include "../src/request.h"
 #ifndef REQUESTRESPONSE_C
 #define REQUESTRESPONSE_C
-#include "../src/requestresponse.c"
+#include "../src/requestresponse.h"
 #endif /* ifndef REQUESTRESPONSE_C */
 
+
 typedef struct {
 	char *fullLine;
 	char *method;
 	char *host;
+	int port;
 	char *protocol;
 	double version;
 	char *path;
 	char *queryString;
-} firstLine;
+} requestTestFirstLine;
 
 
-static firstLine line1Examples[] = {
+static requestTestFirstLine requestLine1Examples[] = {
-	//Full Line,                                             Method  Host           Protocol  Version  Path
+	//Full Line,                                             Method     Host           Port  Protocol  Version  Path                 Query string
-	{ "GET /search?q=test HTTP/2",                           "GET",  "",            "",       2,       "/search",           "?q=test"},
+	{ "GET /search?q=test HTTP/2",                           "GET",     "",            80,   "http",   2,       "/search",           "?q=test"},
-	{ "POST http://example.com/test HTTP/1.1",               "POST", "example.com", "http",   1.1,     "/test",             "" },
+	{ "GET / HTTP/2",                                        "GET",     "",            80,   "http",   2,       "/",                 ""},
-	{ "POST https://example.com/test/test2/test3 HTTP/1.1",  "POST", "example.com", "https",  1.1,     "/test/test2/test3", ""},
+	{ "POST http://example.com/test HTTP/1.1",               "POST",    "example.com", 80,   "http",   1.1,     "/test",             "" },
-	{ NULL,                                                   NULL,  NULL,           NULL,    0,       NULL,                NULL }
+	{ "CONNECT example.com:443 HTTP/1.1",                    "CONNECT", "example.com", 443,  "https",  1.1,     "",                  "" },
+	{ "POST https://example.com/test/test2/test3 HTTP/1.1",  "POST",    "example.com", 443,  "https",  1.1,     "/test/test2/test3", ""},
+	{ "POST https://example.com:4444/test/ HTTP/1.1",        "POST",    "example.com", 4444, "https",  1.1,     "/test/",            ""},
+	{ NULL,                                                   NULL,     NULL,          80,    NULL,    0,       NULL,                NULL }
 };
 
 
@@ -34,72 +44,133 @@ static firstLine line1Examples[] = {
 MunitResult testFirstLineProtocols(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Request *req;
-	for ( firstLine *line = line1Examples; line->fullLine != NULL; line++ ){
+	const char *firstLine = munit_parameters_get(params, "L1" );
+	requestTestFirstLine *line = requestLine1Examples;
+
+	while ( line->fullLine != NULL && strcmp( line->fullLine, firstLine ) != 0 )
+		line++;
+	
+	if ( line->fullLine == NULL ) return MUNIT_ERROR;
 	req = newRequest();
 	requestFirstLine( req, line->fullLine );
+	munit_assert_not_null( req->protocol );
 	munit_assert_string_equal( req->protocol, line->protocol );
 	free( req );
-	}
 	return MUNIT_OK;
 }
 
 MunitResult testFirstLineMethod(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Request *req;
-	for ( firstLine *line = line1Examples; line->fullLine != NULL; line++ ){
+	const char *firstLine = munit_parameters_get(params, "L1" );
+	requestTestFirstLine *line = requestLine1Examples;
+
+	while ( line->fullLine != NULL && strcmp( line->fullLine, firstLine ) != 0 )
+		line++;
+	
+	if ( line->fullLine == NULL ) return MUNIT_ERROR;
 	req = newRequest();
 	requestFirstLine( req, line->fullLine );
+	munit_assert_not_null( req->method );
 	munit_assert_string_equal( req->method, line->method );
 	free( req );
-	}
 	return MUNIT_OK;
 }
 
 MunitResult testFirstLineHosts(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Request *req;
-	for ( firstLine *line = line1Examples; line->fullLine != NULL; line++ ){
+	const char *firstLine = munit_parameters_get(params, "L1" );
+	requestTestFirstLine *line = requestLine1Examples;
+
+	while ( line->fullLine != NULL && strcmp( line->fullLine, firstLine ) != 0 )
+		line++;
+	
+	if ( line->fullLine == NULL ) return MUNIT_ERROR;
 	req = newRequest();
 	requestFirstLine( req, line->fullLine );
+	munit_assert_not_null( req->host );
 	munit_assert_string_equal( req->host, line->host );
 	free( req );
+	return MUNIT_OK;
 }
+
+MunitResult testFirstLinePorts(const MunitParameter params[],
+		void* user_data_or_fixture){
+	Request *req;
+	const char *firstLine = munit_parameters_get(params, "L1" );
+	requestTestFirstLine *line = requestLine1Examples;
+
+	while ( line->fullLine != NULL && strcmp( line->fullLine, firstLine ) != 0 )
+		line++;
+	
+	if ( line->fullLine == NULL ) return MUNIT_ERROR;
+	req = newRequest();
+	requestFirstLine( req, line->fullLine );
+	munit_assert_int( req->port, ==, line->port );
+	free( req );
 	return MUNIT_OK;
 }
 
 MunitResult testFirstLinePaths(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Request *req;
-	for ( firstLine *line = line1Examples; line->fullLine != NULL; line++ ){
+	const char *firstLine = munit_parameters_get(params, "L1" );
+	requestTestFirstLine *line = requestLine1Examples;
+
+	while ( line->fullLine != NULL && strcmp( line->fullLine, firstLine ) != 0 )
+		line++;
+	
+	if ( line->fullLine == NULL ) return MUNIT_ERROR;
+	
+	
 	req = newRequest();
 	requestFirstLine( req, line->fullLine );
+	munit_assert_not_null( req->path );
 	munit_assert_string_equal( req->path, line->path );
 	free( req );
-	}
 	return MUNIT_OK;
 }
 
 MunitResult testFirstLineVersions(const MunitParameter params[],
 		void* user_data_or_fixture){
+
 	Request *req;
-	for ( firstLine *line = line1Examples; line->fullLine != NULL; line++ ){
+	const char *firstLine = munit_parameters_get(params, "L1" );
+	requestTestFirstLine *line = requestLine1Examples;
+
+	while ( line->fullLine != NULL && strcmp( line->fullLine, firstLine ) != 0 )
+		line++;
+	
+	if ( line->fullLine == NULL ) return MUNIT_ERROR;
+	
+	
 	req = newRequest();
 	requestFirstLine( req, line->fullLine );
 	munit_assert_float( req->version, ==, line->version );
 	free( req );
-	}
+
 	return MUNIT_OK;
 }
 
 MunitResult testFirstLineQueryString(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Request *req;
-	for ( firstLine *line = line1Examples; line->fullLine != NULL; line++ ){
+
+	const char *firstLine = munit_parameters_get(params, "L1" );
+	requestTestFirstLine *line = requestLine1Examples;
+
+	while ( line->fullLine != NULL && strcmp( line->fullLine, firstLine ) != 0 )
+		line++;
+	
+	if ( line->fullLine == NULL ) return MUNIT_ERROR;
+	
+	
 	req = newRequest();
 	requestFirstLine( req, line->fullLine );
+	munit_assert_not_null( req->queryString );
 	munit_assert_string_equal( req->queryString, line->queryString );
 	free( req );
-	}
 	return MUNIT_OK;
 }
 
@@ -118,13 +189,15 @@ MunitResult testRequestToString(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Request *req = newRequest();
 	requestFirstLine( req, "GET /search?q=test HTTP/1.1" );
-	munit_assert_string_equal( requestToString( req, 0 ), "GET /search?q=test HTTP/1.1\r\n" );
+	munit_assert_string_equal( requestToString( req ), "GET /search?q=test HTTP/1.1\r\n\r\n" );
 	requestAddHeader( req, "Host: example.com" );
-	munit_assert_string_equal( requestToString( req, 0 ), "GET /search?q=test HTTP/1.1\r\nHost: example.com\r\n" );
+	munit_assert_string_equal( requestToString( req ), "GET /search?q=test HTTP/1.1\r\nHost: example.com\r\n\r\n" );
 
 	return MUNIT_OK;
 }
 
+MunitParameterEnum test_first_line_params[2] = {NULL, NULL};
+
 static MunitTest request_tests[] = {
 	{
 		"/line1/versions", /* name */
@@ -132,42 +205,49 @@ static MunitTest request_tests[] = {
 		NULL, /* setup */
 		NULL, /* tear_down */
 		MUNIT_TEST_OPTION_NONE, /* options */
-		NULL /* parameters */
+		test_first_line_params /* parameters */
 	}, {
 		"/line1/methods", /* name */
 		testFirstLineMethod, /* test */
 		NULL, /* setup */
 		NULL, /* tear_down */
 		MUNIT_TEST_OPTION_NONE, /* options */
-		NULL /* parameters */
+		test_first_line_params /* parameters */
 	},{
 		"/line1/protocols", /* name */
 		testFirstLineProtocols, /* test */
 		NULL, /* setup */
 		NULL, /* tear_down */
 		MUNIT_TEST_OPTION_NONE, /* options */
-		NULL /* parameters */
+		test_first_line_params /* parameters */
 	},{
 		"/line1/hosts", /* name */
 		testFirstLineHosts, /* test */
 		NULL, /* setup */
 		NULL, /* tear_down */
 		MUNIT_TEST_OPTION_NONE, /* options */
-		NULL /* parameters */
+		test_first_line_params /* parameters */
+	},{
+		"/line1/ports", /* name */
+		testFirstLinePorts, /* test */
+		NULL, /* setup */
+		NULL, /* tear_down */
+		MUNIT_TEST_OPTION_NONE, /* options */
+		test_first_line_params /* parameters */
 	},{
 		"/line1/paths", /* name */
 		testFirstLinePaths, /* test */
 		NULL, /* setup */
 		NULL, /* tear_down */
 		MUNIT_TEST_OPTION_NONE, /* options */
-		NULL /* parameters */
+		test_first_line_params /* parameters */
 	},{
 		"/line1/queryStrings", /* name */
 		testFirstLineQueryString, /* test */
 		NULL, /* setup */
 		NULL, /* tear_down */
 		MUNIT_TEST_OPTION_NONE, /* options */
-		NULL /* parameters */
+		test_first_line_params /* parameters */
 	},{
 		"/headers/add", /* name */
 		testRequestAddHeader, /* test */
@@ -200,6 +280,24 @@ MunitSuite request_test_suite = {
 #define MAINTEST
 
 int main (int argc, char* argv[]) {
+	static char** firstLines;
+	unsigned int count = 1;
+	requestTestFirstLine *current = requestLine1Examples;
+	while ( current->fullLine != NULL ){
+		count++;
+		current++;
+	}
+	printf("There are %i request lines to test", count);
+	firstLines = malloc( sizeof( char * ) * ( count ) );
+
+	for( unsigned int i = 0; i < count; i++ ){
+		firstLines[i] = requestLine1Examples[i].fullLine;
+	}
+
+	test_first_line_params[0].name = "L1";
+	test_first_line_params[0].values = firstLines;
+
+
 	return munit_suite_main(&request_test_suite, NULL, argc, argv);
 }
 

tests/requestresponse.test.c

@@ -10,19 +10,26 @@ typedef struct {
 	char *value;
 } HeaderTestCase;
 
+//Not that some of these have trailing \r and \n in the full lines
+//These should be removed when turning it into a header struct
 static HeaderTestCase testCases[] = {
 	{ "Content-Encoding: gzip", "Content-Encoding", "gzip" },
 	{ "Accept-Ranges: bytes", "Accept-Ranges", "bytes" },
-	{ "Age: 186432", "Age", "186432" },
+	{ "Age: 186432\n", "Age", "186432" },
 	{ "Cache-Control: max-age=604800", "Cache-Control", "max-age=604800" },
 	{ "Content-Type: text/html; charset=UTF-8", "Content-Type", "text/html; charset=UTF-8" },
-	{ "Date: Thu, 06 Jan 2022 18:52:13 GMT", "Date", "Thu, 06 Jan 2022 18:52:13 GMT" },
+	{ "Date: Thu, 06 Jan 2022 18:52:13 GMT\r", "Date", "Thu, 06 Jan 2022 18:52:13 GMT" },
 	{ "Etag: \"3147526947+ident\"", "Etag", "\"3147526947+ident\"" },
 	{ "Expires: Thu, 13 Jan 2022 18:52:13 GMT", "Expires", "Thu, 13 Jan 2022 18:52:13 GMT" },
-	{ "Last-Modified: Thu, 17 Oct 2019 07:18:26 GMT", "Last-Modified", "Thu, 17 Oct 2019 07:18:26 GMT" },
+	{ "Last-Modified: Thu, 17 Oct 2019 07:18:26 GMT\r\n\r\n", "Last-Modified", "Thu, 17 Oct 2019 07:18:26 GMT" },
 	{ "Server: ECS (nyb/1D13)", "Server", "ECS (nyb/1D13)" },
 	{ "X-Cache: HIT", "X-Cache", "HIT" },
-	{ "Content-Length: 648", "Content-Length", "648" }
+	{ "Content-Length: 648", "Content-Length", "648" },
+	//These are request heaedrs but should work the same way
+	{ "Host: example.com\r\n", "Host", "example.com" },
+	{ "User-Agent: curl/7.80.0", "User-Agent", "curl/7.80.0" },
+	{ "Accept: */*", "Accept", "*/*" },
+	{ "Accept: */*", "Accept", "*/*" },
 };
 
 MunitResult testHeadersName(const MunitParameter params[],

tests/response.test.c

@@ -1,17 +1,46 @@
 #ifndef RESPONSE_TEST
 #define RESPONSE_TEST value
 
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/socket.h>
+#include <stdlib.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <string.h>
+#include <stdbool.h>
+
 #include "munit/munit.h"
+#ifndef READLINE_C
+#define READLINE_C
+#include "../src/readline.c"
+#endif
 #ifndef REQUESTRESPONSE_C
-#define REQUESTRESPONSE_C value
+#define REQUESTRESPONSE_C
 #include "../src/requestresponse.c"
 #endif /* ifndef REQUESTRESPONSE_C */
 #include "../src/response.c"
 
+typedef struct {
+	char *fullLine;
+	float version;
+	int statusCode;
+	char *statusMessage;
+} firstLine;
+
+static firstLine line1Examples[] = {
+	//Full Line,               version      status Code       Status Message
+	{ "HTTP/1.1 200 OK",         1.1,        200,               "OK" },
+	{ "HTTP/2.0 404 Not Found",  2.0,        404,               "Not Found" },
+	{ NULL,                      0,          0,                 NULL }
+};
+
 
 MunitResult testResponseNewStatus(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Response *rsp = newResponse();
+	responseBarebones(rsp);
 	munit_assert_int( rsp->statusCode, ==, 200 );
 	return MUNIT_OK;
 }
@@ -19,6 +48,7 @@ MunitResult testResponseNewStatus(const MunitParameter params[],
 MunitResult testResponseNewStatusMessage(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Response *rsp = newResponse();
+	responseBarebones(rsp);
 	munit_assert_string_equal( rsp->statusMessage, "OK" );
 	return MUNIT_OK;
 }
@@ -26,6 +56,7 @@ MunitResult testResponseNewStatusMessage(const MunitParameter params[],
 MunitResult testResponseNewVersion(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Response *rsp = newResponse();
+	responseBarebones(rsp);
 	munit_assert_float( rsp->version, ==, 1.1 );
 	return MUNIT_OK;
 }
@@ -33,6 +64,7 @@ MunitResult testResponseNewVersion(const MunitParameter params[],
 MunitResult testResponseSetBody(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Response *rsp = newResponse();
+	responseBarebones(rsp);
 	responseSetBody( rsp, "Testing", 1 );
 	munit_assert_string_equal( rsp->body, "Testing" );
 	munit_assert_string_equal( getHeader( rsp->headers, "content-length" )->value, "7" );
@@ -42,11 +74,122 @@ MunitResult testResponseSetBody(const MunitParameter params[],
 MunitResult testResponseToString(const MunitParameter params[],
 		void* user_data_or_fixture){
 	Response *rsp = newResponse();
+	responseBarebones(rsp);
+	munit_assert_string_equal( responseToString( rsp ), "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Type: text/plain\r\n\r\n" );
 	responseSetBody( rsp, "Testing", 1 );
-	munit_assert_string_equal( responseToString( rsp ), "HTTP/1.1 200 OK\r\nContent-Length: 7\r\nContent-Type: text/plain\r\n\r\nTesting" );
+	munit_assert_string_equal( responseToString( rsp ), "HTTP/1.1 200 OK\r\nContent-Length: 7\r\nContent-Type: text/plain\r\n\r\nTesting\r\n" );
 	return MUNIT_OK;
 }
 
+MunitResult testResponseFirstLineStatusCode(const MunitParameter params[],
+		void* user_data_or_fixture){
+	Response *rsp;
+	for ( firstLine *line = line1Examples; line->fullLine != NULL; line++ ){
+		rsp = newResponse();
+		responseFirstLine( rsp, line->fullLine );
+		munit_assert_int( rsp->statusCode, ==, line->statusCode );
+		free( rsp );
+	}
+	return MUNIT_OK;
+}
+
+MunitResult testResponseFirstLineStatusMessage(const MunitParameter params[],
+		void* user_data_or_fixture){
+	Response *rsp;
+	for ( firstLine *line = line1Examples; line->fullLine != NULL; line++ ){
+		rsp = newResponse();
+		responseFirstLine( rsp, line->fullLine );
+		munit_assert_string_equal( rsp->statusMessage, line->statusMessage );
+		free( rsp );
+	}
+	return MUNIT_OK;
+}
+
+MunitResult testResponseFirstLineVersion(const MunitParameter params[],
+		void* user_data_or_fixture){
+	Response *rsp;
+	for ( firstLine *line = line1Examples; line->fullLine != NULL; line++ ){
+		rsp = newResponse();
+		responseFirstLine( rsp, line->fullLine );
+		munit_assert_float( rsp->version, ==, line->version );
+		free( rsp );
+	}
+	return MUNIT_OK;
+}
+
+static void* setupSocketTests(const MunitParameter params[], void* user_data) {
+	int client_fd = 0;
+	struct sockaddr_in address;
+	memset( &address, 0, sizeof(address) );
+	struct hostent *host = gethostbyname("example.com");
+	Response *rsp = NULL;
+
+	if ((client_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0){
+		perror("socket failed");
+		return NULL;
+	}
+
+
+	address.sin_family = AF_INET;
+	address.sin_port = htons( 80 );
+	// We want the request to go out to whatever the host was resolved to
+	memcpy( &address.sin_addr, host->h_addr_list[0], host->h_length );
+
+	if((connect(client_fd, (struct sockaddr *)&address, sizeof(address)))<0)  {  
+		perror("connect failed");
+		return NULL;
+	}
+
+	char *toSend = "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n";
+
+	if ( write( client_fd, toSend, strlen(toSend) ) != strlen(toSend) ){
+		perror( "Write Error" );
+		return NULL;
+	}
+
+	rsp = newResponseFromSocket( client_fd );
+	return rsp;
+}
+
+MunitResult testResponseFromSocketHeaderLength(const MunitParameter params[],
+		void* user_data_or_fixture){
+	Response *rsp = ( Response * )user_data_or_fixture;
+	if ( rsp == NULL ) return MUNIT_ERROR;
+
+	//Check there are the right nubmer of haeders
+	// Unfortunately it changes from request to request so this is a "loose"
+	// check
+	munit_assert_int( countHeaders( rsp->headers ), >, 7 );
+
+	return MUNIT_OK;
+}
+
+MunitResult testResponseFromSocketStatusCode(const MunitParameter params[],
+		void* user_data_or_fixture){
+	Response *rsp = ( Response * )user_data_or_fixture;
+	if ( rsp == NULL ) return MUNIT_ERROR;
+
+	//Check there are the right nubmer of haeders
+	munit_assert_int( rsp->statusCode, ==, 200 );
+
+	return MUNIT_OK;
+}
+
+MunitResult testResponseFromSocketBody(const MunitParameter params[],
+		void* user_data_or_fixture){
+	Response *rsp = ( Response * )user_data_or_fixture;
+	Header *contentLength = getHeader( rsp->headers, "content-length" );
+	unsigned int length = 0;
+
+	if ( contentLength == NULL ) return MUNIT_ERROR;
+	
+	munit_assert_int( atoi( contentLength->value ), ==, strlen( (char *)rsp->body ) );
+
+	//Check there are the right nubmer of haeders
+	munit_assert_int( rsp->statusCode, ==, 200 );
+
+	return MUNIT_OK;
+}
 
 static MunitTest response_tests[] = {
 	{
@@ -84,6 +227,48 @@ static MunitTest response_tests[] = {
 		NULL, /* tear_down */
 		MUNIT_TEST_OPTION_NONE, /* options */
 		NULL /* parameters */
+	}, {
+		"/line1/statusCode", /* name */
+		testResponseFirstLineStatusCode, /* test */
+		NULL, /* setup */
+		NULL, /* tear_down */
+		MUNIT_TEST_OPTION_NONE, /* options */
+		NULL /* parameters */
+	}, {
+		"/line1/statusMessage", /* name */
+		testResponseFirstLineStatusMessage, /* test */
+		NULL, /* setup */
+		NULL, /* tear_down */
+		MUNIT_TEST_OPTION_NONE, /* options */
+		NULL /* parameters */
+	}, {
+		"/line1/version", /* name */
+		testResponseFirstLineVersion, /* test */
+		NULL, /* setup */
+		NULL, /* tear_down */
+		MUNIT_TEST_OPTION_NONE, /* options */
+		NULL /* parameters */
+	}, {
+		"/fromSocket/headerLength", /* name */
+		testResponseFromSocketHeaderLength, /* test */
+		setupSocketTests, /* setup */
+		NULL, /* tear_down */
+		MUNIT_TEST_OPTION_NONE, /* options */
+		NULL /* parameters */
+	}, {
+		"/fromSocket/statusCode", /* name */
+		testResponseFromSocketStatusCode, /* test */
+		setupSocketTests, /* setup */
+		NULL, /* tear_down */
+		MUNIT_TEST_OPTION_NONE, /* options */
+		NULL /* parameters */
+	}, {
+		"/fromSocket/body", /* name */
+		testResponseFromSocketBody, /* test */
+		setupSocketTests, /* setup */
+		NULL, /* tear_down */
+		MUNIT_TEST_OPTION_NONE, /* options */
+		NULL /* parameters */
 	},
 /* Mark the end of the array with an entry where the test
 * function is NULL */

tests/util.test.c

@@ -0,0 +1,51 @@
+#ifndef UTIL_TEST
+#define UTIL_TEST
+
+
+#include "munit/munit.h"
+
+#include "../src/util.h"
+
+
+
+
+MunitResult testStrPos(const MunitParameter params[],
+		void* user_data_or_fixture){
+	munit_assert_int( strpos( "thisisatest", "test" ), ==, 7 );
+	munit_assert_int( strpos( "testthisisatest", "test" ), ==, 0 );
+	munit_assert_int( strpos( "blar", "test" ), ==, -1 );
+	return MUNIT_OK;
+}
+
+
+static MunitTest util_tests[] = {
+	{
+		"/util/versions", /* name */
+		testStrPos, /* test */
+		NULL, /* setup */
+		NULL, /* tear_down */
+		MUNIT_TEST_OPTION_NONE, /* options */
+		NULL /* parameters */
+	},
+/* Mark the end of the array with an entry where the test
+* function is NULL */
+	{ NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL }
+};
+
+MunitSuite util_test_suite = {
+	"/request", /* name */
+	util_tests, /* tests */
+	NULL, /* suites */
+	1, /* iterations */
+	MUNIT_SUITE_OPTION_NONE /* options */
+};
+
+#ifndef MAINTEST
+#define MAINTEST
+
+int main (int argc, char* argv[]) {
+	return munit_suite_main(&util_test_suite, NULL, argc, argv);
+}
+
+#endif /* ifndef MAINTEST */
+#endif /* ifndef REQUEST_TEST */