Sending generic HTTP(S) requests in python

During Web Application Penetration tests I always need to automate requests, e.g. for fuzzing. While most of the local proxy/testing softwares (Burp, WebScarab, w3af, etc.) include a repeater/fuzzer feature, I often want to do addtional computations in python (e.g. calculating a hash and sending it as a fuzzed value or comparing parts of the response). The following script will take an entire HTTP(S) request as a string, parse it and send it to the server. As I show with the POST parameter “fuzzableParam” in this example, values can easily be fuzzed.

def send_this_request(http_request_string, remove_headers=None):
    """
    Always HTTP/1.1
    """
    import urllib2
    if remove_headers is None:
        remove_headers=['content-length', 'accept-encoding', 'accept-charset', 
        'accept-language', 'accept', 'keep-alive', 'connection', 'pragma', 
        'cache-control']
    for i, remove_header in enumerate(remove_headers):
        remove_headers[i] = remove_header.lower()
    if '\n\n' in http_request_string:
        headers, body = http_request_string.split('\n\n',1)
    else:
        headers = http_request_string
        body = None
    headers = headers.split('\n')
    request_line = headers[0]
    headers = headers[1:]

    method, rest = request_line.split(" ", 1)
    url, protocol = rest.rsplit(" ", 1)

    merge_host_header_into_url = False
    if url.startswith("http"):
        merge_host_header_into_url = False
    elif url.startswith("/"):
        info("Warning: Defaulting to HTTP. Please write URL as https:// if you want SSL")
        merge_host_header_into_url = True
    else:
        fatalError("Protocol not supported. URL must start with http or /")

    header_tuples = []
    for header in headers:
        name, value = header.split(": ", 1)
        if merge_host_header_into_url and name.lower() == 'host':
            url = 'http://'+value+url
        if not name.lower() in remove_headers:
            header_tuples.append((name, value))
            
    opener = urllib2.build_opener()
    opener.addheaders = header_tuples
    urllib2.install_opener(opener)

    try:
        return urllib2.urlopen(url, body, 15).read()
    except urllib2.HTTPError, e:
        info('The server couldn\'t fulfill the request. Error code:', e.code)
    except urllib2.URLError, e:
        info("URLError:", e.reason)
    except Exception, e:
        error("DIDNT WORK:", e)
        
def info(*text):
    print "[PY-INFO] "+str(" ".join(str(i) for i in text))

def error(*text):
    print "[PY-ERROR] "+str(" ".join(str(i) for i in text))

request = '''POST http://example.com/ HTTP/1.1
Host: example.com
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: de-de,de;q=0.8,en-us;q=0.5,en;q=0.3
Accept-Encoding: gzip, deflate
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
Keep-Alive: 115
Content-Type: application/x-www-form-urlencoded;charset=utf-8
Referer: http://example.com
Content-Length: 132
Cookie: test=somevalue; abc=123
DNT: 1
Connection: keep-alive
Pragma: no-cache
Cache-Control: no-cache

id=123&fuzzableParam='''

additionalValue = "&anotherParam=abc"

for i in ['78', '-1']:
    print send_this_request(request+i+additionalValue)