"HTTPException", "NotConnected", "UnknownProtocol",

"UnknownTransferEncoding", "UnimplementedFileMode",

"IncompleteRead", "InvalidURL", "ImproperConnectionState",

"CannotSendRequest", "CannotSendHeader", "ResponseNotReady",

100: 'Continue',

101: 'Switching Protocols',

200: 'OK',

201: 'Created',

202: 'Accepted',

203: 'Non-Authoritative Information',

204: 'No Content',

205: 'Reset Content',

206: 'Partial Content',

300: 'Multiple Choices',

301: 'Moved Permanently',

302: 'Found',

303: 'See Other',

304: 'Not Modified',

305: 'Use Proxy',

306: '(Unused)',

307: 'Temporary Redirect',

400: 'Bad Request',

401: 'Unauthorized',

402: 'Payment Required',

403: 'Forbidden',

404: 'Not Found',

405: 'Method Not Allowed',

406: 'Not Acceptable',

407: 'Proxy Authentication Required',

408: 'Request Timeout',

409: 'Conflict',

410: 'Gone',

411: 'Length Required',

412: 'Precondition Failed',

413: 'Request Entity Too Large',

414: 'Request-URI Too Long',

415: 'Unsupported Media Type',

416: 'Requested Range Not Satisfiable',

417: 'Expectation Failed',

428: 'Precondition Required',

429: 'Too Many Requests',

431: 'Request Header Fields Too Large',

500: 'Internal Server Error',

501: 'Not Implemented',

502: 'Bad Gateway',

503: 'Service Unavailable',

504: 'Gateway Timeout',

505: 'HTTP Version Not Supported',

511: 'Network Authentication Required',

# XXX The only usage of this method is in

# http.server.CGIHTTPRequestHandler. Maybe move the code there so

# that it doesn't need to be part of the public API. The API has

# never been defined so this could cause backwards compatibility

# issues.

def getallmatchingheaders(self, name):

"""Find all header lines matching a given header name.

name = name.lower() + ':'

n = len(name)

lst = []

hit = 0

for line in self.keys():

if line[:n].lower() == name:

hit = 1

elif not line[:1].isspace():

hit = 0

if hit:

lst.append(line)

"""Parses only RFC2822 headers from a file pointer.

headers = []

while True:

line = fp.readline(_MAXLINE + 1)

if len(line) > _MAXLINE:

raise LineTooLong("header line")

headers.append(line)

if len(headers) > _MAXHEADERS:

raise HTTPException("got more than %d headers" % _MAXHEADERS)

if line in (b'\r\n', b'\n', b''):

break

hstring = b''.join(headers).decode('iso-8859-1')

# See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.

# The bytes from the socket object are iso-8859-1 strings.

# See RFC 2616 sec 2.2 which notes an exception for MIME-encoded

# text following RFC 2047. The basic status line parsing only

# accepts iso-8859-1.

def __init__(self, sock, debuglevel=0, strict=_strict_sentinel, method=None, url=None):

# If the response includes a content-length header, we need to

# make sure that the client doesn't read more than the

# specified number of bytes. If it does, it will block until

# the server times out and closes the connection. This will

# happen if a self.fp.read() is done (without a size) whether

# self.fp is buffered or not. So, no self.fp.read() by

# clients unless they know what they are doing.

self.fp = sock.makefile("rb")

self.debuglevel = debuglevel

if strict is not _strict_sentinel:

warnings.warn("the 'strict' argument isn't supported anymore; "

"http.client now always assumes HTTP/1.x compliant servers.",

DeprecationWarning, 2)

self._method = method

# The HTTPResponse object is returned via urllib. The clients

# of http and urllib expect different attributes for the

# headers. headers is used here and supports urllib. msg is

# provided as a backwards compatibility layer for http

# clients.

self.headers = self.msg = None

# from the Status-Line of the response

self.version = _UNKNOWN # HTTP-Version

self.status = _UNKNOWN # Status-Code

self.reason = _UNKNOWN # Reason-Phrase

self.chunked = _UNKNOWN # is "chunked" being used?

self.chunk_left = _UNKNOWN # bytes left to read in current chunk

self.length = _UNKNOWN # number of bytes left in response

self.will_close = _UNKNOWN # conn will close at end of response

def _read_status(self):

line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")

if len(line) > _MAXLINE:

raise LineTooLong("status line")

if self.debuglevel > 0:

print("reply:", repr(line))

if not line:

# Presumably, the server closed the connection before

# sending a valid response.

raise BadStatusLine(line)

try:

version, status, reason = line.split(None, 2)

except ValueError:

try:

version, status = line.split(None, 1)

reason = ""

except ValueError:

# empty version will cause next test to fail.

version = ""

if not version.startswith("HTTP/"):

self._close_conn()

raise BadStatusLine(line)

# The status code is a three-digit number

try:

status = int(status)

if status < 100 or status > 999:

raise BadStatusLine(line)

except ValueError:

raise BadStatusLine(line)

return version, status, reason

def begin(self):

if self.headers is not None:

# we've already started reading the response

return

# read until we get a non-100 response

while True:

version, status, reason = self._read_status()

if status != CONTINUE:

break

# skip the header from the 100 response

while True:

skip = self.fp.readline(_MAXLINE + 1)

if len(skip) > _MAXLINE:

raise LineTooLong("header line")

skip = skip.strip()

if not skip:

break

if self.debuglevel > 0:

print("header:", skip)

self.code = self.status = status

self.reason = reason.strip()

if version in ("HTTP/1.0", "HTTP/0.9"):

# Some servers might still return "0.9", treat it as 1.0 anyway

self.version = 10

elif version.startswith("HTTP/1."):

self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1

else:

raise UnknownProtocol(version)

self.headers = self.msg = parse_headers(self.fp)

if self.debuglevel > 0:

for hdr in self.headers:

print("header:", hdr, end=" ")

# are we using the chunked-style of transfer encoding?

tr_enc = self.headers.get("transfer-encoding")

if tr_enc and tr_enc.lower() == "chunked":

self.chunked = True

self.chunk_left = None

else:

self.chunked = False

# will the connection close at the end of the response?

self.will_close = self._check_close()

# do we have a Content-Length?

# NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"

self.length = None

length = self.headers.get("content-length")

# are we using the chunked-style of transfer encoding?

tr_enc = self.headers.get("transfer-encoding")

if length and not self.chunked:

try:

self.length = int(length)

except ValueError:

self.length = None

else:

if self.length < 0: # ignore nonsensical negative lengths

self.length = None

else:

self.length = None

# does the body have a fixed length? (of zero)

if (status == NO_CONTENT or status == NOT_MODIFIED or

100 <= status < 200 or # 1xx codes

self._method == "HEAD"):

self.length = 0

# if the connection remains open, and we aren't using chunked, and

# a content-length was not provided, then assume that the connection

# WILL close.

if (not self.will_close and

not self.chunked and

self.length is None):

self.will_close = True

def _check_close(self):

conn = self.headers.get("connection")

if self.version == 11:

# An HTTP/1.1 proxy is assumed to stay open unless

# explicitly closed.

conn = self.headers.get("connection")

if conn and "close" in conn.lower():

return True

return False

# Some HTTP/1.0 implementations have support for persistent

# connections, using rules different than HTTP/1.1.

# For older HTTP, Keep-Alive indicates persistent connection.

if self.headers.get("keep-alive"):

return False

# At least Akamai returns a "Connection: Keep-Alive" header,

# which was supposed to be sent by the client.

if conn and "keep-alive" in conn.lower():

return False

# Proxy-Connection is a netscape hack.

pconn = self.headers.get("proxy-connection")

if pconn and "keep-alive" in pconn.lower():

return False

# otherwise, assume it will close

return True

def _close_conn(self):

fp = self.fp

self.fp = None

fp.close()

def close(self):

super().close() # set "closed" flag

if self.fp:

self._close_conn()

# These implementations are for the benefit of io.BufferedReader.

# XXX This class should probably be revised to act more like

# the "raw stream" that BufferedReader expects.

def flush(self):

super().flush()

if self.fp:

self.fp.flush()

def readable(self):

return True

# End of "raw stream" methods

def isclosed(self):

"""True if the connection is closed."""

# NOTE: it is possible that we will not ever call self.close(). This

# case occurs when will_close is TRUE, length is None, and we

# read up to the last byte, but NOT past it.

#

# IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be

# called, meaning self.isclosed() is meaningful.

return self.fp is None

def read(self, amt=None):

if self.fp is None:

return b""

if self._method == "HEAD":

self._close_conn()

return b""

if amt is not None:

# Amount is given, so call base class version

# (which is implemented in terms of self.readinto)

return super(HTTPResponse, self).read(amt)

else:

# Amount is not given (unbounded read) so we must check self.length

# and self.chunked

if self.chunked:

return self._readall_chunked()

if self.length is None:

s = self.fp.read()

else:

try:

s = self._safe_read(self.length)

except IncompleteRead:

self._close_conn()

raise

self.length = 0

self._close_conn() # we read everything

return s

def readinto(self, b):

if self.fp is None:

return 0

if self._method == "HEAD":

self._close_conn()

return 0

if self.chunked:

return self._readinto_chunked(b)

if self.length is not None:

if len(b) > self.length:

# clip the read to the "end of response"

b = memoryview(b)[0:self.length]

# we do not use _safe_read() here because this may be a .will_close

# connection, and the user is reading more bytes than will be provided

# (for example, reading in 1k chunks)

n = self.fp.readinto(b)

if not n and b:

# Ideally, we would raise IncompleteRead if the content-length

# wasn't satisfied, but it might break compatibility.

self._close_conn()

elif self.length is not None:

self.length -= n

if not self.length:

self._close_conn()

return n

def _read_next_chunk_size(self):

# Read the next chunk size from the file

line = self.fp.readline(_MAXLINE + 1)

if len(line) > _MAXLINE:

raise LineTooLong("chunk size")

i = line.find(b";")

if i >= 0:

line = line[:i] # strip chunk-extensions

try:

return int(line, 16)

except ValueError:

# close the connection as protocol synchronisation is

# probably lost

self._close_conn()

raise

def _read_and_discard_trailer(self):

# read and discard trailer up to the CRLF terminator

### note: we shouldn't have any trailers!

while True:

line = self.fp.readline(_MAXLINE + 1)

if len(line) > _MAXLINE:

raise LineTooLong("trailer line")

if not line:

# a vanishingly small number of sites EOF without

# sending the trailer

break

if line in (b'\r\n', b'\n', b''):

break

def _readall_chunked(self):

assert self.chunked != _UNKNOWN

chunk_left = self.chunk_left

value = []

while True:

if chunk_left is None:

try:

chunk_left = self._read_next_chunk_size()

if chunk_left == 0:

break

except ValueError:

raise IncompleteRead(b''.join(value))

value.append(self._safe_read(chunk_left))

# we read the whole chunk, get another

self._safe_read(2) # toss the CRLF at the end of the chunk

chunk_left = None

self._read_and_discard_trailer()

# we read everything; close the "file"

self._close_conn()

return b''.join(value)

def _readinto_chunked(self, b):

assert self.chunked != _UNKNOWN

chunk_left = self.chunk_left

total_bytes = 0

mvb = memoryview(b)

while True:

if chunk_left is None:

try:

chunk_left = self._read_next_chunk_size()

if chunk_left == 0:

break

except ValueError:

raise IncompleteRead(bytes(b[0:total_bytes]))

if len(mvb) < chunk_left:

n = self._safe_readinto(mvb)

self.chunk_left = chunk_left - n

return total_bytes + n

elif len(mvb) == chunk_left:

n = self._safe_readinto(mvb)

self._safe_read(2) # toss the CRLF at the end of the chunk

self.chunk_left = None

return total_bytes + n

else:

temp_mvb = mvb[0:chunk_left]

n = self._safe_readinto(temp_mvb)

mvb = mvb[n:]

total_bytes += n

# we read the whole chunk, get another

self._safe_read(2) # toss the CRLF at the end of the chunk

chunk_left = None

self._read_and_discard_trailer()

# we read everything; close the "file"

self._close_conn()

return total_bytes

def _safe_read(self, amt):

"""Read the number of bytes requested, compensating for partial reads.

s = []

while amt > 0:

chunk = self.fp.read(min(amt, MAXAMOUNT))

if not chunk:

raise IncompleteRead(b''.join(s), amt)

s.append(chunk)

amt -= len(chunk)

return b"".join(s)

def _safe_readinto(self, b):

"""Same as _safe_read, but for reading into a buffer."""

total_bytes = 0

mvb = memoryview(b)

while total_bytes < len(b):

if MAXAMOUNT < len(mvb):

temp_mvb = mvb[0:MAXAMOUNT]

n = self.fp.readinto(temp_mvb)

else:

n = self.fp.readinto(mvb)

if not n:

raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b))

mvb = mvb[n:]

total_bytes += n

return total_bytes

def fileno(self):

return self.fp.fileno()

def getheader(self, name, default=None):

if self.headers is None:

raise ResponseNotReady()

headers = self.headers.get_all(name) or default

if isinstance(headers, str) or not hasattr(headers, '__iter__'):

return headers

else:

return ', '.join(headers)

def getheaders(self):

"""Return list of (header, value) tuples."""

if self.headers is None:

raise ResponseNotReady()

return list(self.headers.items())

# We override IOBase.__iter__ so that it doesn't check for closed-ness

def __iter__(self):

return self

# For compatibility with old-style urllib responses.

def info(self):

return self.headers

def geturl(self):

return self.url

def getcode(self):

_http_vsn = 11

_http_vsn_str = 'HTTP/1.1'

response_class = HTTPResponse

default_port = HTTP_PORT

auto_open = 1

debuglevel = 0

def __init__(self, host, port=None, strict=_strict_sentinel,

timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):

if strict is not _strict_sentinel:

warnings.warn("the 'strict' argument isn't supported anymore; "

"http.client now always assumes HTTP/1.x compliant servers.",

DeprecationWarning, 2)

self.timeout = timeout

self.source_address = source_address

self.sock = None

self._buffer = []

self.__response = None

self.__state = _CS_IDLE

self._method = None

self._tunnel_host = None

self._tunnel_port = None

self._tunnel_headers = {}

self._set_hostport(host, port)

def set_tunnel(self, host, port=None, headers=None):

""" Sets up the host and the port for the HTTP CONNECT Tunnelling.

self._tunnel_host = host

self._tunnel_port = port

if headers:

self._tunnel_headers = headers

else:

self._tunnel_headers.clear()

def _set_hostport(self, host, port):

if port is None:

i = host.rfind(':')

j = host.rfind(']') # ipv6 addresses have [...]

if i > j:

try:

port = int(host[i+1:])

except ValueError:

if host[i+1:] == "": # http://foo.com:/ == http://foo.com/

port = self.default_port

else:

raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])

host = host[:i]

else:

port = self.default_port

if host and host[0] == '[' and host[-1] == ']':

host = host[1:-1]

self.host = host

self.port = port

def set_debuglevel(self, level):

self.debuglevel = level

def _tunnel(self):

self._set_hostport(self._tunnel_host, self._tunnel_port)

connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port)

connect_bytes = connect_str.encode("ascii")

self.send(connect_bytes)

for header, value in self._tunnel_headers.items():

header_str = "%s: %s\r\n" % (header, value)

header_bytes = header_str.encode("latin-1")

self.send(header_bytes)

self.send(b'\r\n')

response = self.response_class(self.sock, method=self._method)

(version, code, message) = response._read_status()

if code != 200:

self.close()

raise socket.error("Tunnel connection failed: %d %s" % (code,

message.strip()))

while True:

line = response.fp.readline(_MAXLINE + 1)

if len(line) > _MAXLINE:

raise LineTooLong("header line")

if not line:

# for sites which EOF without sending a trailer

break

if line in (b'\r\n', b'\n', b''):

break

def connect(self):

"""Connect to the host and port specified in __init__."""

self.sock = socket.create_connection((self.host,self.port),

self.timeout, self.source_address)

if self._tunnel_host:

self._tunnel()

def close(self):

"""Close the connection to the HTTP server."""

if self.sock:

self.sock.close() # close it manually... there may be other refs

self.sock = None

if self.__response:

self.__response.close()

self.__response = None

self.__state = _CS_IDLE

def send(self, data):

"""Send `data' to the server.

if self.sock is None:

if self.auto_open:

self.connect()

else:

raise NotConnected()

if self.debuglevel > 0:

print("send:", repr(data))

blocksize = 8192

if hasattr(data, "read") :

if self.debuglevel > 0:

print("sendIng a read()able")

encode = False

try:

mode = data.mode

except AttributeError:

# io.BytesIO and other file-like objects don't have a `mode`

# attribute.

pass

else:

if "b" not in mode:

encode = True

if self.debuglevel > 0:

print("encoding file using iso-8859-1")

while 1:

datablock = data.read(blocksize)

if not datablock:

break

if encode:

datablock = datablock.encode("iso-8859-1")

self.sock.sendall(datablock)

return

try:

self.sock.sendall(data)

except TypeError:

if isinstance(data, collections.Iterable):

for d in data:

self.sock.sendall(d)

else:

raise TypeError("data should be a bytes-like object "

"or an iterable, got %r" % type(data))

def _output(self, s):

"""Add a line of output to the current request buffer.

self._buffer.append(s)

def _send_output(self, message_body=None):

"""Send the currently buffered request and clear the buffer.

self._buffer.extend((b"", b""))

msg = b"\r\n".join(self._buffer)

del self._buffer[:]

# If msg and message_body are sent in a single send() call,

# it will avoid performance problems caused by the interaction

# between delayed ack and the Nagle algorithm.

if isinstance(message_body, bytes):

msg += message_body

message_body = None

self.send(msg)

if message_body is not None:

# message_body was not a string (i.e. it is a file), and

# we must run the risk of Nagle.

self.send(message_body)

def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):

"""Send a request to the server.

# if a prior response has been completed, then forget about it.

if self.__response and self.__response.isclosed():

self.__response = None

# in certain cases, we cannot issue another request on this connection.

# this occurs when:

# 1) we are in the process of sending a request. (_CS_REQ_STARTED)

# 2) a response to a previous request has signalled that it is going

# to close the connection upon completion.

# 3) the headers for the previous response have not been read, thus

# we cannot determine whether point (2) is true. (_CS_REQ_SENT)

#

# if there is no prior response, then we can request at will.

#

# if point (2) is true, then we will have passed the socket to the

# response (effectively meaning, "there is no prior response"), and

# will open a new one when a new request is made.

#

# Note: if a prior response exists, then we *can* start a new request.

# We are not allowed to begin fetching the response to this new

# request, however, until that prior response is complete.

#

if self.__state == _CS_IDLE:

self.__state = _CS_REQ_STARTED

else:

raise CannotSendRequest(self.__state)

# Save the method we use, we need it later in the response phase

self._method = method

if not url:

url = '/'

request = '%s %s %s' % (method, url, self._http_vsn_str)

# Non-ASCII characters should have been eliminated earlier

self._output(request.encode('ascii'))

if self._http_vsn == 11:

# Issue some standard headers for better HTTP/1.1 compliance

if not skip_host:

# this header is issued *only* for HTTP/1.1

# connections. more specifically, this means it is

# only issued when the client uses the new

# HTTPConnection() class. backwards-compat clients

# will be using HTTP/1.0 and those clients may be

# issuing this header themselves. we should NOT issue

# it twice; some web servers (such as Apache) barf

# when they see two Host: headers