As I often have to handle some webside login with python, I have been looking for a good module to handle Cookies etc. for.

One module for this is NetConnect but it was lacking some functions I was looking for.

So I ended up writing my own for my needs.
#!/usr/bin/env python
"""Python3 urllib wrapper with Cookie support </div></div>
Supports request and urlretrive on
Conenction which need Cookies
for auth or other"""</span>
__author__ = "Frederik Lauber"
__copyright__ = "Copyright 2014"
__license__ = "GPL3"
__version__ = "0.5"
__maintainer__ = "Frederik Lauber"
__status__ = "Development"
__contact__ = ""
import os
import re
from shutil import copyfileobj
from urllib.parse import urlencode
from urllib.request import build_opener, HTTPCookieProcessor, Request
from http.cookiejar import CookieJar
class CookieCon(object):
"""Object which holds all cookies etc."""
def __init__(self, encoding='utf-8', userAgent=None):
self._encoding = encoding
self._cookiejar = CookieJar()
self._opener = build_opener(HTTPCookieProcessor(self._cookiejar))
if not userAgent is None:
self._opener.addheaders = [('User-agent', userAgent)]
def _encode_dict(self, header_dict):
"""Function used to encode dicts based </div></div>
on the given encoding"""</span>
encoded_headers = dict()
for header, value in header_dict.items():
encoded_header = header.encode(self._encoding)
encoded_value = value.encode(self._encoding)
encoded_headers[encoded_header] = encoded_value
return encoded_headers
def _encode_url(self, params=None):
"""Function used to encode urls based </div></div>
on the given encoding
If params is None or has length 0,
None will be returned.
This was implemented as self._opener
also takes None as an argument if
no params are needed
if params is None or not len(params):
return None
return urlencode(self._encode_dict(params)).encode(self._encoding)
def request(self, url, params=None):
"""Usage: </div></div>
from CookieCon import CookieCon
con = CookieCon()
get-request, returns response
con.request("", {'foo': 'bar'})
post-request, returns response
with, self._encode_url(params)) as sock:
def urlgetfileinfo(self, url):
"""Returns a tuple with the filesize and filename </div></div>
as defined in the Content-Length and Content-Disposition headers.
If the header does not exist, None will be returned"""</span>
with, method="HEAD")) as sock:
header =['Content-Disposition']
filename ='(?<=").*(?=")', header).group(0)
except Exception:
filename = None
filesize = int(sock.headers['Content-Length'])
except KeyError:
filesize = None
return (filename, filesize)
class NoFileName(Exception):
def urlretrieve(self, url, folder, optname = None):
"""Usage: </div></div>
from CookieCon import CookieCon
con = CookieCon()
con.urlretrive("", "~/home/")
Downloads to ~/home/, resumes if file already exists.
The filename is discovered by the Content-Disposition header
con.urlretrive("", "~/home/", "")
Downloads to ~/home/, resumes if file already exists.
The given filename is used.
If no filename is given nor discovered, a NoFileName exception is raised.
This will not check that the given foldername and filename are valid but
raise an exception.
(urlname, urlfilesize) = self.urlgetfileinfo(url)
filename = optname if optname is not None else urlname
if filename is None:
raise self.NoFileName
filesize = float("inf") if urlfilesize is None else urlfilesize
filepath = os.path.join(folder, filename)
currentsize = os.path.getsize(filepath)
except os.error:
currentsize = 0
if currentsize < filesize:
headers = {"Range":"bytes=%s-" % currentsize}
encoded_headers = self._encode_dict(headers)
with, headers=encoded_headers)) as sock:
with open(filepath, "ab") as file:
copyfileobj(sock, file)
</pre></div></figure> Have fun and I hope it is as useful to you as it is to me.