Small script to access my Groupees profile and automatically download all my games and music written in python 3.
It depends on my CookieCon module, so just download both files, call python groupees.py and give it your username and password. It will automatically start downloading all of your games.
If you only want some games, have a look at the source code. Editing it for your needs should be easy!
Groupees.py
#!/usr/bin/env python
#Python3 bindings for groupees.com
#to easiely access your profile,
#keys and downloads
__author__ = "Frederik Lauber"
__copyright__ = "Copyright 2014"
__license__ = "GPL3"
__version__ = "0.5"
__maintainer__ = "Frederik Lauber"
__status__ = "Development"
__contact__ = "https://flambda.de/impressum.html"
import os
import re
from string import Template
import urllib.error
from bs4 import BeautifulSoup
import CookieCon
URL_LOGIN = "https://groupees.com/login"
URL_LOGIN_AUTH = "https://groupees.com/auth/email_password"
URL_PROFILE_PAGE = Template("https://groupees.com/users/${user_id}/purchases?page=${page}")
URL_BASE = Template("https://groupees.com${relative}")
REGEX_USER_ID = r"""(?<=users/)\d+(?=/purchases)"""
REGEX_COVER_URL = r"""(?<=^background-image:url\(').*(?='\))"""
class _groupees_base_exception(Exception):
pass
class _url_exception(_groupees_base_exception):
def __init__(self, url):
self.url = url
def __str__(self):
return "".join([str(self.__class__.__name__), "\nUrl: ", self.url])
class MultipleExceptions(_groupees_base_exception):
def __init__(self, exceptions):
self.exceptions = exceptions
def __str__(self):
tmp = []
for e in self.exceptions:
tmp.append(str(e))
return "\n".join(tmp)
class NeitherLinkNoKey(_groupees_base_exception):
def __str__(self):
return str(self.__class__.__name__)
class ToSmallFile(_url_exception):
pass
class LinkNotReachable(_url_exception):
pass
class NoCover(_groupees_base_exception):
pass
class NoLink(_groupees_base_exception):
pass
class NoKey(_groupees_base_exception):
pass
class LoginFailed(_groupees_base_exception):
pass
class product(object):
@property
def name(self):
#Get the name of the product
if not hasattr(self, "_name"):
self._name = self._soup.find("div", {"class" : "product"}).get_text().strip()
return self._name
@property
def cover_url(self):
#Get the relative path of the cover. Raises NoCover if no cover exists.
if not hasattr(self, "_cover_url"):
cover_url_style = self._soup.find("div", {"class" : "cover"}).get("style")
cover_url_regexed = re.search(REGEX_COVER_URL, str(cover_url_style))
self._cover_url = None if cover_url_regexed is None else cover_url_regexed.group(0)
if self._cover_url is None:
raise NoCover
else:
return self._cover_url
def download_cover(self, folder, filename = None):
self._con.urlretrieve(URL_BASE.substitute(relative = self.cover_url), folder, filename)
@property
def link_urls(self):
if not hasattr(self, "_link_urls"):
tmp = dict()
for a in self._soup.find_all('a'):
tmp[a.string] = a.get('href')
self._link_urls = tmp if len(tmp) else None
if self._link_urls is None:
raise NoLink
else:
return self._link_urls
def download_file(self, platform, folder, filename = None):
self._con.urlretrieve(URL_BASE.substitute(relative = self.link_urls[platform]), folder, filename)
@property
def keys(self):
if not hasattr(self, "_keys"):
tmp = dict()
for li in self._soup.find_all('li'):
platform, key = li.find_all('span')
tmp[platform.string] = key.string
self._keys = tmp if len(tmp) else None
if self._keys is None:
raise NoKey
else:
return self._keys
def __init__(self, con, soup):
self._soup = soup
self._con = con
def auto_download(self, folder):
path = os.path.join(folder, self.name.replace(":", ""))
if not os.path.exists(path): os.makedirs(path)
try:
self.download_cover(path)
except Exception:
pass
for platform in self.link_urls:
try:
self.download_file(platform, path)
except Exception as E:
print(E)
def test(self, filesize_limit = 180):
#Groupees has still a lot of defect profiles.
#This functions tries to detect whose.
#The first criteria is:
# 1. Every product needs at least one key or one link_url
#If a link_url exists, headers are checked. If the file is smaller then
#filesize_limit in kb, it will be regarded as defect.
num = 0
try:
num += len(self.keys)
except NoKey:
pass
try:
num += len(self.link_urls)
except NoLink:
pass
if not num:
raise NeitherLinkNoKey
#check if links have a bigger size than 180kb if they exist
exception_list = []
try:
for platform in self.link_urls:
url = URL_BASE.substitute(relative = self.link_urls[platform])
try:
(filename, filesize) = self._con.urlgetfileinfo(url)
if filesize <= filesize_limit:
exception_list.append(ToSmallFile(url))
except urllib.error.HTTPError:
exception_list.append(LinkNotReachable(url))
except NoLink:
pass
else:
if len(exception_list) == 1:
raise exception_list[0]
elif len(exception_list) > 1:
raise MultipleExceptions(exception_list)
def _get_auth_and_userid(username, password):
connector = CookieCon.CookieCon(userAgent="Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0")
#as of 15.01.2014, groupees refuses auth if no userAgent is given
data_for_user_id = connector.request(URL_LOGIN_AUTH, {'identifier': username, 'password': password})
try:
userid = re.search(REGEX_USER_ID, data_for_user_id).group(0)
except AttributeError:
raise LoginFailed
return (connector, userid)
def collect_products(username, password):
#Takes your user name and password and will return a list of groupees_products. Each groupees_product represents one
#item on your profile like a cd or a game. It can have multiple keys for different platforms and can have multiple downloadable
#file. Please keep in mind that it does not need to have any at all. A NoKey, NoLink or NoCover Exception will be raised once
#you try to access it.
(connector, userid) = _get_auth_and_userid(username, password)
products = []
i = 0
last_request = ""
while not "No purchased Groupees yet" in last_request:
last_request = connector.request(URL_PROFILE_PAGE.substitute(user_id = userid, page = str(i)))
soup = BeautifulSoup(last_request)
for div in soup.findAll("div", {"class" : "product-box"}):
products.append(product(connector, div))
i += 1
return products
def download_all_by_platform(product_list, platform, folder):
for prod in product_list:
try:
prod.download_file(platform, folder)
except groupees_product.NoLink:
pass
def download_all_inverted(product_list, platform_list, folder):
for prod in product_list:
try:
for platform in prod.link_urls:
if platform in platform_list:
pass
else:
try:
prod.download_file(platform, folder)
except KeyError:
pass
except NoLink:
pass
def create_report(product_list):
line_list = []
for i in product_list:
try:
i.test()
except Exception as E:
line_list.append(i.name)
line_list.append(str(E))
return "\n".join(line_list)
if __name__ == "__main__":
import groupees
print("Start")
un = input("Please, type your email address\n")
pw = input("Please, type your password\n")
print("Now trying to accumulate all your products on groupees")
try:
p = groupees.collect_products(un, pw)
except groupees.LoginFailed:
print("Login Failed")
print("Check email and password")
exit(1)
print("All products found")
download_folder = input("Give folder for auto download (default is the current folder)\n")
if download_folder == "":
download_folder = "./"
print("Starting to download")
for i in p:
print("Product:")
print(i.name)
try:
i.auto_download(download_folder)
except Exception as E:
print(i.name)
print(E)
print("Stop")