Groupees.com Grabber

Small script to access my Groupees profile and automatically download all my games and music written in python 3.

It depends on my CookieCon module, so just download both files, call python groupees.py and give it your user name and password. It will automatically start downloading all of your games.

If you only want some games, have a look at the source code. Editing it for your needs should be easy!

Groupees.py

#!/usr/bin/env python
#Python3 bindings for groupees.com
#to easiely access your profile,
#keys and downloads
__author__ = "Frederik Lauber"
__copyright__ = "Copyright 2014"
__license__ = "GPL3"
__version__ = "0.5"
__maintainer__ = "Frederik Lauber"
__status__ = "Development"
__contact__ = "https://flambda.de/impressum.html"

import os
import re
from string import Template
import urllib.error
from bs4 import BeautifulSoup
import CookieCon
URL_LOGIN = "https://groupees.com/login"
URL_LOGIN_AUTH = "https://groupees.com/auth/email_password"
URL_PROFILE_PAGE = Template("https://groupees.com/users/${user_id}/purchases?page=${page}")
URL_BASE = Template("https://groupees.com${relative}")
REGEX_USER_ID = r"""(?<=users/)\d+(?=/purchases)"""
REGEX_COVER_URL = r"""(?<=^background-image:url\(').*(?='\))"""

class _groupees_base_exception(Exception):
	pass

class _url_exception(_groupees_base_exception):
	def __init__(self, url):
		self.url = url
	def __str__(self):
		return "".join([str(self.__class__.__name__), "\nUrl: ", self.url])
class MultipleExceptions(_groupees_base_exception):
	def __init__(self, exceptions):
		self.exceptions = exceptions
	def __str__(self):
		tmp = []
		for e in self.exceptions:
			tmp.append(str(e))
		return "\n".join(tmp)
class NeitherLinkNoKey(_groupees_base_exception):
	def __str__(self):
		return str(self.__class__.__name__)
class ToSmallFile(_url_exception):
	pass
class LinkNotReachable(_url_exception):
	pass
class NoCover(_groupees_base_exception):
	pass
class NoLink(_groupees_base_exception):
	pass
class NoKey(_groupees_base_exception):
	pass
class LoginFailed(_groupees_base_exception):
	pass

class product(object):
	@property
	def name(self):
		#Get the name of the product
		if not hasattr(self, "_name"):
			self._name = self._soup.find("div", {"class" : "product"}).get_text().strip()
		return self._name

	@property
	def cover_url(self):
		#Get the relative path of the cover. Raises NoCover if no cover exists.
		if not hasattr(self, "_cover_url"):
			cover_url_style = self._soup.find("div", {"class" : "cover"}).get("style")
			cover_url_regexed = re.search(REGEX_COVER_URL, str(cover_url_style))
			self._cover_url = None if cover_url_regexed is None else cover_url_regexed.group(0)
		if self._cover_url is None:
			raise NoCover
		else:
			return self._cover_url

	def download_cover(self, folder, filename = None):
		self._con.urlretrieve(URL_BASE.substitute(relative = self.cover_url), folder, filename)

	@property
	def link_urls(self):
		if not hasattr(self, "_link_urls"):
			tmp = dict()
			for a in self._soup.find_all('a'):
				tmp[a.string] = a.get('href')
			self._link_urls = tmp if len(tmp) else None
		if self._link_urls is None:
			raise NoLink
		else:
			return self._link_urls

	def download_file(self, platform, folder, filename = None):
		self._con.urlretrieve(URL_BASE.substitute(relative = self.link_urls[platform]), folder, filename)

	@property
	def keys(self):
		if not hasattr(self, "_keys"):
			tmp = dict()
			for li in self._soup.find_all('li'):
				platform, key = li.find_all('span')
				tmp[platform.string] = key.string
			self._keys = tmp if len(tmp) else None
		if self._keys is None:
			raise NoKey
		else:
			return self._keys

	def __init__(self, con, soup):
		self._soup = soup
		self._con = con

	def auto_download(self, folder):
		path = os.path.join(folder, self.name.replace(":", ""))
		if not os.path.exists(path): os.makedirs(path)
		try:
			self.download_cover(path)
		except Exception:
			pass
		for platform in self.link_urls:
			try:
				self.download_file(platform, path)
			except Exception as E:
				print(E)

	def test(self, filesize_limit = 180):
		#Groupees has still a lot of defect profiles.
		#This functions tries to detect whose. 
		#The first criteria is:
		#	1. Every product needs at least one key or one link_url
		#If a link_url exists, headers are checked. If the file is smaller then
		#filesize_limit in kb, it will be regarded as defect.
		num = 0
		try:
			num += len(self.keys)
		except NoKey:
			pass
		try:
			num += len(self.link_urls)
		except NoLink:
			pass
		if not num:
			raise NeitherLinkNoKey
		#check if links have a bigger size than 180kb if they exist
		exception_list = []
		try:
			for platform in self.link_urls:
				url = URL_BASE.substitute(relative = self.link_urls[platform])
				try:
					(filename, filesize) = self._con.urlgetfileinfo(url)
					if filesize <= filesize_limit:
						exception_list.append(ToSmallFile(url))
				except urllib.error.HTTPError:
					exception_list.append(LinkNotReachable(url))
		except NoLink:
			pass
		else:
			if len(exception_list) == 1:
				raise exception_list[0]
			elif len(exception_list) > 1:
				raise MultipleExceptions(exception_list)

def _get_auth_and_userid(username, password):
	connector = CookieCon.CookieCon(userAgent="Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0")
	#as of 15.01.2014, groupees refuses auth if no userAgent is given
	data_for_user_id = connector.request(URL_LOGIN_AUTH, {'identifier': username, 'password': password})
	try:
		userid = re.search(REGEX_USER_ID, data_for_user_id).group(0)
	except AttributeError:
		raise LoginFailed
	return (connector, userid)

def collect_products(username, password):
	#Takes your user name and password and will return a list of groupees_products. Each groupees_product represents one 
	#item on your profile like a cd or a game. It can have multiple keys for different platforms and can have multiple downloadable
	#file. Please keep in mind that it does not need to have any at all. A NoKey, NoLink or NoCover Exception will be raised once
	#you try to access it.
	(connector, userid) = _get_auth_and_userid(username, password)
	products = []
	i = 0
	last_request = ""
	while not "No purchased Groupees yet" in last_request:
		last_request = connector.request(URL_PROFILE_PAGE.substitute(user_id = userid, page = str(i)))
		soup = BeautifulSoup(last_request)
		for div in soup.findAll("div", {"class" : "product-box"}):
			products.append(product(connector, div))
		i += 1
	return products

def download_all_by_platform(product_list, platform, folder):
	for prod in product_list:
		try:
			prod.download_file(platform, folder)
		except groupees_product.NoLink:
			pass

def download_all_inverted(product_list, platform_list, folder):
	for prod in product_list:
		try:
			for platform in prod.link_urls:
				if platform in platform_list:
					pass
				else:
					try:
						prod.download_file(platform, folder)
					except KeyError:
						pass
		except NoLink:
			pass

def create_report(product_list):
	line_list = []
	for i in product_list:
		try:
			i.test()
		except Exception as E:
			line_list.append(i.name)
			line_list.append(str(E))
	return "\n".join(line_list)

if __name__ == "__main__":
	import groupees
	print("Start")
	un = input("Please, type your email address\n")
	pw = input("Please, type your password\n")
	print("Now trying to accumulate all your products on groupees")
	try:
		p = groupees.collect_products(un, pw)
	except groupees.LoginFailed:
		print("Login Failed")
		print("Check email and password")
		exit(1)
	print("All products found")
	download_folder = input("Give folder for auto download (default is the current folder)\n")
	if download_folder == "":
		download_folder = "./"
	print("Starting to download")
	for i in p:
		print("Product:")
		print(i.name)
		try:
			i.auto_download(download_folder)
		except Exception as E:
			print(i.name)
			print(E)
	print("Stop")