.BANCHO. Bancho now uses mysqldb instead of pymysql

2016-05-31 22:49:30 +02:00
parent 26f64e8610
commit aebf910890
11 changed files with 474 additions and 319 deletions
--- a/helpers/configHelper.py
+++ b/helpers/configHelper.py
@@ -46,10 +46,9 @@ class config:
 			self.config.get("db","username")
 			self.config.get("db","password")
 			self.config.get("db","database")
-			self.config.get("db","pingtime")
+			self.config.get("db","workers")

-			self.config.get("server","server")
-			self.config.get("server","host")
+			self.config.get("server","threads")
 			self.config.get("server","port")
 			self.config.get("server","localizeusers")
 			self.config.get("server","outputpackets")
@@ -57,12 +56,6 @@ class config:
 			self.config.get("server","timeouttime")
 			self.config.get("server","timeoutlooptime")

-			if self.config["server"]["server"] == "flask":
-				# Flask only config
-				self.config.get("flask","threaded")
-				self.config.get("flask","debug")
-				self.config.get("flask","logger")
-
 			self.config.get("discord","enable")
 			self.config.get("discord","boturl")

@@ -85,11 +78,10 @@ class config:
 		self.config.set("db", "username", "root")
 		self.config.set("db", "password", "")
 		self.config.set("db", "database", "ripple")
-		self.config.set("db", "pingtime", "600")
+		self.config.set("db", "workers", "4")

 		self.config.add_section("server")
-		self.config.set("server", "server", "tornado")
-		self.config.set("server", "host", "0.0.0.0")
+		self.config.set("server", "threads", "16")
 		self.config.set("server", "port", "5001")
 		self.config.set("server", "localizeusers", "1")
 		self.config.set("server", "outputpackets", "0")
@@ -97,18 +89,13 @@ class config:
 		self.config.set("server", "timeoutlooptime", "100")
 		self.config.set("server", "timeouttime", "100")

-		self.config.add_section("flask")
-		self.config.set("flask", "threaded", "1")
-		self.config.set("flask", "debug", "0")
-		self.config.set("flask", "logger", "0")
-
 		self.config.add_section("ci")
 		self.config.set("ci", "key", "changeme")

 		self.config.add_section("discord")
 		self.config.set("discord", "enable", "False")
 		self.config.set("discord", "boturl", "")
-		
+
 		# Write ini to file and close
 		self.config.write(f)
 		f.close()
--- a/helpers/databaseHelperNew.py
+++ b/helpers/databaseHelperNew.py
@@ -0,0 +1,118 @@
+import MySQLdb
+import threading
+
+class mysqlWorker:
+	"""
+	Instance of a pettirosso meme
+	"""
+	def __init__(self, wid, host, username, password, database):
+		"""
+		Create a pettirosso meme (mysql worker)
+
+		wid -- worker id
+		host -- hostname
+		username -- MySQL username
+		password -- MySQL password
+		database -- MySQL database name
+		"""
+		self.wid = wid
+		self.connection = MySQLdb.connect(host, username, password, database)
+		self.connection.autocommit(True)
+		self.ready = True
+		self.lock = threading.Lock()
+
+class db:
+	"""
+	A MySQL db connection with multiple workers
+	"""
+
+	def __init__(self, host, username, password, database, workers):
+		"""
+		Create MySQL workers aka pettirossi meme
+
+		host -- hostname
+		username -- MySQL username
+		password -- MySQL password
+		database -- MySQL database name
+		workers -- Number of workers to spawn
+		"""
+		#self.lock = threading.Lock()
+		#self.connection = MySQLdb.connect(host, username, password, database)
+
+		self.workers = []
+		self.lastWorker = 0
+		self.workersNumber = workers
+		for i in range(0,self.workersNumber):
+			print("> Spawning MySQL pettirosso meme {}".format(i))
+			self.workers.append(mysqlWorker(i, host, username, password, database))
+
+	def getWorker(self):
+		"""
+		Return a worker object (round-robin way)
+
+		return -- worker object
+		"""
+		if self.lastWorker >= self.workersNumber-1:
+			self.lastWorker = 0
+		else:
+			self.lastWorker += 1
+		#print("Using worker {}".format(self.lastWorker))
+		return self.workers[self.lastWorker]
+
+	def execute(self, query, params = ()):
+		"""
+		Executes a query
+
+		query -- Query to execute. You can bind parameters with %s
+		params -- Parameters list. First element replaces first %s and so on. Optional.
+		"""
+		# Get a worker and acquire its lock
+		worker = self.getWorker()
+		worker.lock.acquire()
+
+		try:
+			# Create cursor, execute query and commit
+			cursor = worker.connection.cursor(MySQLdb.cursors.DictCursor)
+			cursor.execute(query, params)
+			return cursor.lastrowid
+		finally:
+			# Close the cursor and release worker's lock
+			if cursor:
+				cursor.close()
+			worker.lock.release()
+
+	def fetch(self, query, params = (), all = False):
+		"""
+		Fetch a single value from db that matches given query
+
+		query -- Query to execute. You can bind parameters with %s
+		params -- Parameters list. First element replaces first %s and so on. Optional.
+		all -- Fetch one or all values. Used internally. Use fetchAll if you want to fetch all values.
+		"""
+		# Get a worker and acquire its lock
+		worker = self.getWorker()
+		worker.lock.acquire()
+
+		try:
+			# Create cursor, execute the query and fetch one/all result(s)
+			cursor = worker.connection.cursor(MySQLdb.cursors.DictCursor)
+			cursor.execute(query, params)
+			if all == True:
+				return cursor.fetchall()
+			else:
+				return cursor.fetchone()
+		finally:
+			# Close the cursor and release worker's lock
+			if cursor:
+				cursor.close()
+			worker.lock.release()
+
+	def fetchAll(self, query, params = ()):
+		"""
+		Fetch all values from db that matche given query.
+		Calls self.fetch with all = True.
+
+		query -- Query to execute. You can bind parameters with %s
+		params -- Parameters list. First element replaces first %s and so on. Optional.
+		"""
+		return self.fetch(query, params, True)
--- a/helpers/requestHelper.py
+++ b/helpers/requestHelper.py
@@ -0,0 +1,72 @@
+import tornado
+import tornado.web
+import tornado.gen
+from tornado.ioloop import IOLoop
+from objects import glob
+
+class asyncRequestHandler(tornado.web.RequestHandler):
+	"""
+	Tornado asynchronous request handler
+	create a class that extends this one (requestHelper.asyncRequestHandler)
+	use asyncGet() and asyncPost() instad of get() and post().
+	Done. I'm not kidding.
+	"""
+	@tornado.web.asynchronous
+	@tornado.gen.engine
+	def get(self, *args, **kwargs):
+		yield tornado.gen.Task(runBackground, (self.asyncGet, tuple(args), dict(kwargs)))
+
+	@tornado.web.asynchronous
+	@tornado.gen.engine
+	def post(self, *args, **kwargs):
+		yield tornado.gen.Task(runBackground, (self.asyncPost, tuple(args), dict(kwargs)))
+
+	def asyncGet(self, *args, **kwargs):
+		self.send_error(405)
+		self.finish()
+
+	def asyncPost(self, *args, **kwargs):
+		self.send_error(405)
+		self.finish()
+
+	def getRequestIP(self):
+		realIP = self.request.headers.get("X-Real-IP")
+		if realIP != None:
+			return realIP
+		return self.request.remote_ip
+
+
+def runBackground(data, callback):
+	"""
+	Run a function in the background.
+	Used to handle multiple requests at the same time
+	"""
+	func, args, kwargs = data
+	def _callback(result):
+		IOLoop.instance().add_callback(lambda: callback(result))
+	glob.pool.apply_async(func, args, kwargs, _callback)
+
+
+def checkArguments(arguments, requiredArguments):
+	"""
+	Check that every requiredArguments elements are in arguments
+
+	arguments -- full argument list, from tornado
+	requiredArguments -- required arguments list es: ["u", "ha"]
+	handler -- handler string name to print in exception. Optional
+	return -- True if all arguments are passed, none if not
+	"""
+	for i in requiredArguments:
+		if i not in arguments:
+			return False
+	return True
+
+def printArguments(t):
+	"""
+	Print passed arguments, for debug purposes
+
+	t -- tornado object (self)
+	"""
+	print("ARGS::")
+	for i in t.request.arguments:
+		print ("{}={}".format(i, t.get_argument(i)))
--- a/helpers/responseHelper.py
+++ b/helpers/responseHelper.py
@@ -1,47 +0,0 @@
-import flask
-import gzip
-
-def generateResponse(token, data = None):
-	"""
-	Return a flask response with required headers for osu! client, token and gzip compressed data
-
-	token -- user token
-	data -- plain response body
-	return -- flask response
-	"""
-
-	resp = flask.Response(gzip.compress(data, 6))
-	resp.headers['cho-token'] = token
-	resp.headers['cho-protocol'] = '19'
-	resp.headers['Keep-Alive'] = 'timeout=5, max=100'
-	resp.headers['Connection'] = 'keep-alive'
-	resp.headers['Content-Type'] = 'text/html; charset=UTF-8'
-	resp.headers['Vary'] = 'Accept-Encoding'
-	resp.headers['Content-Encoding'] = 'gzip'
-	return resp
-
-
-def HTMLResponse():
-	"""Return HTML bancho meme response"""
-
-	html = 	"<html><head><title>MA MAURO ESISTE?</title><style type='text/css'>body{width:30%}</style></head><body><pre>"
-	html += "           _                 __<br>"
-	html += "          (_)              /  /<br>"
-	html += "   ______ __ ____   ____  /  /____<br>"
-	html += "  /  ___/  /  _  \\/  _  \\/  /  _  \\<br>"
-	html += " /  /  /  /  /_) /  /_) /  /  ____/<br>"
-	html += "/__/  /__/  .___/  .___/__/ \\_____/<br>"
-	html += "        /  /   /  /<br>"
-	html += "       /__/   /__/<br>"
-	html += "<b>PYTHON > ALL VERSION</b><br><br>"
-	html += "<marquee style='white-space:pre;'><br>"
-	html += "                          .. o  .<br>"
-	html += "                         o.o o . o<br>"
-	html += "                        oo...<br>"
-	html += "                    __[]__<br>"
-	html += "    phwr-->  _\\:D/_/o_o_o_|__     <span style=\"font-family: 'Comic Sans MS'; font-size: 8pt;\">u wot m8</span><br>"
-	html += "             \\\"\"\"\"\"\"\"\"\"\"\"\"\"\"/<br>"
-	html += "              \\ . ..  .. . /<br>"
-	html += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^<br>"
-	html += "</marquee><br><strike>reverse engineering a protocol impossible to reverse engineer since always</strike><br>we are actually reverse engineering bancho successfully. for the third time.</pre></body></html>"
-	return html
--- a/helpers/userHelper.py
+++ b/helpers/userHelper.py
@@ -13,7 +13,7 @@ def getID(username):
 	"""

 	# Get user ID from db
-	userID = glob.db.fetch("SELECT id FROM users WHERE username = ?", [username])
+	userID = glob.db.fetch("SELECT id FROM users WHERE username = %s", [username])

 	# Make sure the query returned something
 	if userID == None:
@@ -34,7 +34,7 @@ def checkLogin(userID, password):
 	"""

 	# Get password data
-	passwordData = glob.db.fetch("SELECT password_md5, salt, password_version FROM users WHERE id = ?", [userID])
+	passwordData = glob.db.fetch("SELECT password_md5, salt, password_version FROM users WHERE id = %s", [userID])

 	# Make sure the query returned something
 	if passwordData == None:
@@ -48,7 +48,7 @@ def checkLogin(userID, password):
 		ok = passwordHelper.checkOldPassword(password, passwordData["salt"], passwordData["password_md5"])
 		if not ok: return False
 		newpass = passwordHelper.genBcrypt(password)
-		glob.db.execute("UPDATE users SET password_md5=?, salt='', password_version='2' WHERE id = ?", [newpass, userID])
+		glob.db.execute("UPDATE users SET password_md5=%s, salt='', password_version='2' WHERE id = %s", [newpass, userID])


 def exists(userID):
@@ -59,7 +59,7 @@ def exists(userID):
 	return -- bool
 	"""

-	result = glob.db.fetch("SELECT id FROM users WHERE id = ?", [userID])
+	result = glob.db.fetch("SELECT id FROM users WHERE id = %s", [userID])
 	if result == None:
 		return False
 	else:
@@ -74,7 +74,7 @@ def getAllowed(userID):
 	return -- allowed int
 	"""

-	return glob.db.fetch("SELECT allowed FROM users WHERE id = ?", [userID])["allowed"]
+	return glob.db.fetch("SELECT allowed FROM users WHERE id = %s", [userID])["allowed"]


 def getRankPrivileges(userID):
@@ -83,7 +83,7 @@ def getRankPrivileges(userID):
 	If you want to get that rank, user getUserGameRank instead
 	"""

-	return glob.db.fetch("SELECT rank FROM users WHERE id = ?", [userID])["rank"]
+	return glob.db.fetch("SELECT rank FROM users WHERE id = %s", [userID])["rank"]


 def getSilenceEnd(userID):
@@ -95,7 +95,7 @@ def getSilenceEnd(userID):
 	return -- UNIX time
 	"""

-	return glob.db.fetch("SELECT silence_end FROM users WHERE id = ?", [userID])["silence_end"]
+	return glob.db.fetch("SELECT silence_end FROM users WHERE id = %s", [userID])["silence_end"]


 def silence(userID, silenceEndTime, silenceReason):
@@ -108,7 +108,7 @@ def silence(userID, silenceEndTime, silenceReason):
 	silenceReason -- Silence reason shown on website
 	"""

-	glob.db.execute("UPDATE users SET silence_end = ?, silence_reason = ? WHERE id = ?", [silenceEndTime, silenceReason, userID])
+	glob.db.execute("UPDATE users SET silence_end = %s, silence_reason = %s WHERE id = %s", [silenceEndTime, silenceReason, userID])

 def getRankedScore(userID, gameMode):
 	"""
@@ -120,7 +120,7 @@ def getRankedScore(userID, gameMode):
 	"""

 	modeForDB = gameModes.getGameModeForDB(gameMode)
-	return glob.db.fetch("SELECT ranked_score_"+modeForDB+" FROM users_stats WHERE id = ?", [userID])["ranked_score_"+modeForDB]
+	return glob.db.fetch("SELECT ranked_score_"+modeForDB+" FROM users_stats WHERE id = %s", [userID])["ranked_score_"+modeForDB]


 def getTotalScore(userID, gameMode):
@@ -133,7 +133,7 @@ def getTotalScore(userID, gameMode):
 	"""

 	modeForDB = gameModes.getGameModeForDB(gameMode)
-	return glob.db.fetch("SELECT total_score_"+modeForDB+" FROM users_stats WHERE id = ?", [userID])["total_score_"+modeForDB]
+	return glob.db.fetch("SELECT total_score_"+modeForDB+" FROM users_stats WHERE id = %s", [userID])["total_score_"+modeForDB]


 def getAccuracy(userID, gameMode):
@@ -146,7 +146,7 @@ def getAccuracy(userID, gameMode):
 	"""

 	modeForDB = gameModes.getGameModeForDB(gameMode)
-	return glob.db.fetch("SELECT avg_accuracy_"+modeForDB+" FROM users_stats WHERE id = ?", [userID])["avg_accuracy_"+modeForDB]
+	return glob.db.fetch("SELECT avg_accuracy_"+modeForDB+" FROM users_stats WHERE id = %s", [userID])["avg_accuracy_"+modeForDB]


 def getGameRank(userID, gameMode):
@@ -159,7 +159,7 @@ def getGameRank(userID, gameMode):
 	"""

 	modeForDB = gameModes.getGameModeForDB(gameMode)
-	result = glob.db.fetch("SELECT position FROM leaderboard_"+modeForDB+" WHERE user = ?", [userID])
+	result = glob.db.fetch("SELECT position FROM leaderboard_"+modeForDB+" WHERE user = %s", [userID])
 	if result == None:
 		return 0
 	else:
@@ -176,7 +176,7 @@ def getPlaycount(userID, gameMode):
 	"""

 	modeForDB = gameModes.getGameModeForDB(gameMode)
-	return glob.db.fetch("SELECT playcount_"+modeForDB+" FROM users_stats WHERE id = ?", [userID])["playcount_"+modeForDB]
+	return glob.db.fetch("SELECT playcount_"+modeForDB+" FROM users_stats WHERE id = %s", [userID])["playcount_"+modeForDB]


 def getUsername(userID):
@@ -187,7 +187,7 @@ def getUsername(userID):
 	return -- username
 	"""

-	return glob.db.fetch("SELECT username FROM users WHERE id = ?", [userID])["username"]
+	return glob.db.fetch("SELECT username FROM users WHERE id = %s", [userID])["username"]


 def getFriendList(userID):
@@ -199,7 +199,7 @@ def getFriendList(userID):
 	"""

 	# Get friends from db
-	friends = glob.db.fetchAll("SELECT user2 FROM users_relationships WHERE user1 = ?", [userID])
+	friends = glob.db.fetchAll("SELECT user2 FROM users_relationships WHERE user1 = %s", [userID])

 	if friends == None or len(friends) == 0:
 		# We have no friends, return 0 list
@@ -225,11 +225,11 @@ def addFriend(userID, friendID):
 		return

 	# check user isn't already a friend of ours
-	if glob.db.fetch("SELECT id FROM users_relationships WHERE user1 = ? AND user2 = ?", [userID, friendID]) != None:
+	if glob.db.fetch("SELECT id FROM users_relationships WHERE user1 = %s AND user2 = %s", [userID, friendID]) != None:
 		return

 	# Set new value
-	glob.db.execute("INSERT INTO users_relationships (user1, user2) VALUES (?, ?)", [userID, friendID])
+	glob.db.execute("INSERT INTO users_relationships (user1, user2) VALUES (%s, %s)", [userID, friendID])


 def removeFriend(userID, friendID):
@@ -242,7 +242,7 @@ def removeFriend(userID, friendID):

 	# Delete user relationship. We don't need to check if the relationship was there, because who gives a shit,
 	# if they were not friends and they don't want to be anymore, be it. ¯\_(ツ)_/¯
-	glob.db.execute("DELETE FROM users_relationships WHERE user1 = ? AND user2 = ?", [userID, friendID])
+	glob.db.execute("DELETE FROM users_relationships WHERE user1 = %s AND user2 = %s", [userID, friendID])


 def getCountry(userID):
@@ -255,7 +255,7 @@ def getCountry(userID):
 	return -- country code (two letters)
 	"""

-	return glob.db.fetch("SELECT country FROM users_stats WHERE id = ?", [userID])["country"]
+	return glob.db.fetch("SELECT country FROM users_stats WHERE id = %s", [userID])["country"]

 def getPP(userID, gameMode):
 	"""
@@ -266,7 +266,7 @@ def getPP(userID, gameMode):
 	"""

 	modeForDB = gameModes.getGameModeForDB(gameMode)
-	return glob.db.fetch("SELECT pp_{} FROM users_stats WHERE id = ?".format(modeForDB), [userID])["pp_{}".format(modeForDB)]
+	return glob.db.fetch("SELECT pp_{} FROM users_stats WHERE id = %s".format(modeForDB), [userID])["pp_{}".format(modeForDB)]

 def setAllowed(userID, allowed):
 	"""
@@ -275,7 +275,7 @@ def setAllowed(userID, allowed):
 	userID -- user
 	allowed -- allowed status. 1: normal, 0: banned
 	"""
-	glob.db.execute("UPDATE users SET allowed = ? WHERE id = ?", [allowed, userID])
+	glob.db.execute("UPDATE users SET allowed = %s WHERE id = %s", [allowed, userID])

 def setCountry(userID, country):
 	"""
@@ -284,7 +284,7 @@ def setCountry(userID, country):
 	userID -- userID
 	country -- country letters
 	"""
-	glob.db.execute("UPDATE users_stats SET country = ? WHERE id = ?", [country, userID])
+	glob.db.execute("UPDATE users_stats SET country = %s WHERE id = %s", [country, userID])

 def getShowCountry(userID):
 	"""
@@ -293,7 +293,7 @@ def getShowCountry(userID):
 	userID -- userID
 	return -- True if country is shown, False if it's hidden
 	"""
-	country = glob.db.fetch("SELECT show_country FROM users_stats WHERE id = ?", [userID])
+	country = glob.db.fetch("SELECT show_country FROM users_stats WHERE id = %s", [userID])
 	if country == None:
 		return False
 	return generalFunctions.stringToBool(country)
@@ -303,5 +303,5 @@ def IPLog(userID, ip):
 	Botnet the user
 	(log his ip for multiaccount detection)
 	"""
-	glob.db.execute("""INSERT INTO ip_user (userid, ip, occurencies) VALUES (?, ?, '1')
+	glob.db.execute("""INSERT INTO ip_user (userid, ip, occurencies) VALUES (%s, %s, '1')
 						ON DUPLICATE KEY UPDATE occurencies = occurencies + 1""", [userID, ip])