.BANCHO. Merge branch tornado-gevent into master

README.md

@@ -10,14 +10,14 @@ This is Ripple's bancho server. It handles:
 ## Requirements
 - Python 3.5
 - MySQLdb (`mysqlclient` or `mysql-python`)
-- Bottle
+- Tornado
 - Gevent
 - Bcrypt
 
 ## How to set up pep.py
 First of all, install all the dependencies
 ```
-$ pip install mysqlclient bottle gevent bcrypt
+$ pip install mysqlclient tornado gevent bcrypt
 ```
 then, run pep.py once to create the default config file and edit it
 ```

constants/fokabotCommands.py

@@ -586,7 +586,6 @@ def tillerinoLast(fro, chan, message):
 				stars = oppaiData["stars"]
 
 		msg += " | {0:.2f} stars".format(stars)
-
 		return msg
 	except Exception as a:
 		log.error(a)

events/loginEvent.py

@@ -2,22 +2,27 @@ from helpers import userHelper
 from constants import serverPackets
 from constants import exceptions
 from objects import glob
+from helpers import consoleHelper
+from constants import bcolors
 from helpers import locationHelper
 from helpers import countryHelper
+import time
+from helpers import generalFunctions
 import sys
 import traceback
+from helpers import requestHelper
+from helpers import discordBotHelper
 from helpers import logHelper as log
 from helpers import chatHelper as chat
 from constants import privileges
-from helpers import requestHelper
 
-def handle(bottleRequest):
+def handle(tornadoRequest):
 	# Data to return
 	responseTokenString = "ayy"
 	responseData = bytes()
 
 	# Get IP from tornado request
-	requestIP = requestHelper.getRequestIP(bottleRequest)
+	requestIP = tornadoRequest.getRequestIP()
 
 	# Avoid exceptions
 	clientData = ["unknown", "unknown", "unknown", "unknown", "unknown"]
@@ -25,8 +30,7 @@ def handle(bottleRequest):
 
 	# Split POST body so we can get username/password/hardware data
 	# 2:-3 thing is because requestData has some escape stuff that we don't need
-	postBody = bottleRequest.body.read()
+	loginData = str(tornadoRequest.request.body)[2:-3].split("\\n")
-	loginData = str(postBody)[2:-3].split("\\n")
 	try:
 		# If true, print error to console
 		err = False

handlers/apiFokabotMessageHandler.py

@@ -1,25 +1,26 @@
+from helpers import requestHelper
 from constants import exceptions
 import json
 from objects import glob
 from helpers import chatHelper
-import bottle
+from helpers import logHelper as log
 
-@bottle.route("/api/v1/fokabotMessage")
+class handler(requestHelper.asyncRequestHandler):
-def GETApiFokabotMessage():
+	def asyncGet(self):
 		statusCode = 400
 		data = {"message": "unknown error"}
 		try:
 			# Check arguments
-		if "k" not in bottle.request.query or "to" not in bottle.request.query or "msg" not in bottle.request.query:
+			if requestHelper.checkArguments(self.request.arguments, ["k", "to", "msg"]) == False:
 				raise exceptions.invalidArgumentsException()
 
 			# Check ci key
-		key = bottle.request.query["k"]
+			key = self.get_argument("k")
 			if key is None or key != glob.conf.config["server"]["cikey"]:
 				raise exceptions.invalidArgumentsException()
 
-		# Send chat message
+			log.info("API REQUEST FOR FOKABOT MESSAGE AAAAAAA")
-		chatHelper.sendMessage("FokaBot", bottle.request.query["to"], bottle.request.query["msg"])
+			chatHelper.sendMessage("FokaBot", self.get_argument("to"), self.get_argument("msg"))
 
 			# Status code and message
 			statusCode = 200
@@ -32,6 +33,7 @@ def GETApiFokabotMessage():
 			data["status"] = statusCode
 
 			# Send response
-		bottle.response.status = statusCode
+			#self.clear()
-		bottle.response.add_header("Content-Type", "application/json")
+			self.write(json.dumps(data))
-		yield json.dumps(data)
+			self.set_status(statusCode)
+			#self.finish(json.dumps(data))

handlers/apiIsOnlineHandler.py

@@ -1,28 +1,36 @@
+from helpers import requestHelper
 from constants import exceptions
 import json
 from objects import glob
-import bottle
 
-@bottle.route("/api/v1/isOnline")
+class handler(requestHelper.asyncRequestHandler):
-def GETApiIsOnline():
+	def asyncGet(self):
 		statusCode = 400
 		data = {"message": "unknown error"}
 		try:
 			# Check arguments
-		if "u" not in bottle.request.query and "id" not in bottle.request.query:
+			if "u" not in self.request.arguments and "id" not in self.request.arguments:
 				raise exceptions.invalidArgumentsException()
 
 			# Get online staus
-		if "u" in bottle.request.query:
+			username = None
-			username = bottle.request.query["u"]
+			userID = None
-			data["result"] = True if glob.tokens.getTokenFromUsername(username) != None else False
+			if "u" in self.request.arguments:
+				username = self.get_argument("u")
 			else:
 				try:
-				userID = int(bottle.request.query["id"])
+					userID = int(self.get_argument("id"))
-				data["result"] = True if glob.tokens.getTokenFromUserID(userID) != None else False
 				except:
 					raise exceptions.invalidArgumentsException()
 
+			if username == None and userID == None:
+				data["result"] = False
+			else:
+				if username != None:
+					data["result"] = True if glob.tokens.getTokenFromUsername(username) != None else False
+				else:
+					data["result"] = True if glob.tokens.getTokenFromUserID(userID) != None else False
+
 			# Status code and message
 			statusCode = 200
 			data["message"] = "ok"
@@ -34,6 +42,7 @@ def GETApiIsOnline():
 			data["status"] = statusCode
 
 			# Send response
-		bottle.response.status = statusCode
+			#self.clear()
-		bottle.response.add_header("Content-Type", "application/json")
+			self.write(json.dumps(data))
-		yield json.dumps(data)
+			self.set_status(statusCode)
+			#self.finish(json.dumps(data))

handlers/apiOnlineUsersHandler.py

@@ -1,9 +1,9 @@
+from helpers import requestHelper
 import json
 from objects import glob
-import bottle
 
-@bottle.route("/api/v1/onlineUsers")
+class handler(requestHelper.asyncRequestHandler):
-def GETApiOnlineUsers():
+	def asyncGet(self):
 		statusCode = 400
 		data = {"message": "unknown error"}
 		try:
@@ -18,6 +18,7 @@ def GETApiOnlineUsers():
 			data["status"] = statusCode
 
 			# Send response
-		bottle.response.status = statusCode
+			#self.clear()
-		bottle.response.add_header("Content-Type", "application/json")
+			self.write(json.dumps(data))
-		yield json.dumps(data)
+			self.set_status(statusCode)
+			#self.finish(json.dumps(data))

handlers/apiServerStatusHandler.py

@@ -1,9 +1,9 @@
+from helpers import requestHelper
 import json
 from objects import glob
-import bottle
 
-@bottle.route("/api/v1/serverStatus")
+class handler(requestHelper.asyncRequestHandler):
-def GETApiServerStatus():
+	def asyncGet(self):
 		statusCode = 400
 		data = {"message": "unknown error"}
 		try:
@@ -18,6 +18,7 @@ def GETApiServerStatus():
 			data["status"] = statusCode
 
 			# Send response
-		bottle.response.status = statusCode
+			#self.clear()
-		bottle.response.add_header("Content-Type", "application/json")
+			self.write(json.dumps(data))
-		yield json.dumps(data)
+			self.set_status(statusCode)
+			#self.finish(json.dumps(data))

handlers/apiVerifiedStatusHandler.py

@@ -1,25 +1,23 @@
+from helpers import requestHelper
+from helpers import logHelper as log
 import json
 from objects import glob
 from constants import exceptions
-import bottle
 
-@bottle.route("/api/v1/verifiedStatus")
+class handler(requestHelper.asyncRequestHandler):
-def GETApiVerifiedStatus():
+	def asyncGet(self):
 		statusCode = 400
 		data = {"message": "unknown error"}
 		try:
 			# Check arguments
-		if "u" not in bottle.request.query:
+			if requestHelper.checkArguments(self.request.arguments, ["u"]) == False:
 				raise exceptions.invalidArgumentsException()
 
 			# Get userID and its verified cache thing
 			# -1: Not in cache
 			# 0: Not verified (multiacc)
 			# 1: Verified
-		userID = bottle.request.query["u"]
+			userID = self.get_argument("u")
-		callback = None
-		if "callback" in bottle.request.query:
-			callback = bottle.request.query["callback"]
 			data["result"] = -1 if userID not in glob.verifiedCache else glob.verifiedCache[userID]
 
 			# Status code and message
@@ -33,16 +31,16 @@ def GETApiVerifiedStatus():
 			data["status"] = statusCode
 
 			# Send response
-		bottle.response.add_header("Access-Control-Allow-Origin", "*")
+			self.add_header("Access-Control-Allow-Origin", "*")
-		bottle.response.add_header("Content-Type", "application/json")
+			self.add_header("Content-Type", "application/json")
 
 			# jquery meme
 			output = ""
-		if callback != None:
+			if "callback" in self.request.arguments:
-			output += callback+"("
+				output += self.get_argument("callback")+"("
 			output += json.dumps(data)
-		if callback != None:
+			if "callback" in self.request.arguments:
 				output += ")"
 
-		bottle.response.status = statusCode
+			self.write(output)
-		yield output
+			self.set_status(statusCode)

handlers/ciTriggerHandler.py

@@ -1,22 +1,22 @@
+from helpers import requestHelper
 from constants import exceptions
 import json
 from objects import glob
 from helpers import systemHelper
 from helpers import logHelper as log
-import bottle
 
-@bottle.route("/api/v1/ciTrigger")
+class handler(requestHelper.asyncRequestHandler):
-def GETCiTrigger():
+	def asyncGet(self):
 		statusCode = 400
 		data = {"message": "unknown error"}
 		try:
 			# Check arguments
-		if "k" not in bottle.request.query:
+			if requestHelper.checkArguments(self.request.arguments, ["k"]) == False:
 				raise exceptions.invalidArgumentsException()
 
 			# Check ci key
-		key = bottle.request.query["k"]
+			key = self.get_argument("k")
-		if key != glob.conf.config["server"]["cikey"]:
+			if key is None or key != glob.conf.config["server"]["cikey"]:
 				raise exceptions.invalidArgumentsException()
 
 			log.info("Ci event triggered!!")
@@ -26,13 +26,14 @@ def GETCiTrigger():
 			statusCode = 200
 			data["message"] = "ok"
 		except exceptions.invalidArgumentsException:
-		statusCode = 403
+			statusCode = 400
 			data["message"] = "invalid ci key"
 		finally:
 			# Add status code to data
 			data["status"] = statusCode
 
 			# Send response
-		bottle.response.status = statusCode
+			#self.clear()
-		bottle.response.add_header("Content-Type", "application/json")
+			self.write(json.dumps(data))
-		yield json.dumps(data)
+			self.set_status(statusCode)
+			#self.finish(json.dumps(data))

handlers/mainHandler.py

@@ -1,11 +1,10 @@
-import bottle
 import datetime
 import gzip
+from helpers import requestHelper
 from objects import glob
-from helpers import packetHelper
-from helpers import logHelper as log
 from constants import exceptions
 from constants import packetIDs
+from helpers import packetHelper
 from constants import serverPackets
 from events import sendPublicMessageEvent
 from events import sendPrivateMessageEvent
@@ -47,17 +46,27 @@ from events import userStatsRequestEvent
 from events import requestStatusUpdateEvent
 from events import userPanelRequestEvent
 
-@bottle.route("/", method="POST")
+# Exception tracking
-def POSTMain():
+import tornado.web
+import tornado.gen
+import sys
+import traceback
+from raven.contrib.tornado import SentryMixin
+from helpers import logHelper as log
+
+class handler(SentryMixin, requestHelper.asyncRequestHandler):
+	@tornado.web.asynchronous
+	@tornado.gen.engine
+	def asyncPost(self):
+		try:
 			# Track time if needed
 			if glob.outputRequestTime == True:
 				# Start time
 				st = datetime.datetime.now()
 
 			# Client's token string and request data
-	#requestTokenString = bottle.request.headers.get("osu-token")
+			requestTokenString = self.request.headers.get("osu-token")
-	requestTokenString = bottle.request.get_header("osu-token")
+			requestData = self.request.body
-	requestData = bottle.request.body.read()
 
 			# Server's token string and request data
 			responseTokenString = "ayy"
@@ -65,7 +74,7 @@ def POSTMain():
 
 			if requestTokenString == None:
 				# No token, first request. Handle login.
-		responseTokenString, responseData = loginEvent.handle(bottle.request)
+				responseTokenString, responseData = loginEvent.handle(self)
 			else:
 				userToken = None	# default value
 				try:
@@ -204,24 +213,32 @@ def POSTMain():
 			# We don't use token object because we might not have a token (failed login)
 			if glob.gzip == True:
 				# First, write the gzipped response
-		responseData = gzip.compress(responseData, int(glob.conf.config["server"]["gziplevel"]))
+				self.write(gzip.compress(responseData, int(glob.conf.config["server"]["gziplevel"])))
 
 				# Then, add gzip headers
-		bottle.response.add_header("Vary", "Accept-Encoding")
+				self.add_header("Vary", "Accept-Encoding")
-		bottle.response.add_header("Content-Encoding", "gzip")
+				self.add_header("Content-Encoding", "gzip")
 			else:
 				# First, write the response
-		responseData = responseData
+				self.write(responseData)
 
 			# Add all the headers AFTER the response has been written
-	bottle.response.status = 200
+			self.set_status(200)
-	bottle.response.add_header("cho-token", responseTokenString)
+			self.add_header("cho-token", responseTokenString)
-	bottle.response.add_header("cho-protocol", "19")
+			self.add_header("cho-protocol", "19")
-	bottle.response.add_header("Content-Type", "text/html; charset=UTF-8")
+			#self.add_header("Keep-Alive", "timeout=5, max=100")
-	yield responseData
+			#self.add_header("Connection", "keep-alive")
+			self.add_header("Content-Type", "text/html; charset=UTF-8")
+		except:
+			log.error("Unknown error!\n```\n{}\n{}```".format(sys.exc_info(), traceback.format_exc()))
+			if glob.sentry:
+				yield tornado.gen.Task(self.captureException, exc_info=True)
+		#finally:
+		#	self.finish()
 
-@bottle.route("/", method="GET")
+	@tornado.web.asynchronous
-def GETMain():
+	@tornado.gen.engine
+	def asyncGet(self):
 		html = 	"<html><head><title>MA MAURO ESISTE?</title><style type='text/css'>body{width:30%}</style></head><body><pre>"
 		html += "           _                 __<br>"
 		html += "          (_)              /  /<br>"
@@ -242,4 +259,6 @@ def GETMain():
 		html += "              \\ . ..  .. . /<br>"
 		html += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^<br>"
 		html += "</marquee><br><strike>reverse engineering a protocol impossible to reverse engineer since always</strike><br>we are actually reverse engineering bancho successfully. for the third time.<br><br><i>&copy; Ripple team, 2016</i></pre></body></html>"
-	yield html
+		self.write(html)
+		#yield tornado.gen.Task(self.captureMessage, "test")
+		#self.finish()

helpers/requestHelper.py

@@ -1,7 +1,89 @@
+import tornado
+import tornado.web
+import tornado.gen
+from tornado.ioloop import IOLoop
 from objects import glob
+from raven.contrib.tornado import SentryMixin
+from raven.contrib.tornado import AsyncSentryClient
+import gevent
 
-def getRequestIP(bottleRequest):
+class asyncRequestHandler(tornado.web.RequestHandler):
-	realIP = bottleRequest.headers.get("X-Forwarded-For") if glob.cloudflare == True else bottleRequest.headers.get("X-Real-IP")
+	"""
+	Tornado asynchronous request handler
+	create a class that extends this one (requestHelper.asyncRequestHandler)
+	use asyncGet() and asyncPost() instad of get() and post().
+	Done. I'm not kidding.
+	"""
+	@tornado.web.asynchronous
+	@tornado.gen.engine
+	def get(self, *args, **kwargs):
+		try:
+			yield tornado.gen.Task(runBackground, (self.asyncGet, tuple(args), dict(kwargs)))
+		except Exception as e:
+			yield tornado.gen.Task(self.captureException, exc_info=True)
+		finally:
+			if not self._finished:
+				self.finish()
+
+	@tornado.web.asynchronous
+	@tornado.gen.engine
+	def post(self, *args, **kwargs):
+		try:
+			yield tornado.gen.Task(runBackground, (self.asyncPost, tuple(args), dict(kwargs)))
+		except Exception as e:
+			yield tornado.gen.Task(self.captureException, exc_info=True)
+		finally:
+			if not self._finished:
+				self.finish()
+
+	def asyncGet(self, *args, **kwargs):
+		self.send_error(405)
+		self.finish()
+
+	def asyncPost(self, *args, **kwargs):
+		self.send_error(405)
+		self.finish()
+
+	def getRequestIP(self):
+		realIP = self.request.headers.get("X-Forwarded-For") if glob.cloudflare == True else self.request.headers.get("X-Real-IP")
 		if realIP != None:
 			return realIP
-	return bottleRequest.environ.get("REMOTE_ADDR")
+		return self.request.remote_ip
+
+
+def runBackground(data, callback):
+	"""
+	Run a function in the background.
+	Used to handle multiple requests at the same time
+	"""
+	func, args, kwargs = data
+	def _callback(result):
+		IOLoop.instance().add_callback(lambda: callback(result))
+	#glob.pool.apply_async(func, args, kwargs, _callback)
+	g = gevent.Greenlet(func, *args, **kwargs)
+	g.link(_callback)
+	g.start()
+
+def checkArguments(arguments, requiredArguments):
+	"""
+	Check that every requiredArguments elements are in arguments
+
+	arguments -- full argument list, from tornado
+	requiredArguments -- required arguments list es: ["u", "ha"]
+	handler -- handler string name to print in exception. Optional
+	return -- True if all arguments are passed, none if not
+	"""
+	for i in requiredArguments:
+		if i not in arguments:
+			return False
+	return True
+
+def printArguments(t):
+	"""
+	Print passed arguments, for debug purposes
+
+	t -- tornado object (self)
+	"""
+	print("ARGS::")
+	for i in t.request.arguments:
+		print ("{}={}".format(i, t.get_argument(i)))

pep.py

@@ -3,11 +3,17 @@ import sys
 import os
 import threading
 
-# Bottle
+# Tornado
-import bottle
+import tornado.ioloop
+import tornado.web
+import tornado.httpserver
+import tornado.gen
 from gevent import monkey as brit_monkey
 brit_monkey.patch_all()
 
+# Raven
+from raven.contrib.tornado import AsyncSentryClient
+
 # pep.py files
 from constants import bcolors
 from helpers import configHelper
@@ -20,13 +26,27 @@ from helpers import databaseHelperNew
 from helpers import generalFunctions
 from helpers import logHelper as log
 
-# Raven
+from handlers import mainHandler
-from raven import Client
+from handlers import apiIsOnlineHandler
-from raven.contrib.bottle import Sentry
+from handlers import apiOnlineUsersHandler
+from handlers import apiServerStatusHandler
+from handlers import ciTriggerHandler
+from handlers import apiVerifiedStatusHandler
+from handlers import apiFokabotMessageHandler
 
-# IRC
 from irc import ircserver
 
+def make_app():
+	return tornado.web.Application([
+		(r"/", mainHandler.handler),
+		(r"/api/v1/isOnline", apiIsOnlineHandler.handler),
+		(r"/api/v1/onlineUsers", apiOnlineUsersHandler.handler),
+		(r"/api/v1/serverStatus", apiServerStatusHandler.handler),
+		(r"/api/v1/ciTrigger", ciTriggerHandler.handler),
+		(r"/api/v1/verifiedStatus", apiVerifiedStatusHandler.handler),
+		(r"/api/v1/fokabotMessage", apiFokabotMessageHandler.handler)
+	])
+
 if __name__ == "__main__":
 	# Server start
 	consoleHelper.printServerStartHeader(True)
@@ -140,22 +160,13 @@ if __name__ == "__main__":
 		consoleHelper.printColored("[!] Warning! Server running in debug mode!", bcolors.YELLOW)
 
 	# Make app
-	app = bottle.app()
+	application = make_app()
-	app.catchall = False
-	from handlers import mainHandler
-	from handlers import apiIsOnlineHandler
-	from handlers import apiOnlineUsersHandler
-	from handlers import apiServerStatusHandler
-	from handlers import ciTriggerHandler
-	from handlers import apiVerifiedStatusHandler
-	from handlers import apiFokabotMessageHandler
 
 	# Set up sentry
 	try:
 		glob.sentry = generalFunctions.stringToBool(glob.conf.config["sentry"]["enable"])
 		if glob.sentry == True:
-			client = Client(glob.conf.config["sentry"]["banchodns"], release=glob.VERSION)
+			application.sentry_client = AsyncSentryClient(glob.conf.config["sentry"]["banchodns"], release=glob.VERSION)
-			app = Sentry(app, client)
 		else:
 			consoleHelper.printColored("[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
 	except:
@@ -185,8 +196,9 @@ if __name__ == "__main__":
 		consoleHelper.printColored("[!] Invalid server port! Please check your config.ini and run the server again", bcolors.RED)
 
 	# Server start message and console output
-	log.logMessage("Server started!", discord=True, stdout=False)
+	log.logMessage("Server started!", discord=True, of="info.txt", stdout=False)
-	consoleHelper.printColored("> Bottle listening for HTTP(s) clients on 127.0.0.1:{}...".format(serverPort), bcolors.GREEN)
+	consoleHelper.printColored("> Tornado listening for HTTP(s) clients on 127.0.0.1:{}...".format(serverPort), bcolors.GREEN)
 
-	# Start bottle
+	# Start tornado
-	bottle.run(app=app, host="0.0.0.0", port=serverPort, server="gevent")
+	application.listen(serverPort)
+	tornado.ioloop.IOLoop.instance().start()