Add redis support, remove userID cache

.gitignore

@@ -6,3 +6,4 @@ filters.txt
 common_funzia
 common_refractor
 common_memato
+redistest.py

common

@@ -1 +1 @@
-Subproject commit 5c4ce6b7c8d03de8c25b379b2ebfb2229982af97
+Subproject commit 199dfad2bb98feff734eb669351e671f52632148

helpers/configHelper.py

@@ -37,6 +37,11 @@ class config:
 			self.config.get("db","database")
 			self.config.get("db","workers")
 
+			self.config.get("redis","host")
+			self.config.get("redis","port")
+			self.config.get("redis","database")
+			self.config.get("redis","password")
+
 			self.config.get("server","port")
 			self.config.get("server","threads")
 			self.config.get("server","gzip")
@@ -87,6 +92,12 @@ class config:
 		self.config.set("db", "database", "ripple")
 		self.config.set("db", "workers", "4")
 
+		self.config.add_section("redis")
+		self.config.set("redis", "host", "localhost")
+		self.config.set("redis", "port", "6379")
+		self.config.set("redis", "database", "0")
+		self.config.set("redis", "password", "")
+
 		self.config.add_section("server")
 		self.config.set("server", "port", "5001")
 		self.config.set("server", "threads", "16")

objects/glob.py

@@ -20,6 +20,7 @@ except:
 DATADOG_PREFIX = "peppy"
 application = None
 db = None
+redis = None
 conf = None
 banchoConf = None
 tokens = tokenList.tokenList()
@@ -31,7 +32,6 @@ schiavo = schiavo.schiavo()
 dog = datadogClient.datadogClient()
 verifiedCache = {}
 chatFilters = None
-userIDCache = {}
 pool = None
 ircServer = None
 busyThreads = 0

pep.py

@@ -9,6 +9,7 @@ import tornado.httpserver
 import tornado.ioloop
 import tornado.web
 from raven.contrib.tornado import AsyncSentryClient
+import redis
 
 from common import generalUtils
 from common.constants import bcolors
@@ -82,7 +83,7 @@ if __name__ == "__main__":
 
 		# Connect to db
 		try:
-			consoleHelper.printNoNl("> Connecting to MySQL database...")
+			consoleHelper.printNoNl("> Connecting to MySQL database... ")
 			glob.db = dbConnector.db(glob.conf.config["db"]["host"], glob.conf.config["db"]["username"], glob.conf.config["db"]["password"], glob.conf.config["db"]["database"], int(glob.conf.config["db"]["workers"]))
 			consoleHelper.printNoNl(" ")
 			consoleHelper.printDone()
@@ -92,6 +93,26 @@ if __name__ == "__main__":
 			consoleHelper.printColored("[!] Error while connection to database. Please check your config.ini and run the server again", bcolors.RED)
 			raise
 
+		# Connect to redis
+		try:
+			consoleHelper.printNoNl("> Connecting to redis... ")
+			glob.redis = redis.Redis(glob.conf.config["redis"]["host"], glob.conf.config["redis"]["port"], glob.conf.config["redis"]["database"], glob.conf.config["redis"]["password"])
+			glob.redis.ping()
+			consoleHelper.printNoNl(" ")
+			consoleHelper.printDone()
+		except:
+			# Exception while connecting to db
+			consoleHelper.printError()
+			consoleHelper.printColored("[!] Error while connection to redis. Please check your config.ini and run the server again", bcolors.RED)
+			raise
+
+		# Empty redis cache
+		try:
+			glob.redis.eval("return redis.call('del', unpack(redis.call('keys', ARGV[1])))", 0, "peppy:*")
+		except redis.exceptions.ResponseError:
+			# Script returns error if there are no keys starting with peppy:*
+			pass
+
 		# Load bancho_settings
 		try:
 			consoleHelper.printNoNl("> Loading bancho settings from DB... ")
@@ -151,11 +172,6 @@ if __name__ == "__main__":
 		glob.tokens.spamProtectionResetLoop()
 		consoleHelper.printDone()
 
-		# Cache user ids
-		consoleHelper.printNoNl("> Caching user IDs... ")
-		userUtils.cacheUserIDs()
-		consoleHelper.printDone()
-
 		# Localize warning
 		glob.localize = generalUtils.stringToBool(glob.conf.config["localize"]["enable"])
 		if not glob.localize:
@@ -210,7 +226,7 @@ if __name__ == "__main__":
 						datadogClient.periodicCheck("ram_file_locks", lambda: generalUtils.getTotalSize(glob.fLocks)),
 						datadogClient.periodicCheck("ram_datadog", lambda: generalUtils.getTotalSize(glob.datadogClient)),
 						datadogClient.periodicCheck("ram_verified_cache", lambda: generalUtils.getTotalSize(glob.verifiedCache)),
-						datadogClient.periodicCheck("ram_userid_cache", lambda: generalUtils.getTotalSize(glob.userIDCache)),
+						#datadogClient.periodicCheck("ram_userid_cache", lambda: generalUtils.getTotalSize(glob.userIDCache)),
 						#datadogClient.periodicCheck("ram_pool", lambda: generalUtils.getTotalSize(glob.pool)),
 						datadogClient.periodicCheck("ram_irc", lambda: generalUtils.getTotalSize(glob.ircServer)),
 						datadogClient.periodicCheck("ram_tornado", lambda: generalUtils.getTotalSize(glob.application)),

requirements.txt

@@ -4,4 +4,5 @@ mysqlclient
 psutil
 raven
 bcrypt>=3.1.1
-dill
+dill
+redis