.BANCHO. Add file buffers

helpers/logHelper.py

@@ -63,12 +63,7 @@ def logMessage(message, alertType = "INFO", messageColor = bcolors.ENDC, discord
 
 	# Log to file if needed
 	if of is not None:
-		try:
+		glob.fileBuffers.write(".data/"+of, finalMessage+ENDL)
-			glob.fLocks.lockFile(of)
-			with open(".data/{}".format(of), "a") as f:
-				f.write(finalMessage+ENDL)
-		finally:
-			glob.fLocks.unlockFile(of)
 
 def warning(message, discord = None, alertDev = False):
 	"""

helpers/systemHelper.py

@@ -1,14 +1,25 @@
 from objects import glob
 from constants import serverPackets
+from helpers import consoleHelper
 import psutil
 import os
 import sys
 import threading
 import signal
 from helpers import logHelper as log
+from constants import bcolors
 import time
 import math
 
+def dispose():
+	"""
+	Perform some clean up. Called on shutdown.
+	:return:
+	"""
+	print("> Disposing server... ")
+	glob.fileBuffers.flushAll()
+	consoleHelper.printColored("Goodbye!", bcolors.GREEN)
+
 def runningUnderUnix():
 	"""
 	Get if the server is running under UNIX or NT
@@ -49,11 +60,13 @@ def scheduleShutdown(sendRestartTime, restart, message = "", delay=20):
 def restartServer():
 	"""Restart pep.py script"""
 	log.info("Restarting pep.py...")
+	dispose()
 	os.execv(sys.executable, [sys.executable] + sys.argv)
 
 def shutdownServer():
 	"""Shutdown pep.py"""
 	log.info("Shutting down pep.py...")
+	dispose()
 	sig = signal.SIGKILL if runningUnderUnix() else signal.CTRL_C_EVENT
 	os.kill(os.getpid(), sig)
 

objects/fileBuffer.py

@@ -0,0 +1,79 @@
+from objects import glob
+
+class buffer():
+	"""
+	A file buffer object.
+	This buffer caches data in memory and when it's full, it writes the content to a file.
+	"""
+	def __init__(self, fileName, writeType="a", maxLength=512):
+		"""
+		A file buffer object
+
+		:param fileName: Path and name of file on disk .
+		:param writeType: File write type. Optional. Default: "a" .
+		:param maxLength: Max length before writing buffer to disk. Optional. Default: 512.
+		"""
+		self.content = ""
+		self.length = 0
+		self.fileName = fileName
+		self.writeType = writeType
+		self.maxLength = maxLength
+
+	def write(self, newData):
+		"""
+		Add data to buffer.
+		If the total length of the data in buffer is greater than or equal to self.maxLength,
+		the content is written on the disk and the buffer resets
+
+		:param newData: Data to append to buffer
+		:return:
+		"""
+		self.content += newData
+		self.length += len(newData)
+		if self.length >= self.maxLength:
+			self.flush()
+
+	def flush(self):
+		"""
+		Write buffer content to disk and reset its content
+
+		:return:
+		"""
+		try:
+			glob.fLocks.lockFile(self.fileName)
+			with open(self.fileName, self.writeType) as f:
+				f.write(self.content)
+		finally:
+			glob.fLocks.unlockFile(self.fileName)
+
+		self.content = ""
+		self.length = 0
+
+class buffersList():
+	"""
+	A list of buffers
+	"""
+	def __init__(self):
+		self.buffers = {}
+
+	def write(self, fileName, content):
+		"""
+		Write some data to an existing buffer in this list (or create a new one if it doesn't exist).
+		If the buffer is full, the data is written to the file and the buffer resets.
+
+		:param fileName: Path of file/buffer
+		:param content: New content
+		:return:
+		"""
+		if fileName not in self.buffers:
+			self.buffers[fileName] = buffer(fileName)
+		self.buffers[fileName].write(content)
+
+	def flushAll(self):
+		"""
+		Write all buffers to file and flush them
+
+		:return:
+		"""
+		for _, value in self.buffers.items():
+			value.flush()

objects/glob.py

@@ -4,6 +4,7 @@ from objects import tokenList
 from objects import channelList
 from objects import matchList
 from objects import fileLocks
+from objects import fileBuffer
 import time
 
 try:
@@ -22,6 +23,7 @@ channels = channelList.channelList()
 matches = matchList.matchList()
 restarting = False
 fLocks = fileLocks.fileLocks()
+fileBuffers = fileBuffer.buffersList()
 verifiedCache = {}
 cloudflare = False
 chatFilters = None

pep.py

@@ -26,6 +26,7 @@ from helpers import databaseHelperNew
 from helpers import generalFunctions
 from helpers import logHelper as log
 from helpers import userHelper
+from helpers import systemHelper as system
 
 from handlers import mainHandler
 from handlers import apiIsOnlineHandler
@@ -49,161 +50,167 @@ def make_app():
 	])
 
 if __name__ == "__main__":
-	# Server start
-	consoleHelper.printServerStartHeader(True)
-
-	# Read config.ini
-	consoleHelper.printNoNl("> Loading config file... ")
-	glob.conf = configHelper.config("config.ini")
-
-	if glob.conf.default:
-		# We have generated a default config.ini, quit server
-		consoleHelper.printWarning()
-		consoleHelper.printColored("[!] config.ini not found. A default one has been generated.", bcolors.YELLOW)
-		consoleHelper.printColored("[!] Please edit your config.ini and run the server again.", bcolors.YELLOW)
-		sys.exit()
-
-	# If we haven't generated a default config.ini, check if it's valid
-	if not glob.conf.checkConfig():
-		consoleHelper.printError()
-		consoleHelper.printColored("[!] Invalid config.ini. Please configure it properly", bcolors.RED)
-		consoleHelper.printColored("[!] Delete your config.ini to generate a default one", bcolors.RED)
-		sys.exit()
-	else:
-		consoleHelper.printDone()
-
-	# Create data folder if needed
-	consoleHelper.printNoNl("> Checking folders... ")
-	paths = [".data"]
-	for i in paths:
-		if not os.path.exists(i):
-			os.makedirs(i, 0o770)
-	consoleHelper.printDone()
-
-	# Connect to db
 	try:
-		consoleHelper.printNoNl("> Connecting to MySQL db")
+		# Server start
-		glob.db = databaseHelperNew.db(glob.conf.config["db"]["host"], glob.conf.config["db"]["username"], glob.conf.config["db"]["password"], glob.conf.config["db"]["database"], int(glob.conf.config["db"]["workers"]))
+		consoleHelper.printServerStartHeader(True)
-		consoleHelper.printNoNl(" ")
-		consoleHelper.printDone()
-	except:
-		# Exception while connecting to db
-		consoleHelper.printError()
-		consoleHelper.printColored("[!] Error while connection to database. Please check your config.ini and run the server again", bcolors.RED)
-		raise
 
-	# Load bancho_settings
+		# Read config.ini
-	try:
+		consoleHelper.printNoNl("> Loading config file... ")
-		consoleHelper.printNoNl("> Loading bancho settings from DB... ")
+		glob.conf = configHelper.config("config.ini")
-		glob.banchoConf = banchoConfig.banchoConfig()
-		consoleHelper.printDone()
-	except:
-		consoleHelper.printError()
-		consoleHelper.printColored("[!] Error while loading bancho_settings. Please make sure the table in DB has all the required rows", bcolors.RED)
-		raise
 
-	# Delete old bancho sessions
+		if glob.conf.default:
-	consoleHelper.printNoNl("> Deleting cached bancho sessions from DB... ")
+			# We have generated a default config.ini, quit server
-	glob.tokens.deleteBanchoSessions()
+			consoleHelper.printWarning()
-	consoleHelper.printDone()
+			consoleHelper.printColored("[!] config.ini not found. A default one has been generated.", bcolors.YELLOW)
+			consoleHelper.printColored("[!] Please edit your config.ini and run the server again.", bcolors.YELLOW)
+			sys.exit()
 
-	try:
+		# If we haven't generated a default config.ini, check if it's valid
-		consoleHelper.printNoNl("> Loading chat filters... ")
+		if not glob.conf.checkConfig():
-		glob.chatFilters = chatFilters.chatFilters()
+			consoleHelper.printError()
-		consoleHelper.printDone()
+			consoleHelper.printColored("[!] Invalid config.ini. Please configure it properly", bcolors.RED)
-	except:
+			consoleHelper.printColored("[!] Delete your config.ini to generate a default one", bcolors.RED)
-		consoleHelper.printError()
+			sys.exit()
-		consoleHelper.printColored("[!] Error while loading chat filters. Make sure there is a filters.txt file present", bcolors.RED)
-		raise
-
-	# Initialize chat channels
-	print("> Initializing chat channels... ")
-	glob.channels.loadChannels()
-	consoleHelper.printDone()
-
-	# Start fokabot
-	consoleHelper.printNoNl("> Connecting FokaBot... ")
-	fokabot.connect()
-	consoleHelper.printDone()
-
-	# Initialize user timeout check loop
-	consoleHelper.printNoNl("> Initializing user timeout check loop... ")
-	glob.tokens.usersTimeoutCheckLoop()
-	consoleHelper.printDone()
-
-	# Initialize spam protection reset loop
-	consoleHelper.printNoNl("> Initializing spam protection reset loop... ")
-	glob.tokens.spamProtectionResetLoop()
-	consoleHelper.printDone()
-
-	# Cache user ids
-	consoleHelper.printNoNl("> Caching user IDs... ")
-	userHelper.cacheUserIDs()
-	consoleHelper.printDone()
-
-	# Localize warning
-	glob.localize = generalFunctions.stringToBool(glob.conf.config["localize"]["enable"])
-	if not glob.localize:
-		consoleHelper.printColored("[!] Warning! Users localization is disabled!", bcolors.YELLOW)
-
-	# Discord
-	glob.discord = generalFunctions.stringToBool(glob.conf.config["discord"]["enable"])
-	if not glob.discord:
-		consoleHelper.printColored("[!] Warning! Discord logging is disabled!", bcolors.YELLOW)
-
-	# Gzip
-	glob.gzip = generalFunctions.stringToBool(glob.conf.config["server"]["gzip"])
-	glob.gziplevel = int(glob.conf.config["server"]["gziplevel"])
-	if not glob.gzip:
-		consoleHelper.printColored("[!] Warning! Gzip compression is disabled!", bcolors.YELLOW)
-
-	# Debug mode
-	glob.debug = generalFunctions.stringToBool(glob.conf.config["debug"]["enable"])
-	glob.outputPackets = generalFunctions.stringToBool(glob.conf.config["debug"]["packets"])
-	glob.outputRequestTime = generalFunctions.stringToBool(glob.conf.config["debug"]["time"])
-	if glob.debug:
-		consoleHelper.printColored("[!] Warning! Server running in debug mode!", bcolors.YELLOW)
-
-	# Make app
-	application = make_app()
-
-	# Set up sentry
-	try:
-		glob.sentry = generalFunctions.stringToBool(glob.conf.config["sentry"]["enable"])
-		if glob.sentry:
-			application.sentry_client = AsyncSentryClient(glob.conf.config["sentry"]["banchodns"], release=glob.VERSION)
 		else:
-			consoleHelper.printColored("[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
+			consoleHelper.printDone()
-	except:
-		consoleHelper.printColored("[!] Error while starting sentry client! Please check your config.ini and run the server again", bcolors.RED)
 
-	# Cloudflare memes
+		# Create data folder if needed
-	glob.cloudflare = generalFunctions.stringToBool(glob.conf.config["server"]["cloudflare"])
+		consoleHelper.printNoNl("> Checking folders... ")
+		paths = [".data"]
+		for i in paths:
+			if not os.path.exists(i):
+				os.makedirs(i, 0o770)
+		consoleHelper.printDone()
 
-	# IRC start message and console output
+		# Flush file buffers at exit
-	glob.irc = generalFunctions.stringToBool(glob.conf.config["irc"]["enable"])
+		#atexit.register(lambda: glob.fileBuffers.flushAll())
-	if glob.irc:
+
-		# IRC port
+		# Connect to db
 		try:
-			ircPort = int(glob.conf.config["irc"]["port"])
+			consoleHelper.printNoNl("> Connecting to MySQL db")
+			glob.db = databaseHelperNew.db(glob.conf.config["db"]["host"], glob.conf.config["db"]["username"], glob.conf.config["db"]["password"], glob.conf.config["db"]["database"], int(glob.conf.config["db"]["workers"]))
+			consoleHelper.printNoNl(" ")
+			consoleHelper.printDone()
 		except:
-			consoleHelper.printColored("[!] Invalid IRC port! Please check your config.ini and run the server again", bcolors.RED)
+			# Exception while connecting to db
-		log.logMessage("IRC server started!", discord=True, of="info.txt", stdout=False)
+			consoleHelper.printError()
-		consoleHelper.printColored("> IRC server listening on 127.0.0.1:{}...".format(ircPort), bcolors.GREEN)
+			consoleHelper.printColored("[!] Error while connection to database. Please check your config.ini and run the server again", bcolors.RED)
-		threading.Thread(target=lambda: ircserver.main(port=ircPort)).start()
+			raise
-	else:
-		consoleHelper.printColored("[!] Warning! IRC server is disabled!", bcolors.YELLOW)
 
-	# Server port
+		# Load bancho_settings
-	try:
+		try:
-		serverPort = int(glob.conf.config["server"]["port"])
+			consoleHelper.printNoNl("> Loading bancho settings from DB... ")
-	except:
+			glob.banchoConf = banchoConfig.banchoConfig()
-		consoleHelper.printColored("[!] Invalid server port! Please check your config.ini and run the server again", bcolors.RED)
+			consoleHelper.printDone()
+		except:
+			consoleHelper.printError()
+			consoleHelper.printColored("[!] Error while loading bancho_settings. Please make sure the table in DB has all the required rows", bcolors.RED)
+			raise
 
-	# Server start message and console output
+		# Delete old bancho sessions
-	log.logMessage("Server started!", discord=True, of="info.txt", stdout=False)
+		consoleHelper.printNoNl("> Deleting cached bancho sessions from DB... ")
-	consoleHelper.printColored("> Tornado listening for HTTP(s) clients on 127.0.0.1:{}...".format(serverPort), bcolors.GREEN)
+		glob.tokens.deleteBanchoSessions()
+		consoleHelper.printDone()
 
-	# Start tornado
+		try:
-	application.listen(serverPort)
+			consoleHelper.printNoNl("> Loading chat filters... ")
-	tornado.ioloop.IOLoop.instance().start()
+			glob.chatFilters = chatFilters.chatFilters()
+			consoleHelper.printDone()
+		except:
+			consoleHelper.printError()
+			consoleHelper.printColored("[!] Error while loading chat filters. Make sure there is a filters.txt file present", bcolors.RED)
+			raise
+
+		# Initialize chat channels
+		print("> Initializing chat channels... ")
+		glob.channels.loadChannels()
+		consoleHelper.printDone()
+
+		# Start fokabot
+		consoleHelper.printNoNl("> Connecting FokaBot... ")
+		fokabot.connect()
+		consoleHelper.printDone()
+
+		# Initialize user timeout check loop
+		consoleHelper.printNoNl("> Initializing user timeout check loop... ")
+		glob.tokens.usersTimeoutCheckLoop()
+		consoleHelper.printDone()
+
+		# Initialize spam protection reset loop
+		consoleHelper.printNoNl("> Initializing spam protection reset loop... ")
+		glob.tokens.spamProtectionResetLoop()
+		consoleHelper.printDone()
+
+		# Cache user ids
+		consoleHelper.printNoNl("> Caching user IDs... ")
+		userHelper.cacheUserIDs()
+		consoleHelper.printDone()
+
+		# Localize warning
+		glob.localize = generalFunctions.stringToBool(glob.conf.config["localize"]["enable"])
+		if not glob.localize:
+			consoleHelper.printColored("[!] Warning! Users localization is disabled!", bcolors.YELLOW)
+
+		# Discord
+		glob.discord = generalFunctions.stringToBool(glob.conf.config["discord"]["enable"])
+		if not glob.discord:
+			consoleHelper.printColored("[!] Warning! Discord logging is disabled!", bcolors.YELLOW)
+
+		# Gzip
+		glob.gzip = generalFunctions.stringToBool(glob.conf.config["server"]["gzip"])
+		glob.gziplevel = int(glob.conf.config["server"]["gziplevel"])
+		if not glob.gzip:
+			consoleHelper.printColored("[!] Warning! Gzip compression is disabled!", bcolors.YELLOW)
+
+		# Debug mode
+		glob.debug = generalFunctions.stringToBool(glob.conf.config["debug"]["enable"])
+		glob.outputPackets = generalFunctions.stringToBool(glob.conf.config["debug"]["packets"])
+		glob.outputRequestTime = generalFunctions.stringToBool(glob.conf.config["debug"]["time"])
+		if glob.debug:
+			consoleHelper.printColored("[!] Warning! Server running in debug mode!", bcolors.YELLOW)
+
+		# Make app
+		application = make_app()
+
+		# Set up sentry
+		try:
+			glob.sentry = generalFunctions.stringToBool(glob.conf.config["sentry"]["enable"])
+			if glob.sentry:
+				application.sentry_client = AsyncSentryClient(glob.conf.config["sentry"]["banchodns"], release=glob.VERSION)
+			else:
+				consoleHelper.printColored("[!] Warning! Sentry logging is disabled!", bcolors.YELLOW)
+		except:
+			consoleHelper.printColored("[!] Error while starting sentry client! Please check your config.ini and run the server again", bcolors.RED)
+
+		# Cloudflare memes
+		glob.cloudflare = generalFunctions.stringToBool(glob.conf.config["server"]["cloudflare"])
+
+		# IRC start message and console output
+		glob.irc = generalFunctions.stringToBool(glob.conf.config["irc"]["enable"])
+		if glob.irc:
+			# IRC port
+			try:
+				ircPort = int(glob.conf.config["irc"]["port"])
+			except:
+				consoleHelper.printColored("[!] Invalid IRC port! Please check your config.ini and run the server again", bcolors.RED)
+			log.logMessage("IRC server started!", discord=True, of="info.txt", stdout=False)
+			consoleHelper.printColored("> IRC server listening on 127.0.0.1:{}...".format(ircPort), bcolors.GREEN)
+			threading.Thread(target=lambda: ircserver.main(port=ircPort)).start()
+		else:
+			consoleHelper.printColored("[!] Warning! IRC server is disabled!", bcolors.YELLOW)
+
+		# Server port
+		try:
+			serverPort = int(glob.conf.config["server"]["port"])
+		except:
+			consoleHelper.printColored("[!] Invalid server port! Please check your config.ini and run the server again", bcolors.RED)
+
+		# Server start message and console output
+		log.logMessage("Server started!", discord=True, of="info.txt", stdout=False)
+		consoleHelper.printColored("> Tornado listening for HTTP(s) clients on 127.0.0.1:{}...".format(serverPort), bcolors.GREEN)
+
+		# Start tornado
+		application.listen(serverPort)
+		tornado.ioloop.IOLoop.instance().start()
+	finally:
+		system.dispose()