.BANCHO. Add file buffers

helpers/logHelper.py

@@ -63,12 +63,7 @@ def logMessage(message, alertType = "INFO", messageColor = bcolors.ENDC, discord
 
 	# Log to file if needed
 	if of is not None:
-		try:
-			glob.fLocks.lockFile(of)
-			with open(".data/{}".format(of), "a") as f:
-				f.write(finalMessage+ENDL)
-		finally:
-			glob.fLocks.unlockFile(of)
+		glob.fileBuffers.write(".data/"+of, finalMessage+ENDL)
 
 def warning(message, discord = None, alertDev = False):
 	"""

helpers/systemHelper.py

@@ -1,14 +1,25 @@
 from objects import glob
 from constants import serverPackets
+from helpers import consoleHelper
 import psutil
 import os
 import sys
 import threading
 import signal
 from helpers import logHelper as log
+from constants import bcolors
 import time
 import math
 
+def dispose():
+	"""
+	Perform some clean up. Called on shutdown.
+	:return:
+	"""
+	print("> Disposing server... ")
+	glob.fileBuffers.flushAll()
+	consoleHelper.printColored("Goodbye!", bcolors.GREEN)
+
 def runningUnderUnix():
 	"""
 	Get if the server is running under UNIX or NT
@@ -49,11 +60,13 @@ def scheduleShutdown(sendRestartTime, restart, message = "", delay=20):
 def restartServer():
 	"""Restart pep.py script"""
 	log.info("Restarting pep.py...")
+	dispose()
 	os.execv(sys.executable, [sys.executable] + sys.argv)
 
 def shutdownServer():
 	"""Shutdown pep.py"""
 	log.info("Shutting down pep.py...")
+	dispose()
 	sig = signal.SIGKILL if runningUnderUnix() else signal.CTRL_C_EVENT
 	os.kill(os.getpid(), sig)
 

objects/fileBuffer.py

@@ -0,0 +1,79 @@
+from objects import glob
+
+class buffer():
+	"""
+	A file buffer object.
+	This buffer caches data in memory and when it's full, it writes the content to a file.
+	"""
+	def __init__(self, fileName, writeType="a", maxLength=512):
+		"""
+		A file buffer object
+
+		:param fileName: Path and name of file on disk .
+		:param writeType: File write type. Optional. Default: "a" .
+		:param maxLength: Max length before writing buffer to disk. Optional. Default: 512.
+		"""
+		self.content = ""
+		self.length = 0
+		self.fileName = fileName
+		self.writeType = writeType
+		self.maxLength = maxLength
+
+	def write(self, newData):
+		"""
+		Add data to buffer.
+		If the total length of the data in buffer is greater than or equal to self.maxLength,
+		the content is written on the disk and the buffer resets
+
+		:param newData: Data to append to buffer
+		:return:
+		"""
+		self.content += newData
+		self.length += len(newData)
+		if self.length >= self.maxLength:
+			self.flush()
+
+	def flush(self):
+		"""
+		Write buffer content to disk and reset its content
+
+		:return:
+		"""
+		try:
+			glob.fLocks.lockFile(self.fileName)
+			with open(self.fileName, self.writeType) as f:
+				f.write(self.content)
+		finally:
+			glob.fLocks.unlockFile(self.fileName)
+
+		self.content = ""
+		self.length = 0
+
+class buffersList():
+	"""
+	A list of buffers
+	"""
+	def __init__(self):
+		self.buffers = {}
+
+	def write(self, fileName, content):
+		"""
+		Write some data to an existing buffer in this list (or create a new one if it doesn't exist).
+		If the buffer is full, the data is written to the file and the buffer resets.
+
+		:param fileName: Path of file/buffer
+		:param content: New content
+		:return:
+		"""
+		if fileName not in self.buffers:
+			self.buffers[fileName] = buffer(fileName)
+		self.buffers[fileName].write(content)
+
+	def flushAll(self):
+		"""
+		Write all buffers to file and flush them
+
+		:return:
+		"""
+		for _, value in self.buffers.items():
+			value.flush()

objects/glob.py

@@ -4,6 +4,7 @@ from objects import tokenList
 from objects import channelList
 from objects import matchList
 from objects import fileLocks
+from objects import fileBuffer
 import time
 
 try:
@@ -22,6 +23,7 @@ channels = channelList.channelList()
 matches = matchList.matchList()
 restarting = False
 fLocks = fileLocks.fileLocks()
+fileBuffers = fileBuffer.buffersList()
 verifiedCache = {}
 cloudflare = False
 chatFilters = None

pep.py

@@ -26,6 +26,7 @@ from helpers import databaseHelperNew
 from helpers import generalFunctions
 from helpers import logHelper as log
 from helpers import userHelper
+from helpers import systemHelper as system
 
 from handlers import mainHandler
 from handlers import apiIsOnlineHandler
@@ -49,6 +50,7 @@ def make_app():
 	])
 
 if __name__ == "__main__":
+	try:
 		# Server start
 		consoleHelper.printServerStartHeader(True)
 
@@ -80,6 +82,9 @@ if __name__ == "__main__":
 				os.makedirs(i, 0o770)
 		consoleHelper.printDone()
 
+		# Flush file buffers at exit
+		#atexit.register(lambda: glob.fileBuffers.flushAll())
+
 		# Connect to db
 		try:
 			consoleHelper.printNoNl("> Connecting to MySQL db")
@@ -207,3 +212,5 @@ if __name__ == "__main__":
 		# Start tornado
 		application.listen(serverPort)
 		tornado.ioloop.IOLoop.instance().start()
+	finally:
+		system.dispose()