diff --git a/helpers/logHelper.py b/helpers/logHelper.py index bf8d357..77ee10b 100644 --- a/helpers/logHelper.py +++ b/helpers/logHelper.py @@ -63,12 +63,7 @@ def logMessage(message, alertType = "INFO", messageColor = bcolors.ENDC, discord # Log to file if needed if of is not None: - try: - glob.fLocks.lockFile(of) - with open(".data/{}".format(of), "a") as f: - f.write(finalMessage+ENDL) - finally: - glob.fLocks.unlockFile(of) + glob.fileBuffers.write(".data/"+of, finalMessage+ENDL) def warning(message, discord = None, alertDev = False): """ diff --git a/helpers/systemHelper.py b/helpers/systemHelper.py index ef0383f..8c3445e 100644 --- a/helpers/systemHelper.py +++ b/helpers/systemHelper.py @@ -1,14 +1,25 @@ from objects import glob from constants import serverPackets +from helpers import consoleHelper import psutil import os import sys import threading import signal from helpers import logHelper as log +from constants import bcolors import time import math +def dispose(): + """ + Perform some clean up. Called on shutdown. + :return: + """ + print("> Disposing server... ") + glob.fileBuffers.flushAll() + consoleHelper.printColored("Goodbye!", bcolors.GREEN) + def runningUnderUnix(): """ Get if the server is running under UNIX or NT @@ -49,11 +60,13 @@ def scheduleShutdown(sendRestartTime, restart, message = "", delay=20): def restartServer(): """Restart pep.py script""" log.info("Restarting pep.py...") + dispose() os.execv(sys.executable, [sys.executable] + sys.argv) def shutdownServer(): """Shutdown pep.py""" log.info("Shutting down pep.py...") + dispose() sig = signal.SIGKILL if runningUnderUnix() else signal.CTRL_C_EVENT os.kill(os.getpid(), sig) diff --git a/objects/fileBuffer.py b/objects/fileBuffer.py new file mode 100644 index 0000000..83a6d16 --- /dev/null +++ b/objects/fileBuffer.py @@ -0,0 +1,79 @@ +from objects import glob + +class buffer(): + """ + A file buffer object. + This buffer caches data in memory and when it's full, it writes the content to a file. + """ + def __init__(self, fileName, writeType="a", maxLength=512): + """ + A file buffer object + + :param fileName: Path and name of file on disk . + :param writeType: File write type. Optional. Default: "a" . + :param maxLength: Max length before writing buffer to disk. Optional. Default: 512. + """ + self.content = "" + self.length = 0 + self.fileName = fileName + self.writeType = writeType + self.maxLength = maxLength + + def write(self, newData): + """ + Add data to buffer. + If the total length of the data in buffer is greater than or equal to self.maxLength, + the content is written on the disk and the buffer resets + + :param newData: Data to append to buffer + :return: + """ + self.content += newData + self.length += len(newData) + if self.length >= self.maxLength: + self.flush() + + def flush(self): + """ + Write buffer content to disk and reset its content + + :return: + """ + try: + glob.fLocks.lockFile(self.fileName) + with open(self.fileName, self.writeType) as f: + f.write(self.content) + finally: + glob.fLocks.unlockFile(self.fileName) + + self.content = "" + self.length = 0 + +class buffersList(): + """ + A list of buffers + """ + def __init__(self): + self.buffers = {} + + def write(self, fileName, content): + """ + Write some data to an existing buffer in this list (or create a new one if it doesn't exist). + If the buffer is full, the data is written to the file and the buffer resets. + + :param fileName: Path of file/buffer + :param content: New content + :return: + """ + if fileName not in self.buffers: + self.buffers[fileName] = buffer(fileName) + self.buffers[fileName].write(content) + + def flushAll(self): + """ + Write all buffers to file and flush them + + :return: + """ + for _, value in self.buffers.items(): + value.flush() \ No newline at end of file diff --git a/objects/glob.py b/objects/glob.py index efd2191..d86a64a 100644 --- a/objects/glob.py +++ b/objects/glob.py @@ -4,6 +4,7 @@ from objects import tokenList from objects import channelList from objects import matchList from objects import fileLocks +from objects import fileBuffer import time try: @@ -22,6 +23,7 @@ channels = channelList.channelList() matches = matchList.matchList() restarting = False fLocks = fileLocks.fileLocks() +fileBuffers = fileBuffer.buffersList() verifiedCache = {} cloudflare = False chatFilters = None diff --git a/pep.py b/pep.py index a7490e1..f9ba43e 100644 --- a/pep.py +++ b/pep.py @@ -26,6 +26,7 @@ from helpers import databaseHelperNew from helpers import generalFunctions from helpers import logHelper as log from helpers import userHelper +from helpers import systemHelper as system from handlers import mainHandler from handlers import apiIsOnlineHandler @@ -49,161 +50,167 @@ def make_app(): ]) if __name__ == "__main__": - # Server start - consoleHelper.printServerStartHeader(True) - - # Read config.ini - consoleHelper.printNoNl("> Loading config file... ") - glob.conf = configHelper.config("config.ini") - - if glob.conf.default: - # We have generated a default config.ini, quit server - consoleHelper.printWarning() - consoleHelper.printColored("[!] config.ini not found. A default one has been generated.", bcolors.YELLOW) - consoleHelper.printColored("[!] Please edit your config.ini and run the server again.", bcolors.YELLOW) - sys.exit() - - # If we haven't generated a default config.ini, check if it's valid - if not glob.conf.checkConfig(): - consoleHelper.printError() - consoleHelper.printColored("[!] Invalid config.ini. Please configure it properly", bcolors.RED) - consoleHelper.printColored("[!] Delete your config.ini to generate a default one", bcolors.RED) - sys.exit() - else: - consoleHelper.printDone() - - # Create data folder if needed - consoleHelper.printNoNl("> Checking folders... ") - paths = [".data"] - for i in paths: - if not os.path.exists(i): - os.makedirs(i, 0o770) - consoleHelper.printDone() - - # Connect to db try: - consoleHelper.printNoNl("> Connecting to MySQL db") - glob.db = databaseHelperNew.db(glob.conf.config["db"]["host"], glob.conf.config["db"]["username"], glob.conf.config["db"]["password"], glob.conf.config["db"]["database"], int(glob.conf.config["db"]["workers"])) - consoleHelper.printNoNl(" ") - consoleHelper.printDone() - except: - # Exception while connecting to db - consoleHelper.printError() - consoleHelper.printColored("[!] Error while connection to database. Please check your config.ini and run the server again", bcolors.RED) - raise + # Server start + consoleHelper.printServerStartHeader(True) - # Load bancho_settings - try: - consoleHelper.printNoNl("> Loading bancho settings from DB... ") - glob.banchoConf = banchoConfig.banchoConfig() - consoleHelper.printDone() - except: - consoleHelper.printError() - consoleHelper.printColored("[!] Error while loading bancho_settings. Please make sure the table in DB has all the required rows", bcolors.RED) - raise + # Read config.ini + consoleHelper.printNoNl("> Loading config file... ") + glob.conf = configHelper.config("config.ini") - # Delete old bancho sessions - consoleHelper.printNoNl("> Deleting cached bancho sessions from DB... ") - glob.tokens.deleteBanchoSessions() - consoleHelper.printDone() + if glob.conf.default: + # We have generated a default config.ini, quit server + consoleHelper.printWarning() + consoleHelper.printColored("[!] config.ini not found. A default one has been generated.", bcolors.YELLOW) + consoleHelper.printColored("[!] Please edit your config.ini and run the server again.", bcolors.YELLOW) + sys.exit() - try: - consoleHelper.printNoNl("> Loading chat filters... ") - glob.chatFilters = chatFilters.chatFilters() - consoleHelper.printDone() - except: - consoleHelper.printError() - consoleHelper.printColored("[!] Error while loading chat filters. Make sure there is a filters.txt file present", bcolors.RED) - raise - - # Initialize chat channels - print("> Initializing chat channels... ") - glob.channels.loadChannels() - consoleHelper.printDone() - - # Start fokabot - consoleHelper.printNoNl("> Connecting FokaBot... ") - fokabot.connect() - consoleHelper.printDone() - - # Initialize user timeout check loop - consoleHelper.printNoNl("> Initializing user timeout check loop... ") - glob.tokens.usersTimeoutCheckLoop() - consoleHelper.printDone() - - # Initialize spam protection reset loop - consoleHelper.printNoNl("> Initializing spam protection reset loop... ") - glob.tokens.spamProtectionResetLoop() - consoleHelper.printDone() - - # Cache user ids - consoleHelper.printNoNl("> Caching user IDs... ") - userHelper.cacheUserIDs() - consoleHelper.printDone() - - # Localize warning - glob.localize = generalFunctions.stringToBool(glob.conf.config["localize"]["enable"]) - if not glob.localize: - consoleHelper.printColored("[!] Warning! Users localization is disabled!", bcolors.YELLOW) - - # Discord - glob.discord = generalFunctions.stringToBool(glob.conf.config["discord"]["enable"]) - if not glob.discord: - consoleHelper.printColored("[!] Warning! Discord logging is disabled!", bcolors.YELLOW) - - # Gzip - glob.gzip = generalFunctions.stringToBool(glob.conf.config["server"]["gzip"]) - glob.gziplevel = int(glob.conf.config["server"]["gziplevel"]) - if not glob.gzip: - consoleHelper.printColored("[!] Warning! Gzip compression is disabled!", bcolors.YELLOW) - - # Debug mode - glob.debug = generalFunctions.stringToBool(glob.conf.config["debug"]["enable"]) - glob.outputPackets = generalFunctions.stringToBool(glob.conf.config["debug"]["packets"]) - glob.outputRequestTime = generalFunctions.stringToBool(glob.conf.config["debug"]["time"]) - if glob.debug: - consoleHelper.printColored("[!] Warning! Server running in debug mode!", bcolors.YELLOW) - - # Make app - application = make_app() - - # Set up sentry - try: - glob.sentry = generalFunctions.stringToBool(glob.conf.config["sentry"]["enable"]) - if glob.sentry: - application.sentry_client = AsyncSentryClient(glob.conf.config["sentry"]["banchodns"], release=glob.VERSION) + # If we haven't generated a default config.ini, check if it's valid + if not glob.conf.checkConfig(): + consoleHelper.printError() + consoleHelper.printColored("[!] Invalid config.ini. Please configure it properly", bcolors.RED) + consoleHelper.printColored("[!] Delete your config.ini to generate a default one", bcolors.RED) + sys.exit() else: - consoleHelper.printColored("[!] Warning! Sentry logging is disabled!", bcolors.YELLOW) - except: - consoleHelper.printColored("[!] Error while starting sentry client! Please check your config.ini and run the server again", bcolors.RED) + consoleHelper.printDone() - # Cloudflare memes - glob.cloudflare = generalFunctions.stringToBool(glob.conf.config["server"]["cloudflare"]) + # Create data folder if needed + consoleHelper.printNoNl("> Checking folders... ") + paths = [".data"] + for i in paths: + if not os.path.exists(i): + os.makedirs(i, 0o770) + consoleHelper.printDone() - # IRC start message and console output - glob.irc = generalFunctions.stringToBool(glob.conf.config["irc"]["enable"]) - if glob.irc: - # IRC port + # Flush file buffers at exit + #atexit.register(lambda: glob.fileBuffers.flushAll()) + + # Connect to db try: - ircPort = int(glob.conf.config["irc"]["port"]) + consoleHelper.printNoNl("> Connecting to MySQL db") + glob.db = databaseHelperNew.db(glob.conf.config["db"]["host"], glob.conf.config["db"]["username"], glob.conf.config["db"]["password"], glob.conf.config["db"]["database"], int(glob.conf.config["db"]["workers"])) + consoleHelper.printNoNl(" ") + consoleHelper.printDone() except: - consoleHelper.printColored("[!] Invalid IRC port! Please check your config.ini and run the server again", bcolors.RED) - log.logMessage("IRC server started!", discord=True, of="info.txt", stdout=False) - consoleHelper.printColored("> IRC server listening on 127.0.0.1:{}...".format(ircPort), bcolors.GREEN) - threading.Thread(target=lambda: ircserver.main(port=ircPort)).start() - else: - consoleHelper.printColored("[!] Warning! IRC server is disabled!", bcolors.YELLOW) + # Exception while connecting to db + consoleHelper.printError() + consoleHelper.printColored("[!] Error while connection to database. Please check your config.ini and run the server again", bcolors.RED) + raise - # Server port - try: - serverPort = int(glob.conf.config["server"]["port"]) - except: - consoleHelper.printColored("[!] Invalid server port! Please check your config.ini and run the server again", bcolors.RED) + # Load bancho_settings + try: + consoleHelper.printNoNl("> Loading bancho settings from DB... ") + glob.banchoConf = banchoConfig.banchoConfig() + consoleHelper.printDone() + except: + consoleHelper.printError() + consoleHelper.printColored("[!] Error while loading bancho_settings. Please make sure the table in DB has all the required rows", bcolors.RED) + raise - # Server start message and console output - log.logMessage("Server started!", discord=True, of="info.txt", stdout=False) - consoleHelper.printColored("> Tornado listening for HTTP(s) clients on 127.0.0.1:{}...".format(serverPort), bcolors.GREEN) + # Delete old bancho sessions + consoleHelper.printNoNl("> Deleting cached bancho sessions from DB... ") + glob.tokens.deleteBanchoSessions() + consoleHelper.printDone() - # Start tornado - application.listen(serverPort) - tornado.ioloop.IOLoop.instance().start() \ No newline at end of file + try: + consoleHelper.printNoNl("> Loading chat filters... ") + glob.chatFilters = chatFilters.chatFilters() + consoleHelper.printDone() + except: + consoleHelper.printError() + consoleHelper.printColored("[!] Error while loading chat filters. Make sure there is a filters.txt file present", bcolors.RED) + raise + + # Initialize chat channels + print("> Initializing chat channels... ") + glob.channels.loadChannels() + consoleHelper.printDone() + + # Start fokabot + consoleHelper.printNoNl("> Connecting FokaBot... ") + fokabot.connect() + consoleHelper.printDone() + + # Initialize user timeout check loop + consoleHelper.printNoNl("> Initializing user timeout check loop... ") + glob.tokens.usersTimeoutCheckLoop() + consoleHelper.printDone() + + # Initialize spam protection reset loop + consoleHelper.printNoNl("> Initializing spam protection reset loop... ") + glob.tokens.spamProtectionResetLoop() + consoleHelper.printDone() + + # Cache user ids + consoleHelper.printNoNl("> Caching user IDs... ") + userHelper.cacheUserIDs() + consoleHelper.printDone() + + # Localize warning + glob.localize = generalFunctions.stringToBool(glob.conf.config["localize"]["enable"]) + if not glob.localize: + consoleHelper.printColored("[!] Warning! Users localization is disabled!", bcolors.YELLOW) + + # Discord + glob.discord = generalFunctions.stringToBool(glob.conf.config["discord"]["enable"]) + if not glob.discord: + consoleHelper.printColored("[!] Warning! Discord logging is disabled!", bcolors.YELLOW) + + # Gzip + glob.gzip = generalFunctions.stringToBool(glob.conf.config["server"]["gzip"]) + glob.gziplevel = int(glob.conf.config["server"]["gziplevel"]) + if not glob.gzip: + consoleHelper.printColored("[!] Warning! Gzip compression is disabled!", bcolors.YELLOW) + + # Debug mode + glob.debug = generalFunctions.stringToBool(glob.conf.config["debug"]["enable"]) + glob.outputPackets = generalFunctions.stringToBool(glob.conf.config["debug"]["packets"]) + glob.outputRequestTime = generalFunctions.stringToBool(glob.conf.config["debug"]["time"]) + if glob.debug: + consoleHelper.printColored("[!] Warning! Server running in debug mode!", bcolors.YELLOW) + + # Make app + application = make_app() + + # Set up sentry + try: + glob.sentry = generalFunctions.stringToBool(glob.conf.config["sentry"]["enable"]) + if glob.sentry: + application.sentry_client = AsyncSentryClient(glob.conf.config["sentry"]["banchodns"], release=glob.VERSION) + else: + consoleHelper.printColored("[!] Warning! Sentry logging is disabled!", bcolors.YELLOW) + except: + consoleHelper.printColored("[!] Error while starting sentry client! Please check your config.ini and run the server again", bcolors.RED) + + # Cloudflare memes + glob.cloudflare = generalFunctions.stringToBool(glob.conf.config["server"]["cloudflare"]) + + # IRC start message and console output + glob.irc = generalFunctions.stringToBool(glob.conf.config["irc"]["enable"]) + if glob.irc: + # IRC port + try: + ircPort = int(glob.conf.config["irc"]["port"]) + except: + consoleHelper.printColored("[!] Invalid IRC port! Please check your config.ini and run the server again", bcolors.RED) + log.logMessage("IRC server started!", discord=True, of="info.txt", stdout=False) + consoleHelper.printColored("> IRC server listening on 127.0.0.1:{}...".format(ircPort), bcolors.GREEN) + threading.Thread(target=lambda: ircserver.main(port=ircPort)).start() + else: + consoleHelper.printColored("[!] Warning! IRC server is disabled!", bcolors.YELLOW) + + # Server port + try: + serverPort = int(glob.conf.config["server"]["port"]) + except: + consoleHelper.printColored("[!] Invalid server port! Please check your config.ini and run the server again", bcolors.RED) + + # Server start message and console output + log.logMessage("Server started!", discord=True, of="info.txt", stdout=False) + consoleHelper.printColored("> Tornado listening for HTTP(s) clients on 127.0.0.1:{}...".format(serverPort), bcolors.GREEN) + + # Start tornado + application.listen(serverPort) + tornado.ioloop.IOLoop.instance().start() + finally: + system.dispose() \ No newline at end of file