2016-05-19 20:53:09 +00:00
import os
2016-09-04 08:56:10 +00:00
import sys
2016-07-14 10:37:07 +00:00
import threading
2016-10-02 20:48:14 +00:00
from multiprocessing . pool import ThreadPool
2016-09-04 08:56:10 +00:00
import tornado . gen
import tornado . httpserver
2016-05-31 20:49:30 +00:00
import tornado . ioloop
import tornado . web
2016-06-15 17:01:00 +00:00
from raven . contrib . tornado import AsyncSentryClient
2016-11-15 19:36:29 +00:00
import redis
2016-06-15 17:01:00 +00:00
2018-10-09 08:54:35 +00:00
import json
import shutil
from distutils . version import LooseVersion
2018-07-13 10:30:23 +00:00
from common import generalUtils , agpl
2016-10-02 20:48:14 +00:00
from common . constants import bcolors
from common . db import dbConnector
2016-10-06 21:06:59 +00:00
from common . ddog import datadogClient
2016-10-02 20:48:14 +00:00
from common . log import logUtils as log
2016-11-20 10:31:51 +00:00
from common . redis import pubSub
2016-10-02 20:48:14 +00:00
from common . web import schiavo
from handlers import apiFokabotMessageHandler
2016-06-02 20:33:39 +00:00
from handlers import apiIsOnlineHandler
from handlers import apiOnlineUsersHandler
from handlers import apiServerStatusHandler
2016-07-20 09:59:53 +00:00
from handlers import apiVerifiedStatusHandler
2016-10-02 20:48:14 +00:00
from handlers import ciTriggerHandler
from handlers import mainHandler
2016-10-09 17:12:18 +00:00
from handlers import heavyHandler
2016-10-02 20:48:14 +00:00
from helpers import configHelper
from helpers import consoleHelper
from helpers import systemHelper as system
2016-07-14 10:37:07 +00:00
from irc import ircserver
2016-10-02 20:48:14 +00:00
from objects import banchoConfig
from objects import chatFilters
from objects import fokabot
from objects import glob
2016-11-20 10:31:51 +00:00
from pubSubHandlers import changeUsernameHandler
from pubSubHandlers import disconnectHandler
from pubSubHandlers import banHandler
2016-12-20 20:22:25 +00:00
from pubSubHandlers import notificationHandler
2016-11-20 10:31:51 +00:00
from pubSubHandlers import updateSilenceHandler
from pubSubHandlers import updateStatsHandler
2016-10-02 20:48:14 +00:00
2018-07-13 10:30:23 +00:00
2016-05-31 20:49:30 +00:00
def make_app ( ) :
return tornado . web . Application ( [
2016-06-02 20:33:39 +00:00
( r " / " , mainHandler . handler ) ,
( r " /api/v1/isOnline " , apiIsOnlineHandler . handler ) ,
( r " /api/v1/onlineUsers " , apiOnlineUsersHandler . handler ) ,
( r " /api/v1/serverStatus " , apiServerStatusHandler . handler ) ,
( r " /api/v1/ciTrigger " , ciTriggerHandler . handler ) ,
2016-07-20 09:59:53 +00:00
( r " /api/v1/verifiedStatus " , apiVerifiedStatusHandler . handler ) ,
2016-10-09 17:12:18 +00:00
( r " /api/v1/fokabotMessage " , apiFokabotMessageHandler . handler ) ,
( r " /stress " , heavyHandler . handler )
2016-05-31 20:49:30 +00:00
] )
2016-04-19 17:40:59 +00:00
2018-07-13 10:30:23 +00:00
2016-04-19 17:40:59 +00:00
if __name__ == " __main__ " :
2018-07-13 10:30:23 +00:00
# AGPL license agreement
try :
agpl . check_license ( " ripple " , " pep.py " )
except agpl . LicenseError as e :
print ( str ( e ) )
sys . exit ( 1 )
2016-09-04 10:01:10 +00:00
try :
# Server start
consoleHelper . printServerStartHeader ( True )
# Read config.ini
consoleHelper . printNoNl ( " > Loading config file... " )
glob . conf = configHelper . config ( " config.ini " )
if glob . conf . default :
# We have generated a default config.ini, quit server
consoleHelper . printWarning ( )
consoleHelper . printColored ( " [!] config.ini not found. A default one has been generated. " , bcolors . YELLOW )
consoleHelper . printColored ( " [!] Please edit your config.ini and run the server again. " , bcolors . YELLOW )
sys . exit ( )
# If we haven't generated a default config.ini, check if it's valid
if not glob . conf . checkConfig ( ) :
consoleHelper . printError ( )
consoleHelper . printColored ( " [!] Invalid config.ini. Please configure it properly " , bcolors . RED )
consoleHelper . printColored ( " [!] Delete your config.ini to generate a default one " , bcolors . RED )
sys . exit ( )
else :
consoleHelper . printDone ( )
2018-10-09 08:54:35 +00:00
# Read additional config file
consoleHelper . printNoNl ( " > Loading additional config file... " )
try :
if not os . path . isfile ( glob . conf . config [ " custom " ] [ " config " ] ) :
consoleHelper . printWarning ( )
consoleHelper . printColored ( " [!] Missing config file at {} ; A default one has been generated at this location. " . format ( glob . conf . config [ " custom " ] [ " config " ] ) , bcolors . YELLOW )
shutil . copy ( " common/default_config.json " , glob . conf . config [ " custom " ] [ " config " ] )
with open ( glob . conf . config [ " custom " ] [ " config " ] , " r " ) as f :
glob . conf . extra = json . load ( f )
consoleHelper . printDone ( )
except :
consoleHelper . printWarning ( )
consoleHelper . printColored ( " [!] Unable to load custom config at {} " . format ( glob . conf . config [ " custom " ] [ " config " ] ) , bcolors . RED )
consoleHelper . printColored ( " [!] Make sure you have the latest osufx common submodule! " , bcolors . RED )
sys . exit ( )
# Check if running common module is usable
if glob . COMMON_VERSION == " Unknown " :
consoleHelper . printWarning ( )
consoleHelper . printColored ( " [!] You do not seem to be using osufx ' s common submodule... nothing will work... " , bcolors . RED )
consoleHelper . printColored ( " [!] You can download or fork the submodule from {} https://github.com/osufx/ripple-python-common " . format ( bcolors . UNDERLINE ) , bcolors . RED )
sys . exit ( )
elif LooseVersion ( glob . COMMON_VERSION_REQ ) > LooseVersion ( glob . COMMON_VERSION ) :
consoleHelper . printColored ( " [!] Your common submodule version is below the required version number for this version of pep.py. " , bcolors . RED )
consoleHelper . printColored ( " [!] You are highly adviced to update your common submodule as stability may vary with outdated modules. " , bcolors . RED )
2016-09-04 10:01:10 +00:00
# Create data folder if needed
consoleHelper . printNoNl ( " > Checking folders... " )
paths = [ " .data " ]
for i in paths :
if not os . path . exists ( i ) :
os . makedirs ( i , 0o770 )
2016-04-19 17:40:59 +00:00
consoleHelper . printDone ( )
2016-09-04 10:01:10 +00:00
# Connect to db
try :
2016-11-15 19:36:29 +00:00
consoleHelper . printNoNl ( " > Connecting to MySQL database... " )
2016-10-02 20:48:14 +00:00
glob . db = dbConnector . db ( glob . conf . config [ " db " ] [ " host " ] , glob . conf . config [ " db " ] [ " username " ] , glob . conf . config [ " db " ] [ " password " ] , glob . conf . config [ " db " ] [ " database " ] , int ( glob . conf . config [ " db " ] [ " workers " ] ) )
2016-09-04 10:01:10 +00:00
consoleHelper . printNoNl ( " " )
consoleHelper . printDone ( )
except :
# Exception while connecting to db
consoleHelper . printError ( )
consoleHelper . printColored ( " [!] Error while connection to database. Please check your config.ini and run the server again " , bcolors . RED )
raise
2016-11-15 19:36:29 +00:00
# Connect to redis
try :
consoleHelper . printNoNl ( " > Connecting to redis... " )
glob . redis = redis . Redis ( glob . conf . config [ " redis " ] [ " host " ] , glob . conf . config [ " redis " ] [ " port " ] , glob . conf . config [ " redis " ] [ " database " ] , glob . conf . config [ " redis " ] [ " password " ] )
glob . redis . ping ( )
consoleHelper . printNoNl ( " " )
consoleHelper . printDone ( )
except :
# Exception while connecting to db
consoleHelper . printError ( )
consoleHelper . printColored ( " [!] Error while connection to redis. Please check your config.ini and run the server again " , bcolors . RED )
raise
# Empty redis cache
try :
2016-11-20 12:03:07 +00:00
# TODO: Make function or some redis meme
2016-11-20 13:17:05 +00:00
glob . redis . set ( " ripple:online_users " , 0 )
2016-11-15 19:36:29 +00:00
glob . redis . eval ( " return redis.call( ' del ' , unpack(redis.call( ' keys ' , ARGV[1]))) " , 0 , " peppy:* " )
except redis . exceptions . ResponseError :
# Script returns error if there are no keys starting with peppy:*
pass
2016-11-17 19:07:06 +00:00
# Save peppy version in redis
glob . redis . set ( " peppy:version " , glob . VERSION )
2016-09-04 10:01:10 +00:00
# Load bancho_settings
try :
consoleHelper . printNoNl ( " > Loading bancho settings from DB... " )
glob . banchoConf = banchoConfig . banchoConfig ( )
consoleHelper . printDone ( )
except :
consoleHelper . printError ( )
consoleHelper . printColored ( " [!] Error while loading bancho_settings. Please make sure the table in DB has all the required rows " , bcolors . RED )
raise
# Delete old bancho sessions
consoleHelper . printNoNl ( " > Deleting cached bancho sessions from DB... " )
glob . tokens . deleteBanchoSessions ( )
2016-04-19 17:40:59 +00:00
consoleHelper . printDone ( )
2016-09-04 14:07:10 +00:00
# Create threads pool
try :
consoleHelper . printNoNl ( " > Creating threads pool... " )
glob . pool = ThreadPool ( int ( glob . conf . config [ " server " ] [ " threads " ] ) )
consoleHelper . printDone ( )
2016-12-26 09:33:05 +00:00
except ValueError :
2016-09-04 14:07:10 +00:00
consoleHelper . printError ( )
consoleHelper . printColored ( " [!] Error while creating threads pool. Please check your config.ini and run the server again " , bcolors . RED )
2016-09-04 10:01:10 +00:00
try :
consoleHelper . printNoNl ( " > Loading chat filters... " )
glob . chatFilters = chatFilters . chatFilters ( )
consoleHelper . printDone ( )
except :
consoleHelper . printError ( )
consoleHelper . printColored ( " [!] Error while loading chat filters. Make sure there is a filters.txt file present " , bcolors . RED )
raise
2016-12-11 10:07:35 +00:00
# Start fokabot
2018-02-14 16:44:37 +00:00
consoleHelper . printNoNl ( " > Connecting bot... " )
2016-12-11 10:07:35 +00:00
fokabot . connect ( )
consoleHelper . printDone ( )
2016-09-04 10:01:10 +00:00
# Initialize chat channels
print ( " > Initializing chat channels... " )
glob . channels . loadChannels ( )
2016-04-19 17:40:59 +00:00
consoleHelper . printDone ( )
2016-10-01 19:19:03 +00:00
# Initialize stremas
2016-10-04 20:10:07 +00:00
consoleHelper . printNoNl ( " > Creating packets streams... " )
2016-10-01 19:19:03 +00:00
glob . streams . add ( " main " )
2016-10-04 20:10:07 +00:00
glob . streams . add ( " lobby " )
2016-10-01 19:19:03 +00:00
consoleHelper . printDone ( )
2016-09-04 10:01:10 +00:00
# Initialize user timeout check loop
consoleHelper . printNoNl ( " > Initializing user timeout check loop... " )
glob . tokens . usersTimeoutCheckLoop ( )
consoleHelper . printDone ( )
# Initialize spam protection reset loop
consoleHelper . printNoNl ( " > Initializing spam protection reset loop... " )
glob . tokens . spamProtectionResetLoop ( )
2016-08-08 03:19:52 +00:00
consoleHelper . printDone ( )
2016-07-14 13:38:28 +00:00
2018-03-23 19:59:04 +00:00
# Initialize multiplayer cleanup loop
consoleHelper . printNoNl ( " > Initializing multiplayer cleanup loop... " )
glob . matches . cleanupLoop ( )
consoleHelper . printDone ( )
2016-09-04 10:01:10 +00:00
# Localize warning
2016-10-02 20:48:14 +00:00
glob . localize = generalUtils . stringToBool ( glob . conf . config [ " localize " ] [ " enable " ] )
2016-09-04 10:01:10 +00:00
if not glob . localize :
consoleHelper . printColored ( " [!] Warning! Users localization is disabled! " , bcolors . YELLOW )
# Discord
2016-10-02 20:48:14 +00:00
if generalUtils . stringToBool ( glob . conf . config [ " discord " ] [ " enable " ] ) :
2016-12-26 08:52:43 +00:00
glob . schiavo = schiavo . schiavo ( glob . conf . config [ " discord " ] [ " boturl " ] , " **pep.py** " )
2016-10-02 20:48:14 +00:00
else :
2016-09-04 10:01:10 +00:00
consoleHelper . printColored ( " [!] Warning! Discord logging is disabled! " , bcolors . YELLOW )
# Gzip
2016-10-02 20:48:14 +00:00
glob . gzip = generalUtils . stringToBool ( glob . conf . config [ " server " ] [ " gzip " ] )
2016-09-04 10:01:10 +00:00
glob . gziplevel = int ( glob . conf . config [ " server " ] [ " gziplevel " ] )
if not glob . gzip :
consoleHelper . printColored ( " [!] Warning! Gzip compression is disabled! " , bcolors . YELLOW )
# Debug mode
2016-10-02 20:48:14 +00:00
glob . debug = generalUtils . stringToBool ( glob . conf . config [ " debug " ] [ " enable " ] )
glob . outputPackets = generalUtils . stringToBool ( glob . conf . config [ " debug " ] [ " packets " ] )
glob . outputRequestTime = generalUtils . stringToBool ( glob . conf . config [ " debug " ] [ " time " ] )
2016-09-04 10:01:10 +00:00
if glob . debug :
consoleHelper . printColored ( " [!] Warning! Server running in debug mode! " , bcolors . YELLOW )
# Make app
2016-09-16 16:21:34 +00:00
glob . application = make_app ( )
2016-09-04 10:01:10 +00:00
# Set up sentry
2016-07-14 10:37:07 +00:00
try :
2016-10-02 20:48:14 +00:00
glob . sentry = generalUtils . stringToBool ( glob . conf . config [ " sentry " ] [ " enable " ] )
2016-09-04 10:01:10 +00:00
if glob . sentry :
2017-08-26 20:28:05 +00:00
glob . application . sentry_client = AsyncSentryClient ( glob . conf . config [ " sentry " ] [ " banchodsn " ] , release = glob . VERSION )
2016-09-04 10:01:10 +00:00
else :
consoleHelper . printColored ( " [!] Warning! Sentry logging is disabled! " , bcolors . YELLOW )
2016-07-14 10:37:07 +00:00
except :
2016-09-04 10:01:10 +00:00
consoleHelper . printColored ( " [!] Error while starting sentry client! Please check your config.ini and run the server again " , bcolors . RED )
2016-10-06 21:06:59 +00:00
# Set up datadog
try :
if generalUtils . stringToBool ( glob . conf . config [ " datadog " ] [ " enable " ] ) :
glob . dog = datadogClient . datadogClient (
glob . conf . config [ " datadog " ] [ " apikey " ] ,
glob . conf . config [ " datadog " ] [ " appkey " ] ,
[
datadogClient . periodicCheck ( " online_users " , lambda : len ( glob . tokens . tokens ) ) ,
datadogClient . periodicCheck ( " multiplayer_matches " , lambda : len ( glob . matches . matches ) ) ,
2016-11-13 11:25:38 +00:00
2016-12-09 10:47:00 +00:00
#datadogClient.periodicCheck("ram_clients", lambda: generalUtils.getTotalSize(glob.tokens)),
#datadogClient.periodicCheck("ram_matches", lambda: generalUtils.getTotalSize(glob.matches)),
#datadogClient.periodicCheck("ram_channels", lambda: generalUtils.getTotalSize(glob.channels)),
#datadogClient.periodicCheck("ram_file_buffers", lambda: generalUtils.getTotalSize(glob.fileBuffers)),
#datadogClient.periodicCheck("ram_file_locks", lambda: generalUtils.getTotalSize(glob.fLocks)),
#datadogClient.periodicCheck("ram_datadog", lambda: generalUtils.getTotalSize(glob.datadogClient)),
#datadogClient.periodicCheck("ram_verified_cache", lambda: generalUtils.getTotalSize(glob.verifiedCache)),
#datadogClient.periodicCheck("ram_irc", lambda: generalUtils.getTotalSize(glob.ircServer)),
#datadogClient.periodicCheck("ram_tornado", lambda: generalUtils.getTotalSize(glob.application)),
#datadogClient.periodicCheck("ram_db", lambda: generalUtils.getTotalSize(glob.db)),
2016-10-06 21:06:59 +00:00
] )
else :
consoleHelper . printColored ( " [!] Warning! Datadog stats tracking is disabled! " , bcolors . YELLOW )
except :
consoleHelper . printColored ( " [!] Error while starting Datadog client! Please check your config.ini and run the server again " , bcolors . RED )
2016-09-04 10:01:10 +00:00
# IRC start message and console output
2016-10-02 20:48:14 +00:00
glob . irc = generalUtils . stringToBool ( glob . conf . config [ " irc " ] [ " enable " ] )
2016-09-04 10:01:10 +00:00
if glob . irc :
# IRC port
2016-12-26 09:33:05 +00:00
ircPort = 0
2016-09-04 10:01:10 +00:00
try :
ircPort = int ( glob . conf . config [ " irc " ] [ " port " ] )
2016-12-26 09:33:05 +00:00
except ValueError :
2016-09-04 10:01:10 +00:00
consoleHelper . printColored ( " [!] Invalid IRC port! Please check your config.ini and run the server again " , bcolors . RED )
2017-01-06 11:23:45 +00:00
log . logMessage ( " IRC server started! " , discord = " bunker " , of = " info.txt " , stdout = False )
2016-09-04 10:01:10 +00:00
consoleHelper . printColored ( " > IRC server listening on 127.0.0.1: {} ... " . format ( ircPort ) , bcolors . GREEN )
threading . Thread ( target = lambda : ircserver . main ( port = ircPort ) ) . start ( )
else :
consoleHelper . printColored ( " [!] Warning! IRC server is disabled! " , bcolors . YELLOW )
# Server port
2016-12-26 09:33:05 +00:00
serverPort = 0
2016-09-04 10:01:10 +00:00
try :
serverPort = int ( glob . conf . config [ " server " ] [ " port " ] )
2016-12-26 09:33:05 +00:00
except ValueError :
2016-09-04 10:01:10 +00:00
consoleHelper . printColored ( " [!] Invalid server port! Please check your config.ini and run the server again " , bcolors . RED )
2016-04-19 17:40:59 +00:00
2016-09-04 10:01:10 +00:00
# Server start message and console output
2017-01-06 11:23:45 +00:00
log . logMessage ( " Server started! " , discord = " bunker " , of = " info.txt " , stdout = False )
2016-09-04 10:01:10 +00:00
consoleHelper . printColored ( " > Tornado listening for HTTP(s) clients on 127.0.0.1: {} ... " . format ( serverPort ) , bcolors . GREEN )
2016-05-19 20:53:09 +00:00
2016-11-20 10:31:51 +00:00
# Connect to pubsub channels
pubSub . listener ( glob . redis , {
" peppy:disconnect " : disconnectHandler . handler ( ) ,
" peppy:change_username " : changeUsernameHandler . handler ( ) ,
" peppy:reload_settings " : lambda x : x == b " reload " and glob . banchoConf . reload ( ) ,
" peppy:update_cached_stats " : updateStatsHandler . handler ( ) ,
" peppy:silence " : updateSilenceHandler . handler ( ) ,
" peppy:ban " : banHandler . handler ( ) ,
2016-12-20 20:22:25 +00:00
" peppy:notification " : notificationHandler . handler ( ) ,
2016-11-20 10:31:51 +00:00
} ) . start ( )
2016-09-04 10:01:10 +00:00
# Start tornado
2016-09-16 16:21:34 +00:00
glob . application . listen ( serverPort )
2016-09-04 10:01:10 +00:00
tornado . ioloop . IOLoop . instance ( ) . start ( )
finally :
system . dispose ( )