Properly handle exceptions in periodic loops

constants/exceptions.py

@@ -105,3 +105,6 @@ class invalidUserException(Exception):
 
 class wrongChannelException(Exception):
 	pass
+
+class periodicLoopException(Exception):
+	pass

objects/matchList.py

@@ -1,6 +1,8 @@
 import threading
 import time
 
+from common.sentry import sentry
+from constants.exceptions import periodicLoopException
 from objects import match
 from objects import glob
 from constants import serverPackets
@@ -67,6 +69,7 @@ class matchList:
 		del self.matches[matchID]
 		log.info("MPROOM{}: Room disposed manually".format(_match.matchID))
 
+	@sentry.capture()
 	def cleanupLoop(self):
 		"""
 		Start match cleanup loop.
@@ -76,9 +79,11 @@ class matchList:
 		This method starts an infinite loop, call it only once!
 		:return:
 		"""
+		try:
 			log.debug("Checking empty matches")
 			t = int(time.time())
 			emptyMatches = []
+			exceptions = []
 
 			# Collect all empty matches
 			for key, m in self.matches.items():
@@ -90,7 +95,18 @@ class matchList:
 
 			# Dispose all empty matches
 			for matchID in emptyMatches:
+				try:
 					self.disposeMatch(matchID)
+				except Exception as e:
+					exceptions.append(e)
+					log.error(
+						"Something wrong happened while disposing a timed out match. Reporting to Sentry when "
+						"the loop ends."
+					)
 
+			# Re-raise exception if needed
+			if exceptions:
+				raise periodicLoopException(exceptions)
+		finally:
 			# Schedule a new check (endless loop)
 			threading.Timer(30, self.cleanupLoop).start()

objects/tokenList.py

@@ -5,11 +5,14 @@ import redis
 
 from common.ripple import userUtils
 from common.log import logUtils as log
+from common.sentry import sentry
 from constants import serverPackets
+from constants.exceptions import periodicLoopException
 from events import logoutEvent
 from objects import glob
 from objects import osuToken
 
+
 class tokenList:
 	def __init__(self):
 		self.tokens = {}
@@ -171,6 +174,7 @@ class tokenList:
 		for _, value in self.tokens.items():
 			value.enqueue(packet)
 
+	@sentry.capture()
 	def usersTimeoutCheckLoop(self):
 		"""
 		Start timed out users disconnect loop.
@@ -178,7 +182,9 @@ class tokenList:
 		CALL THIS FUNCTION ONLY ONCE!
 		:return:
 		"""
+		try:
 			log.debug("Checking timed out clients")
+			exceptions = []
 			timedOutTokens = []		# timed out users
 			timeoutLimit = int(time.time()) - 100
 			for key, value in self.tokens.items():
@@ -193,12 +199,24 @@ class tokenList:
 			for i in timedOutTokens:
 				log.debug("{} timed out!!".format(self.tokens[i].username))
 				self.tokens[i].enqueue(serverPackets.notification("Your connection to the server timed out."))
+				try:
 					logoutEvent.handle(self.tokens[i], None)
+				except Exception as e:
+					exceptions.append(e)
+					log.error(
+						"Something wrong happened while disconnecting a timed out client. Reporting to Sentry "
+						"when the loop ends."
+					)
 			del timedOutTokens
 
+			# Re-raise exceptions if needed
+			if exceptions:
+				raise periodicLoopException(exceptions)
+		finally:
 			# Schedule a new check (endless loop)
 			threading.Timer(100, self.usersTimeoutCheckLoop).start()
 
+	@sentry.capture()
 	def spamProtectionResetLoop(self):
 		"""
 		Start spam protection reset loop.
@@ -207,10 +225,11 @@ class tokenList:
 
 		:return:
 		"""
+		try:
 			# Reset spamRate for every token
 			for _, value in self.tokens.items():
 				value.spamRate = 0
-
+		finally:
 			# Schedule a new check (endless loop)
 			threading.Timer(10, self.spamProtectionResetLoop).start()