Many (maybe)unactive threads and huge memory usage #481
Description
Please answer these questions before submitting your issue. Thanks!
-
What version of pyTelegramBotAPI are you using? 3.6.1
-
What OS are you using? Ubuntu 16.04
-
What version of python are you using? 3.5
def __threaded_polling(self, none_stop=False, interval=0, timeout=3):
logger.info('Started polling.')
self.__stop_polling.clear()
error_interval = .25
polling_thread = util.WorkerThread(name="PollingThread")
or_event = util.OrEvent(
polling_thread.done_event,
polling_thread.exception_event,
self.worker_pool.exception_event
)
while not self.__stop_polling.wait(interval):
or_event.clear()
try:
polling_thread.put(self.__retrieve_updates, timeout)
or_event.wait() # wait for polling thread finish, polling thread error or thread pool error
polling_thread.raise_exceptions()
self.worker_pool.raise_exceptions()
error_interval = .25
except apihelper.ApiException as e:
logger.error(e)
if not none_stop:
self.__stop_polling.set()
logger.info("Exception occurred. Stopping.")
else:
polling_thread.clear_exceptions()
self.worker_pool.clear_exceptions()
logger.info("Waiting for {0} seconds until retry".format(error_interval))
time.sleep(error_interval)
error_interval *= 2
except KeyboardInterrupt:
logger.info("KeyboardInterrupt received.")
self.__stop_polling.set()
polling_thread.stop()
break
logger.info('Stopped polling.')`
I was observing the api's polling code and noted that polling_thread isn't stopped when you explicitly call the bot to stop polling (using bot.stop_polling(), this only can be done when you have an auxiliary thread to call it). I did a test and stopped polling many times and started polling again with the same telebot instance. The code I used to do the test is at the very end of the post. After each cycle, the number of active threads increased, without decreasing anytime. I had a problem that, after running the bot for one day, because each thread used some memory, the memory usage became super high.
Note on the images that the memory usage is slowly increasing, in only 8 minutes (the first print was made at 19:02, the last at 19:10).
I tried to modify the api code in my pc, and i'm testing it now. Apparently with a simple modification the threads are closed after polling, but i don't know if it may create bugs on other parts of the api.
def __threaded_polling(self, none_stop=False, interval=0, timeout=3):
logger.info('Started polling.')
self.__stop_polling.clear()
error_interval = .25
polling_thread = util.WorkerThread(name="PollingThread")
or_event = util.OrEvent(
polling_thread.done_event,
polling_thread.exception_event,
self.worker_pool.exception_event
)
#######HERE#############
stopped_polling_thread = False
#######################
while not self.__stop_polling.wait(interval):
or_event.clear()
try:
polling_thread.put(self.__retrieve_updates, timeout)
or_event.wait() # wait for polling thread finish, polling thread error or thread pool error
polling_thread.raise_exceptions()
self.worker_pool.raise_exceptions()
error_interval = .25
except apihelper.ApiException as e:
logger.error(e)
if not none_stop:
self.__stop_polling.set()
logger.info("Exception occurred. Stopping.")
else:
polling_thread.clear_exceptions()
self.worker_pool.clear_exceptions()
logger.info("Waiting for {0} seconds until retry".format(error_interval))
time.sleep(error_interval)
error_interval *= 2
except KeyboardInterrupt:
logger.info("KeyboardInterrupt received.")
self.__stop_polling.set()
polling_thread.stop()
#######HERE#############
stopped_polling_thread = True
########################
break
#######HERE#############
if stopped_polling_thread == False:
polling_thread.stop()
#############################
logger.info('Stopped polling.')
The code I used to test:
import time
from threading import Thread
import gc
import telebot
def handle_message_not_understood(bot):
@bot.message_handler(func = lambda msg: True)
def msg_not_undestood(msg):
user_id = msg.chat.id
bot.send_message(user_id, "#message_not_understood")
def handle_cancel(bot):
@bot.message_handler(func = lambda msg: True, commands = ['cancel'])
def cancel(msg):
user_id = msg.chat.id
bot.send_message(user_id, "#cancel")
def handle_help(bot):
@bot.message_handler(func = lambda msg: True, commands = ['help'])
def help(msg):
user_id = msg.chat.id
bot.send_message(user_id, "#help_msg")
class MessageHandler():
def __init__(self):
self.last_restart = time.time()
self.bot = telebot.TeleBot('INSERT TOKEN HERE')
self.continue_flag = False
self.first = True
def idle_time(self):
return time.time() - self.last_restart
def restart_bot(self):
if self.bot != None:
self.bot.stop_polling()
def stop(self):
self.continue_flag = False
if self.bot != None:
self.bot.stop_polling()
def setup_bot(self):
if self.first:
print("MESSAGE HANDLERS ADD")
handle_cancel(self.bot)
handle_help(self.bot)
handle_message_not_understood(self.bot)
self.first = False
def run(self):
self.continue_flag = True
print("message handler on")
self.setup_bot()
cycles = 0
while self.continue_flag:
#try:
# if cycles % 2 == 0:
# self.logger.warning("Collect garbage")
# gc.collect()
self.last_restart = time.time()
cycles += 1
print("Start again! Cycles: " + str(cycles))
self.bot.polling()
class MessageHandlerThread(Thread):
def __init__(self, message_handler):
Thread.__init__(self)
self.message_handler = message_handler
def stop(self):
self.message_handler.stop()
def run(self):
self.message_handler.run()
def restart_bot(self):
self.message_handler.restart_bot()
def idle_time(self):
return self.message_handler.idle_time()
if __name__ == '__main__':
message_handler = MessageHandler()
message_handler_thread = MessageHandlerThread(message_handler)
message_handler_thread.start()
while True:
aux = message_handler_thread.idle_time()
print(aux)
if message_handler_thread.idle_time() > 2:
message_handler_thread.restart_bot()
time.sleep(2)