implemented logger,

dropped zmq because of constant inconsistency,
daemon process now outputs to file,
deleted client.py


Former-commit-id: d3251e5f6404d0dce42da49e5d577651e718e168
This commit is contained in:
Arthur Lu 2022-02-19 08:52:49 +00:00
parent 2130182212
commit 56447603e1
4 changed files with 144 additions and 113 deletions

View File

@ -3,7 +3,6 @@ from exceptions import ConfigurationError
from cerberus import Validator
from data import set_database_config, get_database_config
from interface import stderr, stdout, INF, ERR
class Configuration:
@ -218,34 +217,34 @@ class Configuration:
def __getitem__(self, key):
return self.config[key]
def resolve_config_conflicts(self, send, client): # needs improvement with new localization scheme
def resolve_config_conflicts(self, logger, client): # needs improvement with new localization scheme
sync = self.sync
priority = self.priority
if sync:
if priority == "local" or priority == "client":
send(stdout, INF, "config-preference set to local/client, loading local config information")
logger.info("config-preference set to local/client, loading local config information")
remote_config = get_database_config(client)
if remote_config != self.config["variable"]:
set_database_config(client, self.config["variable"])
send(stdout, INF, "database config was different and was updated")
logger.info("database config was different and was updated")
# no change to config
elif priority == "remote" or priority == "database":
send(stdout, INF, "config-preference set to remote/database, loading remote config information")
logger.info("config-preference set to remote/database, loading remote config information")
remote_config = get_database_config(client)
if remote_config != self.config["variable"]:
self.config["variable"] = remote_config
self.save_config()
# change variable to match remote
send(stdout, INF, "local config was different and was updated")
logger.info("local config was different and was updated")
else:
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"")
else:
if priority == "local" or priority == "client":
send(stdout, INF, "config-preference set to local/client, loading local config information")
logger.info("config-preference set to local/client, loading local config information")
# no change to config
elif priority == "remote" or priority == "database":
send(stdout, INF, "config-preference set to remote/database, loading database config information")
logger.info("config-preference set to remote/database, loading database config information")
self.config["variable"] = get_database_config(client)
# change variable to match remote without updating local version
else:

View File

@ -1,44 +1,87 @@
import sys
import time
from os import system, name
from logging import Logger as L
import datetime
import platform
import json
empty_delim = " "
hard_divided_delim = "|"
soft_divided_delim = "|"
l_brack = "["
r_brack = "]"
class Logger(L):
ERR = "[ERR]"
INF = "[INF]"
send = None
file = None
debug = False
stdout = sys.stdout
stderr = sys.stderr
levels = {
10:"[DEBUG] ",
20:"[INFO] ",
30:"[WARNING] ",
40:"[ERROR] ",
50:"[CRITICAL]",
}
def log(target, level, message, code = 0):
def __init__(self, verbose, profile, debug, file = None):
super().__init__("tra_logger")
self.debug = debug
self.file = file
if profile:
self.send = self._send_null
elif verbose:
self.send = self._send_scli
elif debug:
self.send = self._send_scli
elif file != None:
self.send = self._send_file
else:
self.send = self._send_null
message = time.ctime() + empty_delim + str(level) + l_brack + f"{code:+05}" + r_brack + empty_delim + soft_divided_delim + empty_delim + message
print(message, file = target)
def _send_null(self, msg):
pass
def clear():
if name == "nt":
system("cls")
else:
system("clear")
def _send_scli(self, msg):
print(msg)
def splash(version):
def _send_file(self, msg):
f = open(self.file, 'a')
f.write(msg + "\n")
f.close()
def hrule():
print("#"+38*"-"+"#")
def box(s):
temp = "|"
temp += s
temp += (40-len(s)-2)*" "
temp += "|"
print(temp)
hrule()
box(" superscript version: " + version)
box(" os: " + platform.system())
box(" python: " + platform.python_version())
hrule()
def get_time_formatted(self):
return datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S %Z")
def log(self, level, msg):
self.send(self.get_time_formatted() + "| " + self.levels[level] + ": " + msg)
def debug(self, msg):
self.log(10, msg)
def info(self, msg):
self.log(20, msg)
def warning(self, msg):
self.log(30, msg)
def error(self, msg):
self.log(40, msg)
def critical(self, msg):
self.log(50, msg)
def splash(self, version):
def hrule():
self.send("#"+38*"-"+"#")
def box(s):
temp = "|"
temp += s
temp += (40-len(s)-2)*" "
temp += "|"
self.send(temp)
hrule()
box(" superscript version: " + version)
box(" os: " + platform.system())
box(" python: " + platform.python_version())
hrule()
def save_module_to_file(self, module, data, results):
f = open(module + ".log", "w")
json.dump({"data": data, "results":results}, f, ensure_ascii=False, indent=4)
f.close()

View File

@ -149,32 +149,27 @@ __author__ = (
# imports:
import json
import os, sys, time
import pymongo # soon to be deprecated
import traceback
import warnings
import zmq
from config import Configuration, ConfigurationError
from data import get_previous_time, set_current_time, check_new_database_matches
from interface import splash, log, ERR, INF, stdout, stderr
from interface import Logger
from module import Match, Metric, Pit
config_path = "config.json"
def main(send, verbose = False, profile = False, debug = False):
def main(logger, verbose, profile, debug, socket_send = None):
def close_all():
if "client" in locals():
client.close()
if "f" in locals():
f.close()
warnings.filterwarnings("ignore")
exit_code = 0
if verbose:
splash(__version__)
logger.splash(__version__)
modules = {"match": Match, "metric": Metric, "pit": Pit}
@ -184,23 +179,25 @@ def main(send, verbose = False, profile = False, debug = False):
loop_start = time.time()
send(stdout, INF, "current time: " + str(loop_start))
logger.info("current time: " + str(loop_start))
config = Configuration(config_path)
send(stdout, INF, "found and loaded config at <" + config_path + ">")
logger.info("found and loaded config at <" + config_path + ">")
apikey, tbakey = config.database, config.tba
send(stdout, INF, "found and loaded database and tba keys")
logger.info("found and loaded database and tba keys")
client = pymongo.MongoClient(apikey)
send(stdout, INF, "established connection to database")
previous_time = get_previous_time(client)
send(stdout, INF, "analysis backtimed to: " + str(previous_time))
logger.info("established connection to database")
config.resolve_config_conflicts(send, client)
previous_time = get_previous_time(client)
logger.info("analysis backtimed to: " + str(previous_time))
config.resolve_config_conflicts(logger, client)
config_modules, competition = config.modules, config.competition
@ -212,70 +209,73 @@ def main(send, verbose = False, profile = False, debug = False):
if not valid:
continue
current_module.run()
send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds")
logger.info(m + " module finished in " + str(time.time() - start) + " seconds")
if debug:
f = open(m + ".log", "w+")
json.dump({"data": current_module.data, "results":current_module.results}, f, ensure_ascii=False, indent=4)
f.close()
logger.save_module_to_file(m, current_module.data, current_module.results) # logging flag check done in logger
set_current_time(client, loop_start)
close_all()
send(stdout, INF, "closed threads and database client")
send(stdout, INF, "finished all tasks in " + str(time.time() - loop_start) + " seconds, looping")
logger.info("closed threads and database client")
logger.info("finished all tasks in " + str(time.time() - loop_start) + " seconds, looping")
if profile:
exit_code = 0
break
if debug:
exit_code = 0
break
event_delay = config["variable"]["event-delay"]
if event_delay:
send(stdout, INF, "loop delayed until database returns new matches")
logger.info("loop delayed until database returns new matches")
new_match = False
while not new_match:
time.sleep(1)
new_match = check_new_database_matches(client, competition)
send(stdout, INF, "database returned new matches")
logger.info("database returned new matches")
else:
loop_delay = float(config["variable"]["loop-delay"])
remaining_time = loop_delay - (time.time() - loop_start)
if remaining_time > 0:
send(stdout, INF, "loop delayed by " + str(remaining_time) + " seconds")
logger.info("loop delayed by " + str(remaining_time) + " seconds")
time.sleep(remaining_time)
except KeyboardInterrupt:
send(stdout, INF, "detected KeyboardInterrupt, killing threads")
logger.info("detected KeyboardInterrupt, killing threads")
close_all()
send(stdout, INF, "terminated threads, exiting")
logger.info("terminated threads, exiting")
break
except ConfigurationError as e:
send(stderr, ERR, "encountered a configuration error: " + str(e), code = e.code)
traceback.print_exc(file = stderr)
logger.error("encountered a configuration error: " + str(e))
logger.error("".join(traceback.format_exception(e)))
#traceback.print_exc(file = stderr)
exit_code = 1
close_all()
break
except Exception as e:
send(stderr, ERR, "encountered an exception while running", code = 1)
traceback.print_exc(file = stderr)
logger.error("encountered an exception while running")
logger.error("".join(traceback.format_exception(e)))
#traceback.print_exc(file = stderr)
exit_code = 1
close_all()
break
return exit_code
def start(pid_path, verbose = False, profile = False, debug = False):
def start(pid_path, verbose, profile, debug):
if profile:
def send(target, level, message, code = 0):
pass
logger = Logger(verbose, profile, debug)
import cProfile, pstats, io
profile = cProfile.Profile()
profile.enable()
exit_code = main(send, profile = True)
exit_code = main(logger, verbose, profile, debug)
profile.disable()
f = open("profile.txt", 'w+')
ps = pstats.Stats(profile, stream = f).sort_stats('cumtime')
@ -284,35 +284,38 @@ def start(pid_path, verbose = False, profile = False, debug = False):
elif verbose:
exit_code = main(log, verbose = verbose)
logger = Logger(verbose, profile, debug)
exit_code = main(logger, verbose, profile, debug)
sys.exit(exit_code)
elif debug:
exit_code = main(log, verbose = True, profile = True, debug = debug)
logger = Logger(verbose, profile, debug)
exit_code = main(logger, verbose, profile, debug)
sys.exit(exit_code)
else:
f = open('errorlog.log', 'w+')
logfile = "logfile.log"
f = open(logfile, 'w+')
f.close()
e = open('errorlog.log', 'w+')
with daemon.DaemonContext(
working_directory = os.getcwd(),
pidfile = pidfile.TimeoutPIDLockFile(pid_path),
stderr = f
working_directory = os.getcwd(),
pidfile = pidfile.TimeoutPIDLockFile(pid_path),
stderr = e
):
context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.bind("tcp://*:5678")
logger = Logger(verbose, profile, debug, file = logfile)
socket.send(b'status')
exit_code = main(logger, verbose, profile, debug)
def send(target, level, message, code = 0):
socket.send(bytes("status: " + message, 'utf-8'))
exit_code = main(send)
socket.close()
f.close()
sys.exit(exit_code)
def stop(pid_path):
@ -334,12 +337,12 @@ def stop(pid_path):
if os.path.exists(pid_path):
os.remove(pid_path)
else:
traceback.print_exc(file = stderr)
traceback.print_exc(file = sys.stderr)
sys.exit(1)
def restart(pid_path):
stop(pid_path)
start(pid_path)
start(pid_path, False, False, False)
if __name__ == "__main__":
@ -353,17 +356,17 @@ if __name__ == "__main__":
pid_path = "tra-daemon.pid"
if len(sys.argv) == 2:
if 'start' == sys.argv[1]:
start(pid_path)
start(pid_path, False, False, False)
elif 'stop' == sys.argv[1]:
stop(pid_path)
elif 'restart' == sys.argv[1]:
restart(pid_path)
elif 'verbose' == sys.argv[1]:
start(None, verbose = True)
start(None, True, False, False)
elif 'profile' == sys.argv[1]:
start(None, profile=True)
start(None, False, True, False)
elif 'debug' == sys.argv[1]:
start(None, debug = True)
start(None, False, False, True)
else:
print("usage: %s start|stop|restart|verbose|profile|debug" % sys.argv[0])
sys.exit(2)

View File

@ -1,14 +0,0 @@
import signal
import zmq
signal.signal(signal.SIGINT, signal.SIG_DFL)
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect('tcp://localhost:5678')
socket.setsockopt(zmq.SUBSCRIBE, b'status')
while True:
message = socket.recv_multipart()
print(f'Received: {message}')