alu/tra-superscript
1
0
Fork 0
mirror of https://github.com/titanscouting/tra-superscript.git synced 2024-11-12 22:26:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11 from titanscouting/daemonize-superscript

Browse Source 
Merge changes in daemonize-superscript to refactor-superscript
This commit is contained in:
Arthur Lu 2021-08-18 17:36:55 -07:00 committed by GitHub
commit 3e8d876f80
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 231 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -9,6 +9,10 @@
**/tra_analysis/ **/tra_analysis/
**/temp/* **/temp/*
**/*.pid
**/profile.*
**/errorlog.txt **/errorlog.txt
/dist/superscript.* /dist/superscript.*
/dist/superscript /dist/superscript

51
src/cli/data.py
View File

@ -1,18 +1,16 @@
import requests import requests
import pymongo
import pandas as pd import pandas as pd
def pull_new_tba_matches(apikey, competition, cutoff): def pull_new_tba_matches(apikey, competition, cutoff):
api_key= apikey api_key= apikey
x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth_Key":api_key}) x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth_Key":api_key}, verify=False)
out = [] out = []
for i in x.json(): for i in x.json():
if i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm": if i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm":
out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]})
return out return out
def get_team_match_data(apikey, competition, team_num): def get_team_match_data(client, competition, team_num):
client = pymongo.MongoClient(apikey)
db = client.data_scouting db = client.data_scouting
mdata = db.matchdata mdata = db.matchdata
out = {} out = {}
@ -20,98 +18,87 @@ def get_team_match_data(apikey, competition, team_num):
out[i['match']] = i['data'] out[i['match']] = i['data']
return pd.DataFrame(out) return pd.DataFrame(out)
def get_team_pit_data(apikey, competition, team_num): def get_team_pit_data(client, competition, team_num):
client = pymongo.MongoClient(apikey)
db = client.data_scouting db = client.data_scouting
mdata = db.pitdata mdata = db.pitdata
out = {} out = {}
return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"] return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"]
def get_team_metrics_data(apikey, competition, team_num): def get_team_metrics_data(client, competition, team_num):
client = pymongo.MongoClient(apikey)
db = client.data_processing db = client.data_processing
mdata = db.team_metrics mdata = db.team_metrics
return mdata.find_one({"competition" : competition, "team": team_num}) return mdata.find_one({"competition" : competition, "team": team_num})
def get_match_data_formatted(apikey, competition): def get_match_data_formatted(client, competition):
client = pymongo.MongoClient(apikey)
db = client.data_scouting db = client.data_scouting
mdata = db.teamlist mdata = db.teamlist
x=mdata.find_one({"competition":competition}) x=mdata.find_one({"competition":competition})
out = {} out = {}
for i in x: for i in x:
try: try:
out[int(i)] = unkeyify_2l(get_team_match_data(apikey, competition, int(i)).transpose().to_dict()) out[int(i)] = unkeyify_2l(get_team_match_data(client, competition, int(i)).transpose().to_dict())
except: except:
pass pass
return out return out
def get_metrics_data_formatted(apikey, competition): def get_metrics_data_formatted(client, competition):
client = pymongo.MongoClient(apikey)
db = client.data_scouting db = client.data_scouting
mdata = db.teamlist mdata = db.teamlist
x=mdata.find_one({"competition":competition}) x=mdata.find_one({"competition":competition})
out = {} out = {}
for i in x: for i in x:
try: try:
out[int(i)] = get_team_metrics_data(apikey, competition, int(i)) out[int(i)] = get_team_metrics_data(client, competition, int(i))
except: except:
pass pass
return out return out
def get_pit_data_formatted(apikey, competition): def get_pit_data_formatted(client, competition):
client = pymongo.MongoClient(apikey)
db = client.data_scouting db = client.data_scouting
mdata = db.teamlist mdata = db.teamlist
x=mdata.find_one({"competition":competition}) x=mdata.find_one({"competition":competition})
out = {} out = {}
for i in x: for i in x:
try: try:
out[int(i)] = get_team_pit_data(apikey, competition, int(i)) out[int(i)] = get_team_pit_data(client, competition, int(i))
except: except:
pass pass
return out return out
def get_pit_variable_data(apikey, competition): def get_pit_variable_data(client, competition):
client = pymongo.MongoClient(apikey)
db = client.data_processing db = client.data_processing
mdata = db.team_pit mdata = db.team_pit
out = {} out = {}
return mdata.find() return mdata.find()
def get_pit_variable_formatted(apikey, competition): def get_pit_variable_formatted(client, competition):
temp = get_pit_variable_data(apikey, competition) temp = get_pit_variable_data(client, competition)
out = {} out = {}
for i in temp: for i in temp:
out[i["variable"]] = i["data"] out[i["variable"]] = i["data"]
return out return out
def push_team_tests_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_tests"): def push_team_tests_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_tests"):
client = pymongo.MongoClient(apikey)
db = client[dbname] db = client[dbname]
mdata = db[colname] mdata = db[colname]
mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "data" : data}, True) mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "data" : data}, True)
def push_team_metrics_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"): def push_team_metrics_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"):
client = pymongo.MongoClient(apikey)
db = client[dbname] db = client[dbname]
mdata = db[colname] mdata = db[colname]
mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True) mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True)
def push_team_pit_data(apikey, competition, variable, data, dbname = "data_processing", colname = "team_pit"): def push_team_pit_data(client, competition, variable, data, dbname = "data_processing", colname = "team_pit"):
client = pymongo.MongoClient(apikey)
db = client[dbname] db = client[dbname]
mdata = db[colname] mdata = db[colname]
mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True) mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True)
def get_analysis_flags(apikey, flag): def get_analysis_flags(client, flag):
client = pymongo.MongoClient(apikey)
db = client.data_processing db = client.data_processing
mdata = db.flags mdata = db.flags
return mdata.find_one({flag:{"$exists":True}}) return mdata.find_one({flag:{"$exists":True}})
def set_analysis_flags(apikey, flag, data): def set_analysis_flags(client, flag, data):
client = pymongo.MongoClient(apikey)
db = client.data_processing db = client.data_processing
mdata = db.flags mdata = db.flags
return mdata.replace_one({flag:{"$exists":True}}, data, True) return mdata.replace_one({flag:{"$exists":True}}, data, True)
@ -158,7 +145,7 @@ def load_metric(apikey, competition, match, group_name, metrics):
db_data = get_team_metrics_data(apikey, competition, team) db_data = get_team_metrics_data(apikey, competition, team)
if get_team_metrics_data(apikey, competition, team) == None: if db_data == None:
elo = {"score": metrics["elo"]["score"]} elo = {"score": metrics["elo"]["score"]}
gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]}

2
src/cli/interface.py
View File

@ -17,7 +17,7 @@ stderr = sys.stderr
def log(target, level, message, code = 0): def log(target, level, message, code = 0):
message = time.ctime() + empty_delim + str(level) + l_brack + f"{code:04}" + r_brack + empty_delim + soft_divided_delim + empty_delim + message message = time.ctime() + empty_delim + str(level) + l_brack + f"{code:+05}" + r_brack + empty_delim + soft_divided_delim + empty_delim + message
print(message, file = target) print(message, file = target)
def clear(): def clear():

12
src/cli/processing.py
View File

@ -36,7 +36,7 @@ def simplestats(data_test):
if test == "regression_sigmoidal": if test == "regression_sigmoidal":
return an.regression(ranges, data, ['sig']) return an.regression(ranges, data, ['sig'])
def matchloop(apikey, competition, data, tests, exec_threads): def matchloop(client, competition, data, tests, exec_threads):
short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"}
@ -88,7 +88,7 @@ def matchloop(apikey, competition, data, tests, exec_threads):
return return_vector return return_vector
def metricloop(tbakey, apikey, competition, timestamp, metrics): # listener based metrics update def metricloop(tbakey, client, competition, timestamp, metrics): # listener based metrics update
elo_N = metrics["elo"]["N"] elo_N = metrics["elo"]["N"]
elo_K = metrics["elo"]["K"] elo_K = metrics["elo"]["K"]
@ -100,8 +100,8 @@ def metricloop(tbakey, apikey, competition, timestamp, metrics): # listener base
for match in matches: for match in matches:
red = load_metric(apikey, competition, match, "red", metrics) red = load_metric(client, competition, match, "red", metrics)
blu = load_metric(apikey, competition, match, "blue", metrics) blu = load_metric(client, competition, match, "blue", metrics)
elo_red_total = 0 elo_red_total = 0
elo_blu_total = 0 elo_blu_total = 0
@ -179,9 +179,9 @@ def metricloop(tbakey, apikey, competition, timestamp, metrics): # listener base
temp_vector.update(red) temp_vector.update(red)
temp_vector.update(blu) temp_vector.update(blu)
push_metric(apikey, competition, temp_vector) push_metric(client, competition, temp_vector)
def pitloop(apikey, competition, pit, tests): def pitloop(client, competition, pit, tests):
return_vector = {} return_vector = {}
for team in pit: for team in pit:

259
src/cli/superscript.py
View File

@ -3,10 +3,23 @@
# Notes: # Notes:
# setup: # setup:
__version__ = "0.9.1" __version__ = "1.0.0"
# changelog should be viewed using print(analysis.__changelog__) # changelog should be viewed using print(analysis.__changelog__)
__changelog__ = """changelog: __changelog__ = """changelog:
1.0.0:
- superscript now runs in PEP 3143 compliant well behaved daemon on Linux systems
- linux superscript daemon has integrated websocket output to monitor progress/status remotely
- linux daemon now sends stderr to errorlog.txt
- added verbose option to linux superscript to allow for interactive output
- moved pymongo import to superscript.py
- added profile option to linux superscript to profile runtime of script
0.9.3:
- improved data loading performance by removing redundant PyMongo client creation (120s to 14s)
- passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions
0.9.2:
- removed unessasary imports from data
- minor changes to interface
0.9.1: 0.9.1:
- fixed bugs in configuration item loading exception handling - fixed bugs in configuration item loading exception handling
0.9.0: 0.9.0:
@ -136,6 +149,10 @@ from multiprocessing import Pool, freeze_support
import time import time
import warnings import warnings
import sys import sys
import asyncio
import websockets
import pymongo
import threading
from interface import splash, log, ERR, INF, stdout, stderr from interface import splash, log, ERR, INF, stdout, stderr
from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit
@ -189,15 +206,17 @@ sample_json = """{
} }
}""" }"""
def main(): def main(send, verbose = False, profile = False):
warnings.filterwarnings("ignore") if verbose or profile:
sys.stderr = open("errorlog.txt", "w")
splash(__version__) warnings.filterwarnings("ignore")
sys.stderr = open("errorlog.txt", "w")
loop_exit_code = 0 splash(__version__)
loop_stored_exception = None
loop_exit_code = 0
loop_stored_exception = None
while True: while True:
@ -206,33 +225,38 @@ def main():
loop_start = time.time() loop_start = time.time()
current_time = time.time() current_time = time.time()
log(stdout, INF, "current time: " + str(current_time)) send(stdout, INF, "current time: " + str(current_time))
send(stdout, INF, "loading config at <" + config_path + ">", code = 0)
config = {} config = {}
if load_config(config_path, config) == 1: if load_config(config_path, config) == 1:
send(stderr, ERR, "could not find config at <" + config_path + ">, generating blank config and exiting", code = 100)
sys.exit(1) sys.exit(1)
send(stdout, INF, "found and opened config at <" + config_path + ">", code = 0)
error_flag = False error_flag = False
try: try:
competition = config["competition"] competition = config["competition"]
except: except:
log(stderr, ERR, "could not find competition field in config", code = 101) send(stderr, ERR, "could not find competition field in config", code = 101)
error_flag = True error_flag = True
try: try:
match_tests = config["statistics"]["match"] match_tests = config["statistics"]["match"]
except: except:
log(stderr, ERR, "could not find match_tests field in config", code = 102) send(stderr, ERR, "could not find match_tests field in config", code = 102)
error_flag = True error_flag = True
try: try:
metrics_tests = config["statistics"]["metric"] metrics_tests = config["statistics"]["metric"]
except: except:
log(stderr, ERR, "could not find metrics_tests field in config", code = 103) send(stderr, ERR, "could not find metrics_tests field in config", code = 103)
error_flag = True error_flag = True
try: try:
pit_tests = config["statistics"]["pit"] pit_tests = config["statistics"]["pit"]
except: except:
log(stderr, ERR, "could not find pit_tests field in config", code = 104) send(stderr, ERR, "could not find pit_tests field in config", code = 104)
error_flag = True error_flag = True
if error_flag: if error_flag:
@ -240,28 +264,28 @@ def main():
error_flag = False error_flag = False
if competition == None or competition == "": if competition == None or competition == "":
log(stderr, ERR, "competition field in config must not be empty", code = 105) send(stderr, ERR, "competition field in config must not be empty", code = 105)
error_flag = True error_flag = True
if match_tests == None: if match_tests == None:
log(stderr, ERR, "match_tests field in config must not be empty", code = 106) send(stderr, ERR, "match_tests field in config must not be empty", code = 106)
error_flag = True error_flag = True
if metrics_tests == None: if metrics_tests == None:
log(stderr, ERR, "metrics_tests field in config must not be empty", code = 107) send(stderr, ERR, "metrics_tests field in config must not be empty", code = 107)
error_flag = True error_flag = True
if pit_tests == None: if pit_tests == None:
log(stderr, ERR, "pit_tests field in config must not be empty", code = 108) send(stderr, ERR, "pit_tests field in config must not be empty", code = 108)
error_flag = True error_flag = True
if error_flag: if error_flag:
sys.exit(1) sys.exit(1)
log(stdout, INF, "found and loaded competition, match_tests, metrics_tests, pit_tests from config") send(stdout, INF, "found and loaded competition, match_tests, metrics_tests, pit_tests from config")
sys_max_threads = os.cpu_count() sys_max_threads = os.cpu_count()
try: try:
cfg_max_threads = config["max-threads"] cfg_max_threads = config["max-threads"]
except: except:
log(stderr, ERR, "max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) send(stderr, ERR, "max-threads field in config must not be empty, refer to documentation for configuration options", code = 109)
sys.exit(1) sys.exit(1)
if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 :
@ -273,101 +297,108 @@ def main():
elif cfg_max_threads == 0: elif cfg_max_threads == 0:
alloc_processes = sys_max_threads alloc_processes = sys_max_threads
else: else:
log(stderr, ERR, "max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, code = 110) send(stderr, ERR, "max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, code = 110)
sys.exit(1) sys.exit(1)
log(stdout, INF, "found and loaded max-threads from config") send(stdout, INF, "found and loaded max-threads from config")
log(stdout, INF, "attempting to start " + str(alloc_processes) + " threads") send(stdout, INF, "attempting to start " + str(alloc_processes) + " threads")
try: try:
exec_threads = Pool(processes = alloc_processes) exec_threads = Pool(processes = alloc_processes)
except Exception as e: except Exception as e:
log(stderr, ERR, "unable to start threads", code = 200) send(stderr, ERR, "unable to start threads", code = 200)
log(stderr, INF, e) send(stderr, INF, e)
sys.exit(1) sys.exit(1)
log(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads")
exit_flag = False exit_flag = False
try: try:
apikey = config["key"]["database"] apikey = config["key"]["database"]
except: except:
log(stderr, ERR, "database key field in config must be present", code = 111) send(stderr, ERR, "database key field in config must be present", code = 111)
exit_flag = True exit_flag = True
try: try:
tbakey = config["key"]["tba"] tbakey = config["key"]["tba"]
except: except:
log(stderr, ERR, "tba key field in config must be present", code = 112) send(stderr, ERR, "tba key field in config must be present", code = 112)
exit_flag = True exit_flag = True
if apikey == None or apikey == "": if apikey == None or apikey == "":
log(stderr, ERR, "database key field in config must not be empty, please populate the database key") send(stderr, ERR, "database key field in config must not be empty, please populate the database key")
exit_flag = True exit_flag = True
if tbakey == None or tbakey == "": if tbakey == None or tbakey == "":
log(stderr, ERR, "tba key field in config must not be empty, please populate the tba key") send(stderr, ERR, "tba key field in config must not be empty, please populate the tba key")
exit_flag = True exit_flag = True
if exit_flag: if exit_flag:
sys.exit(1) sys.exit(1)
log(stdout, INF, "found and loaded database and tba keys") send(stdout, INF, "found and loaded database and tba keys")
previous_time = get_previous_time(apikey) client = pymongo.MongoClient(apikey)
log(stdout, INF, "analysis backtimed to: " + str(previous_time))
previous_time = get_previous_time(client)
send(stdout, INF, "analysis backtimed to: " + str(previous_time))
start = time.time() start = time.time()
log(stdout, INF, "loading match data") send(stdout, INF, "loading match data")
match_data = load_match(apikey, competition) match_data = load_match(client, competition)
log(stdout, INF, "finished loading match data in " + str(time.time() - start) + " seconds") send(stdout, INF, "finished loading match data in " + str(time.time() - start) + " seconds")
start = time.time() start = time.time()
log(stdout, INF, "performing analysis on match data") send(stdout, INF, "performing analysis on match data")
results = matchloop(apikey, competition, match_data, match_tests, exec_threads) results = matchloop(client, competition, match_data, match_tests, exec_threads)
log(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds") send(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds")
start = time.time() start = time.time()
log(stdout, INF, "uploading match results to database") send(stdout, INF, "uploading match results to database")
push_match(apikey, competition, results) push_match(client, competition, results)
log(stdout, INF, "finished uploading match results in " + str(time.time() - start) + " seconds") send(stdout, INF, "finished uploading match results in " + str(time.time() - start) + " seconds")
start = time.time() start = time.time()
log(stdout, INF, "performing analysis on team metrics") send(stdout, INF, "performing analysis on team metrics")
results = metricloop(tbakey, apikey, competition, current_time, metrics_tests) results = metricloop(tbakey, client, competition, current_time, metrics_tests)
log(stdout, INF, "finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") send(stdout, INF, "finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds")
start = time.time() start = time.time()
log(stdout, INF, "loading pit data") send(stdout, INF, "loading pit data")
pit_data = load_pit(apikey, competition) pit_data = load_pit(client, competition)
log(stdout, INF, "finished loading pit data in " + str(time.time() - start) + " seconds") send(stdout, INF, "finished loading pit data in " + str(time.time() - start) + " seconds")
start = time.time() start = time.time()
log(stdout, INF, "performing analysis on pit data") send(stdout, INF, "performing analysis on pit data")
results = pitloop(apikey, competition, pit_data, pit_tests) results = pitloop(client, competition, pit_data, pit_tests)
log(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds") send(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds")
start = time.time() start = time.time()
log(stdout, INF, "uploading pit results to database") send(stdout, INF, "uploading pit results to database")
push_pit(apikey, competition, results) push_pit(client, competition, results)
log(stdout, INF, "finished uploading pit results in " + str(time.time() - start) + " seconds") send(stdout, INF, "finished uploading pit results in " + str(time.time() - start) + " seconds")
set_current_time(apikey, current_time) client.close()
log(stdout, INF, "finished all tests in " + str(time.time() - loop_start) + " seconds, looping")
set_current_time(client, current_time)
send(stdout, INF, "finished all tests in " + str(time.time() - loop_start) + " seconds, looping")
except KeyboardInterrupt: except KeyboardInterrupt:
log(stdout, INF, "detected KeyboardInterrupt, killing threads") send(stdout, INF, "detected KeyboardInterrupt, killing threads")
if "exec_threads" in locals(): if "exec_threads" in locals():
exec_threads.terminate() exec_threads.terminate()
exec_threads.join() exec_threads.join()
exec_threads.close() exec_threads.close()
log(stdout, INF, "terminated threads, exiting") send(stdout, INF, "terminated threads, exiting")
loop_stored_exception = sys.exc_info() loop_stored_exception = sys.exc_info()
loop_exit_code = 0 loop_exit_code = 0
break break
except Exception as e: except Exception as e:
log(stderr, ERR, "encountered an exception while running") send(stderr, ERR, "encountered an exception while running")
print(e, file = stderr) print(e, file = stderr)
loop_exit_code = 1 loop_exit_code = 1
break break
if profile:
return
sys.exit(loop_exit_code) sys.exit(loop_exit_code)
def load_config(path, config_vector): def load_config(path, config_vector):
@ -375,10 +406,8 @@ def load_config(path, config_vector):
f = open(path, "r") f = open(path, "r")
config_vector.update(json.load(f)) config_vector.update(json.load(f))
f.close() f.close()
log(stdout, INF, "found and opened config at <" + path + ">")
return 0 return 0
except: except:
log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting", code = 100)
f = open(path, "w") f = open(path, "w")
f.write(sample_json) f.write(sample_json)
f.close() f.close()
@ -393,7 +422,117 @@ def save_config(path, config_vector):
except: except:
return 1 return 1
def start(pid_path, verbose = False, profile = False):
if profile:
def send(target, level, message, code = 0):
pass
import cProfile, pstats, io
profile = cProfile.Profile()
profile.enable()
main(send, profile = True)
profile.disable()
f = open("profile.txt", 'w+')
ps = pstats.Stats(profile, stream = f).sort_stats('cumtime')
ps.print_stats()
elif verbose:
main(log, verbose = verbose)
else:
f = open('errorlog.txt', 'w+')
with daemon.DaemonContext(
working_directory=os.getcwd(),
pidfile=pidfile.TimeoutPIDLockFile(pid_path),
stderr=f
):
async def handler(client, path):
clients.append(client)
while True:
try:
pong_waiter = await client.ping()
await pong_waiter
time.sleep(3)
except Exception as e:
clients.remove(client)
break
async def send_one(client, data):
await client.send(data)
def send(target, level, message, code = 0):
message_clients = clients.copy()
for client in message_clients:
try:
asyncio.run(send_one(client, message))
except:
pass
clients = []
start_server = websockets.serve(handler, "0.0.0.0", 5678)
asyncio.get_event_loop().run_until_complete(start_server)
threading.Thread(target = asyncio.get_event_loop().run_forever).start()
main(send)
def stop(pid_path):
try:
pf = open(pid_path, 'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
sys.stderr.write("pidfile at <" + pid_path + "> does not exist. Daemon not running?\n")
return
try:
while True:
os.kill(pid, SIGTERM)
time.sleep(0.01)
except OSError as err:
err = str(err)
if err.find("No such process") > 0:
if os.path.exists(pid_path):
os.remove(pid_path)
else:
print(str(err))
sys.exit(1)
def restart(pid_path):
stop(pid_path)
start(pid_path)
if __name__ == "__main__": if __name__ == "__main__":
if sys.platform.startswith("win"): if sys.platform.startswith("win"):
freeze_support() freeze_support()
main() start(None, verbose = True)
else:
import daemon
from daemon import pidfile
from signal import SIGTERM
pid_path = "tra-daemon.pid"
if len(sys.argv) == 2:
if 'start' == sys.argv[1]:
start(pid_path)
elif 'stop' == sys.argv[1]:
stop(pid_path)
elif 'restart' == sys.argv[1]:
restart(pid_path)
elif 'verbose' == sys.argv[1]:
start(None, verbose = True)
elif 'profile' == sys.argv[1]:
start(None, profile=True)
else:
print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0])
sys.exit(2)
sys.exit(0)
else:
print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0])
sys.exit(2)

2
src/cli/superscript.spec
View File

@ -10,6 +10,8 @@ a = Analysis(['superscript.py'],
"dnspython", "dnspython",
"sklearn.utils._weight_vector", "sklearn.utils._weight_vector",
"requests", "requests",
"websockets.legacy",
"websockets.legacy.server",
], ],
hookspath=[], hookspath=[],
runtime_hooks=[], runtime_hooks=[],