added debug option,

improved memory usage slightly,
changed errorlog.txt to errorlog.log,
updated .gitignore


Former-commit-id: bd51efc6c2
This commit is contained in:
Arthur Lu 2021-09-21 04:43:32 +00:00
parent 9be9008ae1
commit 8112effbce
3 changed files with 33 additions and 21 deletions

1
.gitignore vendored
View File

@ -13,6 +13,7 @@
**/profile.* **/profile.*
**/*.log
**/errorlog.txt **/errorlog.txt
/dist/superscript.* /dist/superscript.*
/dist/superscript /dist/superscript

View File

@ -9,11 +9,11 @@ def simplestats(data_test):
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
data = np.array(data_test[0]) data = np.array(data_test[3])
data = data[np.isfinite(data)] data = data[np.isfinite(data)]
ranges = list(range(len(data))) ranges = list(range(len(data)))
test = data_test[1] test = data_test[2]
if test == "basic_stats": if test == "basic_stats":
return an.basic_stats(data) return an.basic_stats(data)
@ -48,13 +48,7 @@ def matchloop(client, competition, data, tests, exec_threads):
value = self[item] = type(self)() value = self[item] = type(self)()
return value return value
return_vector = {} input_vector = []
team_filtered = []
variable_filtered = []
variable_data = []
test_filtered = []
result_filtered = []
return_vector = AutoVivification() return_vector = AutoVivification()
for team in data: for team in data:
@ -65,25 +59,24 @@ def matchloop(client, competition, data, tests, exec_threads):
for test in tests[variable]: for test in tests[variable]:
team_filtered.append(team) input_vector.append((team, variable, test, data[team][variable]))
variable_filtered.append(variable)
variable_data.append((data[team][variable], test))
test_filtered.append(test)
result_filtered = exec_threads.map(simplestats, variable_data) result_filtered = exec_threads.map(simplestats, input_vector)
i = 0 i = 0
result_filtered = list(result_filtered) result_filtered = list(result_filtered)
for result in result_filtered: for result in result_filtered:
filtered = test_filtered[i] filtered = input_vector[i][2]
try: try:
short = short_mapping[filtered] short = short_mapping[filtered]
return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result[short] return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result[short]
except KeyError: # not in mapping except KeyError: # not in mapping
return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result
i += 1 i += 1
return return_vector return return_vector

View File

@ -10,10 +10,12 @@ __changelog__ = """changelog:
1.0.0: 1.0.0:
- superscript now runs in PEP 3143 compliant well behaved daemon on Linux systems - superscript now runs in PEP 3143 compliant well behaved daemon on Linux systems
- linux superscript daemon has integrated websocket output to monitor progress/status remotely - linux superscript daemon has integrated websocket output to monitor progress/status remotely
- linux daemon now sends stderr to errorlog.txt - linux daemon now sends stderr to errorlog.log
- added verbose option to linux superscript to allow for interactive output - added verbose option to linux superscript to allow for interactive output
- moved pymongo import to superscript.py - moved pymongo import to superscript.py
- added profile option to linux superscript to profile runtime of script - added profile option to linux superscript to profile runtime of script
- reduced memory usage slightly by consolidating the unwrapped input data
- added debug option, which performs one loop of analysis and dumps results to local files
0.9.3: 0.9.3:
- improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - improved data loading performance by removing redundant PyMongo client creation (120s to 14s)
- passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions
@ -206,10 +208,10 @@ sample_json = """{
} }
}""" }"""
def main(send, verbose = False, profile = False): def main(send, verbose = False, profile = False, debug = False):
warnings.filterwarnings("ignore") warnings.filterwarnings("ignore")
sys.stderr = open("errorlog.txt", "w") sys.stderr = open("errorlog.log", "w")
loop_exit_code = 0 loop_exit_code = 0
loop_stored_exception = None loop_stored_exception = None
@ -348,6 +350,11 @@ def main(send, verbose = False, profile = False):
results = matchloop(client, competition, match_data, match_tests, exec_threads) results = matchloop(client, competition, match_data, match_tests, exec_threads)
send(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds") send(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds")
if debug:
f = open("matchloop.log", "w+")
json.dump(results, f)
f.close()
start = time.time() start = time.time()
send(stdout, INF, "uploading match results to database") send(stdout, INF, "uploading match results to database")
push_match(client, competition, results) push_match(client, competition, results)
@ -368,6 +375,11 @@ def main(send, verbose = False, profile = False):
results = pitloop(client, competition, pit_data, pit_tests) results = pitloop(client, competition, pit_data, pit_tests)
send(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds") send(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds")
if debug:
f = open("pitloop.log", "w+")
json.dump(results, f)
f.close()
start = time.time() start = time.time()
send(stdout, INF, "uploading pit results to database") send(stdout, INF, "uploading pit results to database")
push_pit(client, competition, results) push_pit(client, competition, results)
@ -420,7 +432,7 @@ def save_config(path, config_vector):
except: except:
return 1 return 1
def start(pid_path, verbose = False, profile = False): def start(pid_path, verbose = False, profile = False, debug = False):
if profile: if profile:
@ -440,6 +452,10 @@ def start(pid_path, verbose = False, profile = False):
main(log, verbose = verbose) main(log, verbose = verbose)
elif debug:
main(log, verbose = True, profile = True, debug = debug)
else: else:
f = open('errorlog.txt', 'w+') f = open('errorlog.txt', 'w+')
@ -527,6 +543,8 @@ if __name__ == "__main__":
start(None, verbose = True) start(None, verbose = True)
elif 'profile' == sys.argv[1]: elif 'profile' == sys.argv[1]:
start(None, profile=True) start(None, profile=True)
elif 'debug' == sys.argv[1]:
start(None, debug = True)
else: else:
print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0]) print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0])
sys.exit(2) sys.exit(2)