From 8112effbce5d028c3a1595a2db9ffe8ae8090fad Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 21 Sep 2021 04:43:32 +0000 Subject: [PATCH 1/2] added debug option, improved memory usage slightly, changed errorlog.txt to errorlog.log, updated .gitignore Former-commit-id: bd51efc6c2248edd987a9e5fc45113fa39af1c46 --- .gitignore | 1 + src/cli/processing.py | 27 ++++++++++----------------- src/cli/superscript.py | 26 ++++++++++++++++++++++---- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 1573f33..0c62b1f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ **/profile.* +**/*.log **/errorlog.txt /dist/superscript.* /dist/superscript \ No newline at end of file diff --git a/src/cli/processing.py b/src/cli/processing.py index fe028e5..73b1224 100644 --- a/src/cli/processing.py +++ b/src/cli/processing.py @@ -9,11 +9,11 @@ def simplestats(data_test): signal.signal(signal.SIGINT, signal.SIG_IGN) - data = np.array(data_test[0]) + data = np.array(data_test[3]) data = data[np.isfinite(data)] ranges = list(range(len(data))) - test = data_test[1] + test = data_test[2] if test == "basic_stats": return an.basic_stats(data) @@ -48,13 +48,7 @@ def matchloop(client, competition, data, tests, exec_threads): value = self[item] = type(self)() return value - return_vector = {} - - team_filtered = [] - variable_filtered = [] - variable_data = [] - test_filtered = [] - result_filtered = [] + input_vector = [] return_vector = AutoVivification() for team in data: @@ -65,25 +59,24 @@ def matchloop(client, competition, data, tests, exec_threads): for test in tests[variable]: - team_filtered.append(team) - variable_filtered.append(variable) - variable_data.append((data[team][variable], test)) - test_filtered.append(test) + input_vector.append((team, variable, test, data[team][variable])) - result_filtered = exec_threads.map(simplestats, variable_data) + result_filtered = exec_threads.map(simplestats, input_vector) + i = 0 result_filtered = list(result_filtered) for result in result_filtered: - filtered = test_filtered[i] + filtered = input_vector[i][2] try: short = short_mapping[filtered] - return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result[short] + return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result[short] except KeyError: # not in mapping - return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result + return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result + i += 1 return return_vector diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 91d11b5..ef15062 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -10,10 +10,12 @@ __changelog__ = """changelog: 1.0.0: - superscript now runs in PEP 3143 compliant well behaved daemon on Linux systems - linux superscript daemon has integrated websocket output to monitor progress/status remotely - - linux daemon now sends stderr to errorlog.txt + - linux daemon now sends stderr to errorlog.log - added verbose option to linux superscript to allow for interactive output - moved pymongo import to superscript.py - added profile option to linux superscript to profile runtime of script + - reduced memory usage slightly by consolidating the unwrapped input data + - added debug option, which performs one loop of analysis and dumps results to local files 0.9.3: - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions @@ -206,10 +208,10 @@ sample_json = """{ } }""" -def main(send, verbose = False, profile = False): +def main(send, verbose = False, profile = False, debug = False): warnings.filterwarnings("ignore") - sys.stderr = open("errorlog.txt", "w") + sys.stderr = open("errorlog.log", "w") loop_exit_code = 0 loop_stored_exception = None @@ -348,6 +350,11 @@ def main(send, verbose = False, profile = False): results = matchloop(client, competition, match_data, match_tests, exec_threads) send(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds") + if debug: + f = open("matchloop.log", "w+") + json.dump(results, f) + f.close() + start = time.time() send(stdout, INF, "uploading match results to database") push_match(client, competition, results) @@ -368,6 +375,11 @@ def main(send, verbose = False, profile = False): results = pitloop(client, competition, pit_data, pit_tests) send(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds") + if debug: + f = open("pitloop.log", "w+") + json.dump(results, f) + f.close() + start = time.time() send(stdout, INF, "uploading pit results to database") push_pit(client, competition, results) @@ -420,7 +432,7 @@ def save_config(path, config_vector): except: return 1 -def start(pid_path, verbose = False, profile = False): +def start(pid_path, verbose = False, profile = False, debug = False): if profile: @@ -440,6 +452,10 @@ def start(pid_path, verbose = False, profile = False): main(log, verbose = verbose) + elif debug: + + main(log, verbose = True, profile = True, debug = debug) + else: f = open('errorlog.txt', 'w+') @@ -527,6 +543,8 @@ if __name__ == "__main__": start(None, verbose = True) elif 'profile' == sys.argv[1]: start(None, profile=True) + elif 'debug' == sys.argv[1]: + start(None, debug = True) else: print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0]) sys.exit(2) From 16882a0a7549496c3cb1836a6fd13ea4b15c8ccd Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 21 Sep 2021 23:36:13 +0000 Subject: [PATCH 2/2] added debug option to usage description, set generic error code to 1 Former-commit-id: 9163244e2cb4f2d198d7a0bfa9f15ba32aebc627 --- src/cli/superscript.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index ef15062..6f00928 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -401,7 +401,7 @@ def main(send, verbose = False, profile = False, debug = False): loop_exit_code = 0 break except Exception as e: - send(stderr, ERR, "encountered an exception while running") + send(stderr, ERR, "encountered an exception while running", code = 1) print(e, file = stderr) loop_exit_code = 1 break @@ -546,9 +546,9 @@ if __name__ == "__main__": elif 'debug' == sys.argv[1]: start(None, debug = True) else: - print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0]) + print("usage: %s start|stop|restart|verbose|profile|debug" % sys.argv[0]) sys.exit(2) sys.exit(0) else: - print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0]) + print("usage: %s start|stop|restart|verbose|profile|debug" % sys.argv[0]) sys.exit(2) \ No newline at end of file