diff --git a/data-analysis/config.json b/data-analysis/config.json index c146d833..f143132b 100644 --- a/data-analysis/config.json +++ b/data-analysis/config.json @@ -1,7 +1,7 @@ { - "max-threads": 1, + "max-threads": 0.5, "team": "", - "competition": "2020ilch", + "competition": "", "key":{ "database":"", "tba":"" diff --git a/data-analysis/superscript.py b/data-analysis/superscript.py index a73b035d..a3f04095 100644 --- a/data-analysis/superscript.py +++ b/data-analysis/superscript.py @@ -3,10 +3,13 @@ # Notes: # setup: -__version__ = "0.8.1" +__version__ = "0.8.2" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 0.8.2: + - readded while true to main function + - added more thread config options 0.8.1: - optimized matchloop further by bypassing GIL 0.8.0: @@ -121,7 +124,9 @@ from tra_analysis import analysis as an import data as d from collections import defaultdict import json +import math import numpy as np +import os from os import system, name from pathlib import Path from multiprocessing import Pool @@ -138,54 +143,67 @@ def main(): warnings.filterwarnings("ignore") - # while (True): + while (True): - current_time = time.time() - print("[OK] time: " + str(current_time)) + current_time = time.time() + print("[OK] time: " + str(current_time)) - config = load_config("config.json") - competition = config["competition"] - match_tests = config["statistics"]["match"] - pit_tests = config["statistics"]["pit"] - metrics_tests = config["statistics"]["metric"] - print("[OK] configs loaded") + config = load_config("config.json") + competition = config["competition"] + match_tests = config["statistics"]["match"] + pit_tests = config["statistics"]["pit"] + metrics_tests = config["statistics"]["metric"] + print("[OK] configs loaded") - print("[OK] starting threads") - exec_threads = Pool(processes = config["max-threads"]) - print("[OK] threads started") + print("[OK] starting threads") + cfg_max_threads = config["max-threads"] + sys_max_threads = os.cpu_count() + if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : + alloc_processes = sys_max_threads + cfg_max_threads + elif cfg_max_threads > 0 and cfg_max_threads < 1: + alloc_processes = math.floor(cfg_max_threads * sys_max_threads) + elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: + alloc_processes = cfg_max_threads + elif cfg_max_threads == 0: + alloc_processes = sys_max_threads + else: + print("[Err] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads)) + exit() + exec_threads = Pool(processes = alloc_processes) + print("[OK] " + str(alloc_processes) + " threads started") - apikey = config["key"]["database"] - tbakey = config["key"]["tba"] - print("[OK] loaded keys") + apikey = config["key"]["database"] + tbakey = config["key"]["tba"] + print("[OK] loaded keys") - previous_time = get_previous_time(apikey) - print("[OK] analysis backtimed to: " + str(previous_time)) + previous_time = get_previous_time(apikey) + print("[OK] analysis backtimed to: " + str(previous_time)) - print("[OK] loading data") - start = time.time() - match_data = load_match(apikey, competition) - pit_data = load_pit(apikey, competition) - print("[OK] loaded data in " + str(time.time() - start) + " seconds") + print("[OK] loading data") + start = time.time() + match_data = load_match(apikey, competition) + pit_data = load_pit(apikey, competition) + print("[OK] loaded data in " + str(time.time() - start) + " seconds") - print("[OK] running match stats") - start = time.time() - matchloop(apikey, competition, match_data, match_tests) - print("[OK] finished match stats in " + str(time.time() - start) + " seconds") + print("[OK] running match stats") + start = time.time() + matchloop(apikey, competition, match_data, match_tests) + print("[OK] finished match stats in " + str(time.time() - start) + " seconds") - print("[OK] running team metrics") - start = time.time() - metricloop(tbakey, apikey, competition, previous_time, metrics_tests) - print("[OK] finished team metrics in " + str(time.time() - start) + " seconds") + print("[OK] running team metrics") + start = time.time() + metricloop(tbakey, apikey, competition, previous_time, metrics_tests) + print("[OK] finished team metrics in " + str(time.time() - start) + " seconds") - print("[OK] running pit analysis") - start = time.time() - pitloop(apikey, competition, pit_data, pit_tests) - print("[OK] finished pit analysis in " + str(time.time() - start) + " seconds") - - set_current_time(apikey, current_time) - print("[OK] finished all tests, looping") + print("[OK] running pit analysis") + start = time.time() + pitloop(apikey, competition, pit_data, pit_tests) + print("[OK] finished pit analysis in " + str(time.time() - start) + " seconds") + + set_current_time(apikey, current_time) + print("[OK] finished all tests, looping") - #clear() + clear() def clear():