superscript.py v 0.8.0

Signed-off-by: Arthur Lu <learthurgo@gmail.com>

data-analysis/superscript.py

@@ -3,10 +3,13 @@
 # Notes:
 # setup:
 
-__version__ = "0.7.0"
+__version__ = "0.8.0"
 
 # changelog should be viewed using print(analysis.__changelog__)
 __changelog__ = """changelog:
+	0.8.0:
+		- added multithreading to matchloop
+		- tweaked user log
 	0.7.0:
 		- finished implementing main function
 	0.6.2:
@@ -114,62 +117,72 @@ __all__ = [
 
 from tra_analysis import analysis as an
 import data as d
+from collections import defaultdict
 import json
 import numpy as np
 from os import system, name
 from pathlib import Path
 import matplotlib.pyplot as plt
+from concurrent.futures import ThreadPoolExecutor
 import time
 import warnings
 
+global exec_threads
+
 def main():
 
+	global exec_threads
+
 	warnings.filterwarnings("ignore")
 
-	while (True):
+	# while (True):
 
-		current_time = time.time()
+	current_time = time.time()
-		print("[OK] time: " + str(current_time))
+	print("[OK] time: " + str(current_time))
 
-		config = load_config("config.json")
+	config = load_config("config.json")
-		competition = config["competition"]
+	competition = config["competition"]
-		match_tests = config["statistics"]["match"]
+	match_tests = config["statistics"]["match"]
-		pit_tests = config["statistics"]["pit"]
+	pit_tests = config["statistics"]["pit"]
-		metrics_tests = config["statistics"]["metric"]
+	metrics_tests = config["statistics"]["metric"]
-		print("[OK] configs loaded")
+	print("[OK] configs loaded")
 
-		apikey = config["key"]["database"]
+	print("[OK] starting threads")
-		tbakey = config["key"]["tba"]
+	exec_threads = ThreadPoolExecutor(max_workers = config["max-threads"])
-		print("[OK] loaded keys")
+	print("[OK] threads started")
 
-		previous_time = get_previous_time(apikey)
+	apikey = config["key"]["database"]
-		print("[OK] analysis backtimed to: " + str(previous_time))
+	tbakey = config["key"]["tba"]
+	print("[OK] loaded keys")
 
-		print("[OK] loading data")
+	previous_time = get_previous_time(apikey)
-		start = time.time()
+	print("[OK] analysis backtimed to: " + str(previous_time))
-		match_data = load_match(apikey, competition)
-		pit_data = load_pit(apikey, competition)
-		print("[OK] loaded data in " + str(time.time() - start) + " seconds")
 
-		print("[OK] running tests")
+	print("[OK] loading data")
-		start = time.time()
+	start = time.time()
-		matchloop(apikey, competition, match_data, match_tests)
+	match_data = load_match(apikey, competition)
-		print("[OK] finished tests in " + str(time.time() - start) + " seconds")
+	pit_data = load_pit(apikey, competition)
+	print("[OK] loaded data in " + str(time.time() - start) + " seconds")
 
-		print("[OK] running metrics")
+	print("[OK] running match stats")
-		start = time.time()
+	start = time.time()
-		metricloop(tbakey, apikey, competition, previous_time, metrics_tests)
+	matchloop(apikey, competition, match_data, match_tests)
-		print("[OK] finished metrics in " + str(time.time() - start) + " seconds")
+	print("[OK] finished match stats in " + str(time.time() - start) + " seconds")
 
-		print("[OK] running pit analysis")
+	print("[OK] running team metrics")
-		start = time.time()
+	start = time.time()
-		pitloop(apikey, competition, pit_data, pit_tests)
+	metricloop(tbakey, apikey, competition, previous_time, metrics_tests)
-		print("[OK] finished pit analysis in " + str(time.time() - start) + " seconds")
+	print("[OK] finished team metrics in " + str(time.time() - start) + " seconds")
-		
-		set_current_time(apikey, current_time)
-		print("[OK] finished all tests, looping")
 
-		clear()
+	print("[OK] running pit analysis")
+	start = time.time()
+	pitloop(apikey, competition, pit_data, pit_tests)
+	print("[OK] finished pit analysis in " + str(time.time() - start) + " seconds")
+	
+	set_current_time(apikey, current_time)
+	print("[OK] finished all tests, looping")
+
+	#clear()
 
 def clear(): 
 	
@@ -219,12 +232,18 @@ def load_match(apikey, competition):
 
 def matchloop(apikey, competition, data, tests): # expects 3D array with [Team][Variable][Match]
 
-	def simplestats(data, test):
+	start = time.time()
 
-		data = np.array(data)
+	global exec_threads
+
+	def simplestats(data_test):
+
+		data = np.array(data_test[0])
 		data = data[np.isfinite(data)]
 		ranges = list(range(len(data)))
 
+		test = data_test[1]
+
 		if test == "basic_stats":
 			return an.basic_stats(data)
 
@@ -246,19 +265,47 @@ def matchloop(apikey, competition, data, tests): # expects 3D array with [Team][
 		if test == "regression_sigmoidal":
 			return an.regression(ranges, data, ['sig'])
 
+	class AutoVivification(dict):
+		def __getitem__(self, item):
+			try:
+				return dict.__getitem__(self, item)
+			except KeyError:
+				value = self[item] = type(self)()
+				return value
+
 	return_vector = {}
+	
+	team_filtered = []
+	variable_filtered = []
+	variable_data = []
+	test_filtered = []
+	result_filtered = []
+	return_vector = AutoVivification()
+
 	for team in data:
-		variable_vector = {}
+
 		for variable in data[team]:
-			test_vector = {}
+
-			variable_data = data[team][variable]
 			if variable in tests:
+
 				for test in tests[variable]:
-					test_vector[test] = simplestats(variable_data, test)
+
-			else:
+					team_filtered.append(team)
-				pass      
+					variable_filtered.append(variable)
-			variable_vector[variable] = test_vector
+					variable_data.append((data[team][variable], test))
-		return_vector[team] = variable_vector
+					test_filtered.append(test)
+
+	result_filtered = exec_threads.map(simplestats, variable_data)
+	i = 0
+
+	result_filtered = list(result_filtered)
+
+	for result in result_filtered:
+
+		return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result
+		i += 1
+
+	print("metrics finished in " + str(time.time() - start))
 
 	push_match(apikey, competition, return_vector)