moved config functions into Configuration class,

simplified exception class by removing error codes,
removed exec_threads from module parameters


Former-commit-id: 545ef765653970e1cdebac692eccd227effb2508

src/config.py

@@ -1,17 +1,16 @@
-import math
 import json
-from multiprocessing import Pool
-import os
-from cerberus import Validator
 from exceptions import ConfigurationError
+from cerberus import Validator
 
 from data import set_database_config, get_database_config
 from interface import stderr, stdout, INF, ERR
 
-config_path = "config.json"
+class Configuration:
 
-sample_json = """
+	path = None
-{
+	config = {}
+
+	_sample_config = {
 		"persistent":{
 			"key":{
 				"database":"",
@@ -23,17 +22,12 @@ sample_json = """
 				}
 			},
 			"config-preference":"local",
-      "synchronize-config":false
+			"synchronize-config":False
 		},
 		"variable":{
-      "max-threads":0.5,
+			"event-delay":False,
-      "team":"",
-      "event-delay":false,
 			"loop-delay":0,
-      "reportable":true,
+			"competition": "2020ilch",
-      "teams":[
-         
-      ],
 			"modules":{
 				"match":{
 					"tests":{
@@ -122,123 +116,137 @@ sample_json = """
 				},
 				"pit":{
 					"tests":{
-               "wheel-mechanism":true,
+						"wheel-mechanism":True,
-               "low-balls":true,
+						"low-balls":True,
-               "high-balls":true,
+						"high-balls":True,
-               "wheel-success":true,
+						"wheel-success":True,
-               "strategic-focus":true,
+						"strategic-focus":True,
-               "climb-mechanism":true,
+						"climb-mechanism":True,
-               "attitude":true
+						"attitude":True
+					}
 				}
 			}
 		}
 	}
-}
-"""
 
-def parse_config_persistent(send, config):
+	_validation_schema = {
-	v = Validator(load_validation_schema(), allow_unknown = True)
+		"persistent": {
-	isValidated = v.validate(config)
+			"type": "dict",
+			"required": True,
+			"require_all": True,
+			"schema": {
+				"key": {
+					"type": "dict",
+					"require_all":True,
+					"schema": {
+						"database": {"type":"string"},
+						"tba": {"type": "string"},
+						"tra": {
+							"type": "dict",
+							"require_all": True,
+							"schema": {
+								"CLIENT_ID": {"type": "string"},
+								"CLIENT_SECRET": {"type": "string"},
+								"url": {"type": "string"}
+							}
+						}
+					}
+				},
+				"config-preference": {"type": "string", "required": True},
+				"synchronize-config": {"type": "boolean", "required": True}
+			}
+		}
+	}	
+
+	def __init__(self, path):
+		self.path = path
+		self.load_config()
+		self.validate_config()
+
+	def load_config(self):
+		try:
+			f = open(self.path, "r")
+			self.config.update(json.load(f))
+			f.close()
+		except:
+			self.config = self._sample_config
+			self.save_config()
+			f.close()
+			raise ConfigurationError("could not find config file at <" + self.path + ">, created new sample config file at that path")
+
+	def save_config(self):
+		f = open(self.path, "w+")
+		json.dump(self.config, f, ensure_ascii=False, indent=4)
+		f.close()
+
+	def validate_config(self):
+		v = Validator(self._validation_schema, allow_unknown = True)
+		isValidated = v.validate(self.config)
 
 		if not isValidated:
-		raise ConfigurationError(v.errors, 101)
+			raise ConfigurationError("config validation error: " + v.errors)
 
-	apikey = config["persistent"]["key"]["database"]
+	def __getattr__(self, name): # simple linear lookup method for common multikey-value paths, TYPE UNSAFE
-	tbakey = config["persistent"]["key"]["tba"]
+		if name == "persistent":
-	preference = config["persistent"]["config-preference"]
+			return self.config["persistent"]
-	sync = config["persistent"]["synchronize-config"]
+		elif name == "key":
-
+			return self.config["persistent"]["key"]
-	return apikey, tbakey, preference, sync
+		elif name == "database":
-
+			# soon to be deprecated
-def parse_config_variable(send, config):
+			return self.config["persistent"]["key"]["database"]
-
+		elif name == "tba":
-	sys_max_threads = os.cpu_count()
+			return self.config["persistent"]["key"]["tba"]
-	try:
+		elif name == "tra":
-		cfg_max_threads = config["variable"]["max-threads"]
+			return self.config["persistent"]["key"]["tra"]
-	except:
+		elif name == "priority":
-		raise ConfigurationError("variable/max-threads field is invalid or missing, refer to documentation for configuration options", 109)
+			return self.config["persistent"]["config-preference"]
-	if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 :
+		elif name == "sync":
-		alloc_processes = sys_max_threads + cfg_max_threads
+			return self.config["persistent"]["synchronize-config"]
-	elif cfg_max_threads > 0 and cfg_max_threads < 1:
+		elif name == "variable":
-		alloc_processes = math.floor(cfg_max_threads * sys_max_threads)
+			return self.config["variable"]
-	elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads:
+		elif name == "event_delay":
-		alloc_processes = cfg_max_threads
+			return self.config["variable"]["event-delay"]
-	elif cfg_max_threads == 0:
+		elif name == "loop_delay":
-		alloc_processes = sys_max_threads
+			return self.config["variable"]["loop-delay"]
+		elif name == "competition":
+			return self.config["variable"]["competition"]
+		elif name == "modules":
+			return self.config["variable"]["modules"]
 		else:
-		raise ConfigurationError("variable/max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, 110)
+			return None
-	try:
-		exec_threads = Pool(processes = alloc_processes)
-	except Exception as e:
-		send(stderr, INF, e)
-		raise ConfigurationError("unable to start threads", 200)
-	send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads")
 
-	try:
+	def __getitem__(self, key):
-		modules = config["variable"]["modules"]
+		return self.config[key]
-	except:
-		raise ConfigurationError("variable/modules field is invalid or missing", 102)
 
-	if modules == None:
+	def resolve_config_conflicts(self, send, client): # needs improvement with new localization scheme
-		raise ConfigurationError("variable/modules field is empty", 106)
+		sync = self.sync
-
+		priority = self.priority
-	send(stdout, INF, "found and loaded competition, match, metrics, pit from config")
-
-	return exec_threads, modules
-
-def resolve_config_conflicts(send, client, config, preference, sync):
 
 		if sync:
-		if preference == "local" or preference == "client":
+			if priority == "local" or priority == "client":
 				send(stdout, INF, "config-preference set to local/client, loading local config information")
 				remote_config = get_database_config(client)
-			if remote_config != config["variable"]:
+				if remote_config != self.config["variable"]:
-				set_database_config(client, config["variable"])
+					set_database_config(client, self.config["variable"])
 					send(stdout, INF, "database config was different and was updated")
-			return config
+				# no change to config
-		elif preference == "remote" or preference == "database":
+			elif priority == "remote" or priority == "database":
 				send(stdout, INF, "config-preference set to remote/database, loading remote config information")
-			remote_config= get_database_config(client)
+				remote_config = get_database_config(client)
-			if remote_config != config["variable"]:
+				if remote_config != self.config["variable"]:
-				config["variable"] = remote_config
+					self.config["variable"] = remote_config
-				if save_config(config_path, config):
+					self.save_config()
-					raise ConfigurationError("local config was different but could not be updated", 121)
+					# change variable to match remote
 					send(stdout, INF, "local config was different and was updated")
-			return config
 			else:
-			raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120)
+				raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"")
 		else:
-		if preference == "local" or preference == "client":
+			if priority == "local" or priority == "client":
 				send(stdout, INF, "config-preference set to local/client, loading local config information")
-			return config
+				# no change to config
-		elif preference == "remote" or preference == "database":
+			elif priority == "remote" or priority == "database":
 				send(stdout, INF, "config-preference set to remote/database, loading database config information")
-			config["variable"] = get_database_config(client)
+				self.config["variable"] = get_database_config(client)
-			return config
+				# change variable to match remote without updating local version
 			else:
-			raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120)
+				raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"")
-
-def load_config(path, config_vector):
-	try:
-		f = open(path, "r")
-		config_vector.update(json.load(f))
-		f.close()
-		return 0
-	except:
-		f = open(path, "w")
-		f.write(sample_json)
-		f.close()
-		return 1
-
-def load_validation_schema():
-	try:
-		with open("validation-schema.json", "r") as f:
-			return json.load(f)
-	except:
-		raise FileNotFoundError("Validation schema not found at validation-schema.json")
-
-def save_config(path, config_vector):
-	f = open(path, "w+")
-	json.dump(config_vector, f, ensure_ascii=False, indent=4)
-	f.close()
-	return 0

src/dep.py

@@ -0,0 +1,141 @@
+# contains deprecated functions, not to be used unless nessasary!
+
+import json
+
+sample_json = """
+{
+   "persistent":{
+      "key":{
+         "database":"",
+         "tba":"",
+         "tra":{
+            "CLIENT_ID":"",
+            "CLIENT_SECRET":"",
+			"url": ""
+         }
+      },
+      "config-preference":"local",
+      "synchronize-config":false
+   },
+   "variable":{
+      "max-threads":0.5,
+      "team":"",
+      "event-delay":false,
+      "loop-delay":0,
+      "reportable":true,
+      "teams":[
+         
+      ],
+      "modules":{
+         "match":{
+            "tests":{
+               "balls-blocked":[
+                  "basic_stats",
+                  "historical_analysis",
+                  "regression_linear",
+                  "regression_logarithmic",
+                  "regression_exponential",
+                  "regression_polynomial",
+                  "regression_sigmoidal"
+               ],
+               "balls-collected":[
+                  "basic_stats",
+                  "historical_analysis",
+                  "regression_linear",
+                  "regression_logarithmic",
+                  "regression_exponential",
+                  "regression_polynomial",
+                  "regression_sigmoidal"
+               ],
+               "balls-lower-teleop":[
+                  "basic_stats",
+                  "historical_analysis",
+                  "regression_linear",
+                  "regression_logarithmic",
+                  "regression_exponential",
+                  "regression_polynomial",
+                  "regression_sigmoidal"
+               ],
+               "balls-lower-auto":[
+                  "basic_stats",
+                  "historical_analysis",
+                  "regression_linear",
+                  "regression_logarithmic",
+                  "regression_exponential",
+                  "regression_polynomial",
+                  "regression_sigmoidal"
+               ],
+               "balls-started":[
+                  "basic_stats",
+                  "historical_analyss",
+                  "regression_linear",
+                  "regression_logarithmic",
+                  "regression_exponential",
+                  "regression_polynomial",
+                  "regression_sigmoidal"
+               ],
+               "balls-upper-teleop":[
+                  "basic_stats",
+                  "historical_analysis",
+                  "regression_linear",
+                  "regression_logarithmic",
+                  "regression_exponential",
+                  "regression_polynomial",
+                  "regression_sigmoidal"
+               ],
+               "balls-upper-auto":[
+                  "basic_stats",
+                  "historical_analysis",
+                  "regression_linear",
+                  "regression_logarithmic",
+                  "regression_exponential",
+                  "regression_polynomial",
+                  "regression_sigmoidal"
+               ]
+            }
+         },
+         "metric":{
+            "tests":{
+               "elo":{
+                  "score":1500,
+                  "N":400,
+                  "K":24
+               },
+               "gl2":{
+                  "score":1500,
+                  "rd":250,
+                  "vol":0.06
+               },
+               "ts":{
+                  "mu":25,
+                  "sigma":8.33
+               }
+            }
+         },
+         "pit":{
+            "tests":{
+               "wheel-mechanism":true,
+               "low-balls":true,
+               "high-balls":true,
+               "wheel-success":true,
+               "strategic-focus":true,
+               "climb-mechanism":true,
+               "attitude":true
+            }
+         }
+      }
+   }
+}
+"""
+
+def load_config(path, config_vector):
+	try:
+		f = open(path, "r")
+		config_vector.update(json.load(f))
+		f.close()
+		return 0
+	except:
+		f = open(path, "w")
+		f.write(sample_json)
+		f.close()
+		return 1

src/exceptions.py

@@ -1,11 +1,7 @@
 class APIError(Exception):
-	code = None
+	def __init__(self, str):
-	def __init__(self, str, endpoint):
 		super().__init__(str)
-		self.endpoint = endpoint
 
 class ConfigurationError (Exception):
-	code = None
+	def __init__(self, str):
-	def __init__(self, str, code):
 		super().__init__(str)
-		self.code = code

src/module.py

@@ -22,7 +22,7 @@ class Module(metaclass = abc.ABCMeta):
 	def validate_config(self, *args, **kwargs):
 		raise NotImplementedError
 	@abc.abstractmethod
-	def run(self, exec_threads, *args, **kwargs):
+	def run(self, *args, **kwargs):
 		raise NotImplementedError
 
 class Match (Module):
@@ -46,9 +46,9 @@ class Match (Module):
 	def validate_config(self):
 		return True, ""
 
-	def run(self, exec_threads):
+	def run(self):
 		self._load_data()
-		self._process_data(exec_threads)
+		self._process_data()
 		self._push_results()
 
 	def _load_data(self):
@@ -85,7 +85,7 @@ class Match (Module):
 		if test == "regression_sigmoidal":
 			return an.regression(ranges, data, ['sig'])
 
-	def _process_data(self, exec_threads):
+	def _process_data(self):
 
 		tests = self.config["tests"]
 		data = self.data
@@ -103,7 +103,6 @@ class Match (Module):
 						input_vector.append((team, variable, test, data[team][variable]))
 
 		self.data = input_vector
-		#self.results = list(exec_threads.map(self._simplestats, self.data))
 		self.results = []
 		for test_var_data in self.data:
 			self.results.append(self._simplestats(test_var_data))
@@ -164,15 +163,15 @@ class Metric (Module):
 	def validate_config(self):
 		return True, ""
 
-	def run(self, exec_threads):
+	def run(self):
 		self._load_data()
-		self._process_data(exec_threads)
+		self._process_data()
 		self._push_results()
 
 	def _load_data(self):
 		self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp)
 
-	def _process_data(self, exec_threads):
+	def _process_data(self):
 
 		elo_N = self.config["tests"]["elo"]["N"]
 		elo_K = self.config["tests"]["elo"]["K"]
@@ -289,15 +288,15 @@ class Pit (Module):
 	def validate_config(self):
 		return True, ""
 
-	def run(self, exec_threads):
+	def run(self):
 		self._load_data()
-		self._process_data(exec_threads)
+		self._process_data()
 		self._push_results()
 
 	def _load_data(self):
 		self.data = d.load_pit(self.apikey, self.competition)
 
-	def _process_data(self, exec_threads):
+	def _process_data(self):
 		tests = self.config["tests"]
 		return_vector = {}
 		for team in self.data:

src/pull.py

@@ -1,7 +1,7 @@
 import requests 
 import json
 from exceptions import APIError
-from config import load_config
+from dep import load_config
 
 url = "https://titanscouting.epochml.org"
 config_tra = {}

src/superscript.py

@@ -23,6 +23,9 @@ __changelog__ = """changelog:
 			- config-preference option selects between prioritizing local config and prioritizing database config
 			- synchronize-config option selects whether to update the non prioritized config with the prioritized one
 			- divided config options between persistent ones (keys), and variable ones (everything else)
+		- generalized behavior of various core components by collecting loose functions in several dependencies into classes
+			- module.py contains classes, each one represents a single data analysis routine
+			- config.py contains the Configuration class, which stores the configuration information and abstracts the getter methods
 	0.9.3:
 		- improved data loading performance by removing redundant PyMongo client creation (120s to 14s)
 		- passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions
@@ -152,16 +155,12 @@ __all__ = [
 # imports:
 
 import json
-from multiprocessing import freeze_support
+import os, sys, time
-import os
+import pymongo # soon to be deprecated
-import pymongo
-import sys
-import time
 import traceback
 import warnings
 import zmq
-import pull
+from config import Configuration, ConfigurationError
-from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError
 from data import get_previous_time, set_current_time, check_new_database_matches
 from interface import splash, log, ERR, INF, stdout, stderr
 from module import Match, Metric, Pit
@@ -171,10 +170,6 @@ config_path = "config.json"
 def main(send, verbose = False, profile = False, debug = False):
 
 	def close_all():
-		if "exec_threads" in locals():
-			exec_threads.terminate()
-			exec_threads.join()
-			exec_threads.close()
 		if "client" in locals():
 			client.close()
 		if "f" in locals():
@@ -196,14 +191,11 @@ def main(send, verbose = False, profile = False, debug = False):
 
 			send(stdout, INF, "current time: " + str(loop_start))
 
-			config = {}
+			config = Configuration(config_path)
-
-			if load_config(config_path, config):
-				raise ConfigurationError("could not find config at <" + config_path + ">, generating blank config and exiting", 110)
 			
 			send(stdout, INF, "found and loaded config at <" + config_path + ">")
 
-			apikey, tbakey, preference, sync = parse_config_persistent(send, config)
+			apikey, tbakey = config.database, config.tba
 
 			send(stdout, INF, "found and loaded database and tba keys")
 
@@ -213,13 +205,10 @@ def main(send, verbose = False, profile = False, debug = False):
 			previous_time = get_previous_time(client)
 			send(stdout, INF, "analysis backtimed to: " + str(previous_time))
 
-			config = resolve_config_conflicts(send, client, config, preference, sync)
+			config.resolve_config_conflicts(send, client)
+
+			config_modules, competition = config.modules, config.competition
 
-			exec_threads, config_modules = parse_config_variable(send, config)
-			if 'competition' in config['variable']:
-				competition = config['variable']['competition']
-			else:
-				competition = pull.get_team_competition()
 			for m in config_modules:
 				if m in modules:
 					start = time.time()
@@ -227,7 +216,7 @@ def main(send, verbose = False, profile = False, debug = False):
 					valid = current_module.validate_config()
 					if not valid:
 						continue
-					current_module.run(exec_threads)
+					current_module.run()
 					send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds")
 					if debug:
 						f = open(m + ".log", "w+")
@@ -360,7 +349,6 @@ def restart(pid_path):
 if __name__ == "__main__":
 
 	if sys.platform.startswith("win"):
-		freeze_support()
 		start(None, verbose = True)
 
 	else:

src/validation-schema.json

@@ -1,27 +0,0 @@
-{
-  "persistent": {
-    "type": "dict",
-    "require_all": true,
-    "schema": {
-      "key": {
-        "type": "dict",
-        "require_all":true,
-        "schema": {
-          "database": {"type":"string"},
-          "tba": {"type": "string"},
-          "tra": {
-            "type": "dict",
-            "require_all": true,
-            "schema": {
-              "CLIENT_ID": {"type": "string"},
-              "CLIENT_SECRET": {"type": "string"},
-			  "url": {"type": "string"}
-            }
-          }
-        }
-      },
-      "config-preference": {"type": "string", "required": true},
-      "synchronize-config": {"type": "boolean", "required": true}
-    }
-  }
-}