mirror of
https://github.com/titanscouting/tra-superscript.git
synced 2024-09-20 00:47:56 +00:00
moved config functions into Configuration class,
simplified exception class by removing error codes, removed exec_threads from module parameters Former-commit-id: 545ef765653970e1cdebac692eccd227effb2508
This commit is contained in:
Arthur Lu
parent
524a0a211d
commit
b43836899d
466
src/config.py
466
src/config.py
@ -1,244 +1,252 @@
|
||||
import math
|
||||
import json
|
||||
from multiprocessing import Pool
|
||||
import os
|
||||
from cerberus import Validator
|
||||
from exceptions import ConfigurationError
|
||||
from cerberus import Validator
|
||||
|
||||
from data import set_database_config, get_database_config
|
||||
from interface import stderr, stdout, INF, ERR
|
||||
|
||||
config_path = "config.json"
|
||||
class Configuration:
|
||||
|
||||
sample_json = """
|
||||
{
|
||||
"persistent":{
|
||||
"key":{
|
||||
"database":"",
|
||||
"tba":"",
|
||||
"tra":{
|
||||
"CLIENT_ID":"",
|
||||
"CLIENT_SECRET":"",
|
||||
"url": ""
|
||||
}
|
||||
},
|
||||
"config-preference":"local",
|
||||
"synchronize-config":false
|
||||
},
|
||||
"variable":{
|
||||
"max-threads":0.5,
|
||||
"team":"",
|
||||
"event-delay":false,
|
||||
"loop-delay":0,
|
||||
"reportable":true,
|
||||
"teams":[
|
||||
path = None
|
||||
config = {}
|
||||
|
||||
],
|
||||
"modules":{
|
||||
"match":{
|
||||
"tests":{
|
||||
"balls-blocked":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-collected":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-lower-teleop":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-lower-auto":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-started":[
|
||||
"basic_stats",
|
||||
"historical_analyss",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-upper-teleop":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-upper-auto":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
]
|
||||
}
|
||||
},
|
||||
"metric":{
|
||||
"tests":{
|
||||
"elo":{
|
||||
"score":1500,
|
||||
"N":400,
|
||||
"K":24
|
||||
},
|
||||
"gl2":{
|
||||
"score":1500,
|
||||
"rd":250,
|
||||
"vol":0.06
|
||||
},
|
||||
"ts":{
|
||||
"mu":25,
|
||||
"sigma":8.33
|
||||
}
|
||||
}
|
||||
},
|
||||
"pit":{
|
||||
"tests":{
|
||||
"wheel-mechanism":true,
|
||||
"low-balls":true,
|
||||
"high-balls":true,
|
||||
"wheel-success":true,
|
||||
"strategic-focus":true,
|
||||
"climb-mechanism":true,
|
||||
"attitude":true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
_sample_config = {
|
||||
"persistent":{
|
||||
"key":{
|
||||
"database":"",
|
||||
"tba":"",
|
||||
"tra":{
|
||||
"CLIENT_ID":"",
|
||||
"CLIENT_SECRET":"",
|
||||
"url": ""
|
||||
}
|
||||
},
|
||||
"config-preference":"local",
|
||||
"synchronize-config":False
|
||||
},
|
||||
"variable":{
|
||||
"event-delay":False,
|
||||
"loop-delay":0,
|
||||
"competition": "2020ilch",
|
||||
"modules":{
|
||||
"match":{
|
||||
"tests":{
|
||||
"balls-blocked":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-collected":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-lower-teleop":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-lower-auto":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-started":[
|
||||
"basic_stats",
|
||||
"historical_analyss",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-upper-teleop":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-upper-auto":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
]
|
||||
}
|
||||
},
|
||||
"metric":{
|
||||
"tests":{
|
||||
"elo":{
|
||||
"score":1500,
|
||||
"N":400,
|
||||
"K":24
|
||||
},
|
||||
"gl2":{
|
||||
"score":1500,
|
||||
"rd":250,
|
||||
"vol":0.06
|
||||
},
|
||||
"ts":{
|
||||
"mu":25,
|
||||
"sigma":8.33
|
||||
}
|
||||
}
|
||||
},
|
||||
"pit":{
|
||||
"tests":{
|
||||
"wheel-mechanism":True,
|
||||
"low-balls":True,
|
||||
"high-balls":True,
|
||||
"wheel-success":True,
|
||||
"strategic-focus":True,
|
||||
"climb-mechanism":True,
|
||||
"attitude":True
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def parse_config_persistent(send, config):
|
||||
v = Validator(load_validation_schema(), allow_unknown = True)
|
||||
isValidated = v.validate(config)
|
||||
_validation_schema = {
|
||||
"persistent": {
|
||||
"type": "dict",
|
||||
"required": True,
|
||||
"require_all": True,
|
||||
"schema": {
|
||||
"key": {
|
||||
"type": "dict",
|
||||
"require_all":True,
|
||||
"schema": {
|
||||
"database": {"type":"string"},
|
||||
"tba": {"type": "string"},
|
||||
"tra": {
|
||||
"type": "dict",
|
||||
"require_all": True,
|
||||
"schema": {
|
||||
"CLIENT_ID": {"type": "string"},
|
||||
"CLIENT_SECRET": {"type": "string"},
|
||||
"url": {"type": "string"}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"config-preference": {"type": "string", "required": True},
|
||||
"synchronize-config": {"type": "boolean", "required": True}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if not isValidated:
|
||||
raise ConfigurationError(v.errors, 101)
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.load_config()
|
||||
self.validate_config()
|
||||
|
||||
apikey = config["persistent"]["key"]["database"]
|
||||
tbakey = config["persistent"]["key"]["tba"]
|
||||
preference = config["persistent"]["config-preference"]
|
||||
sync = config["persistent"]["synchronize-config"]
|
||||
def load_config(self):
|
||||
try:
|
||||
f = open(self.path, "r")
|
||||
self.config.update(json.load(f))
|
||||
f.close()
|
||||
except:
|
||||
self.config = self._sample_config
|
||||
self.save_config()
|
||||
f.close()
|
||||
raise ConfigurationError("could not find config file at <" + self.path + ">, created new sample config file at that path")
|
||||
|
||||
return apikey, tbakey, preference, sync
|
||||
|
||||
def parse_config_variable(send, config):
|
||||
|
||||
sys_max_threads = os.cpu_count()
|
||||
try:
|
||||
cfg_max_threads = config["variable"]["max-threads"]
|
||||
except:
|
||||
raise ConfigurationError("variable/max-threads field is invalid or missing, refer to documentation for configuration options", 109)
|
||||
if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 :
|
||||
alloc_processes = sys_max_threads + cfg_max_threads
|
||||
elif cfg_max_threads > 0 and cfg_max_threads < 1:
|
||||
alloc_processes = math.floor(cfg_max_threads * sys_max_threads)
|
||||
elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads:
|
||||
alloc_processes = cfg_max_threads
|
||||
elif cfg_max_threads == 0:
|
||||
alloc_processes = sys_max_threads
|
||||
else:
|
||||
raise ConfigurationError("variable/max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, 110)
|
||||
try:
|
||||
exec_threads = Pool(processes = alloc_processes)
|
||||
except Exception as e:
|
||||
send(stderr, INF, e)
|
||||
raise ConfigurationError("unable to start threads", 200)
|
||||
send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads")
|
||||
|
||||
try:
|
||||
modules = config["variable"]["modules"]
|
||||
except:
|
||||
raise ConfigurationError("variable/modules field is invalid or missing", 102)
|
||||
|
||||
if modules == None:
|
||||
raise ConfigurationError("variable/modules field is empty", 106)
|
||||
|
||||
send(stdout, INF, "found and loaded competition, match, metrics, pit from config")
|
||||
|
||||
return exec_threads, modules
|
||||
|
||||
def resolve_config_conflicts(send, client, config, preference, sync):
|
||||
|
||||
if sync:
|
||||
if preference == "local" or preference == "client":
|
||||
send(stdout, INF, "config-preference set to local/client, loading local config information")
|
||||
remote_config = get_database_config(client)
|
||||
if remote_config != config["variable"]:
|
||||
set_database_config(client, config["variable"])
|
||||
send(stdout, INF, "database config was different and was updated")
|
||||
return config
|
||||
elif preference == "remote" or preference == "database":
|
||||
send(stdout, INF, "config-preference set to remote/database, loading remote config information")
|
||||
remote_config= get_database_config(client)
|
||||
if remote_config != config["variable"]:
|
||||
config["variable"] = remote_config
|
||||
if save_config(config_path, config):
|
||||
raise ConfigurationError("local config was different but could not be updated", 121)
|
||||
send(stdout, INF, "local config was different and was updated")
|
||||
return config
|
||||
else:
|
||||
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120)
|
||||
else:
|
||||
if preference == "local" or preference == "client":
|
||||
send(stdout, INF, "config-preference set to local/client, loading local config information")
|
||||
return config
|
||||
elif preference == "remote" or preference == "database":
|
||||
send(stdout, INF, "config-preference set to remote/database, loading database config information")
|
||||
config["variable"] = get_database_config(client)
|
||||
return config
|
||||
else:
|
||||
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120)
|
||||
|
||||
def load_config(path, config_vector):
|
||||
try:
|
||||
f = open(path, "r")
|
||||
config_vector.update(json.load(f))
|
||||
def save_config(self):
|
||||
f = open(self.path, "w+")
|
||||
json.dump(self.config, f, ensure_ascii=False, indent=4)
|
||||
f.close()
|
||||
return 0
|
||||
except:
|
||||
f = open(path, "w")
|
||||
f.write(sample_json)
|
||||
f.close()
|
||||
return 1
|
||||
|
||||
def load_validation_schema():
|
||||
try:
|
||||
with open("validation-schema.json", "r") as f:
|
||||
return json.load(f)
|
||||
except:
|
||||
raise FileNotFoundError("Validation schema not found at validation-schema.json")
|
||||
def validate_config(self):
|
||||
v = Validator(self._validation_schema, allow_unknown = True)
|
||||
isValidated = v.validate(self.config)
|
||||
|
||||
def save_config(path, config_vector):
|
||||
f = open(path, "w+")
|
||||
json.dump(config_vector, f, ensure_ascii=False, indent=4)
|
||||
f.close()
|
||||
return 0
|
||||
if not isValidated:
|
||||
raise ConfigurationError("config validation error: " + v.errors)
|
||||
|
||||
def __getattr__(self, name): # simple linear lookup method for common multikey-value paths, TYPE UNSAFE
|
||||
if name == "persistent":
|
||||
return self.config["persistent"]
|
||||
elif name == "key":
|
||||
return self.config["persistent"]["key"]
|
||||
elif name == "database":
|
||||
# soon to be deprecated
|
||||
return self.config["persistent"]["key"]["database"]
|
||||
elif name == "tba":
|
||||
return self.config["persistent"]["key"]["tba"]
|
||||
elif name == "tra":
|
||||
return self.config["persistent"]["key"]["tra"]
|
||||
elif name == "priority":
|
||||
return self.config["persistent"]["config-preference"]
|
||||
elif name == "sync":
|
||||
return self.config["persistent"]["synchronize-config"]
|
||||
elif name == "variable":
|
||||
return self.config["variable"]
|
||||
elif name == "event_delay":
|
||||
return self.config["variable"]["event-delay"]
|
||||
elif name == "loop_delay":
|
||||
return self.config["variable"]["loop-delay"]
|
||||
elif name == "competition":
|
||||
return self.config["variable"]["competition"]
|
||||
elif name == "modules":
|
||||
return self.config["variable"]["modules"]
|
||||
else:
|
||||
return None
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.config[key]
|
||||
|
||||
def resolve_config_conflicts(self, send, client): # needs improvement with new localization scheme
|
||||
sync = self.sync
|
||||
priority = self.priority
|
||||
|
||||
if sync:
|
||||
if priority == "local" or priority == "client":
|
||||
send(stdout, INF, "config-preference set to local/client, loading local config information")
|
||||
remote_config = get_database_config(client)
|
||||
if remote_config != self.config["variable"]:
|
||||
set_database_config(client, self.config["variable"])
|
||||
send(stdout, INF, "database config was different and was updated")
|
||||
# no change to config
|
||||
elif priority == "remote" or priority == "database":
|
||||
send(stdout, INF, "config-preference set to remote/database, loading remote config information")
|
||||
remote_config = get_database_config(client)
|
||||
if remote_config != self.config["variable"]:
|
||||
self.config["variable"] = remote_config
|
||||
self.save_config()
|
||||
# change variable to match remote
|
||||
send(stdout, INF, "local config was different and was updated")
|
||||
else:
|
||||
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"")
|
||||
else:
|
||||
if priority == "local" or priority == "client":
|
||||
send(stdout, INF, "config-preference set to local/client, loading local config information")
|
||||
# no change to config
|
||||
elif priority == "remote" or priority == "database":
|
||||
send(stdout, INF, "config-preference set to remote/database, loading database config information")
|
||||
self.config["variable"] = get_database_config(client)
|
||||
# change variable to match remote without updating local version
|
||||
else:
|
||||
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"")
|
||||
141
src/dep.py
Normal file
141
src/dep.py
Normal file
@ -0,0 +1,141 @@
|
||||
# contains deprecated functions, not to be used unless nessasary!
|
||||
|
||||
import json
|
||||
|
||||
sample_json = """
|
||||
{
|
||||
"persistent":{
|
||||
"key":{
|
||||
"database":"",
|
||||
"tba":"",
|
||||
"tra":{
|
||||
"CLIENT_ID":"",
|
||||
"CLIENT_SECRET":"",
|
||||
"url": ""
|
||||
}
|
||||
},
|
||||
"config-preference":"local",
|
||||
"synchronize-config":false
|
||||
},
|
||||
"variable":{
|
||||
"max-threads":0.5,
|
||||
"team":"",
|
||||
"event-delay":false,
|
||||
"loop-delay":0,
|
||||
"reportable":true,
|
||||
"teams":[
|
||||
|
||||
],
|
||||
"modules":{
|
||||
"match":{
|
||||
"tests":{
|
||||
"balls-blocked":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-collected":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-lower-teleop":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-lower-auto":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-started":[
|
||||
"basic_stats",
|
||||
"historical_analyss",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-upper-teleop":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
],
|
||||
"balls-upper-auto":[
|
||||
"basic_stats",
|
||||
"historical_analysis",
|
||||
"regression_linear",
|
||||
"regression_logarithmic",
|
||||
"regression_exponential",
|
||||
"regression_polynomial",
|
||||
"regression_sigmoidal"
|
||||
]
|
||||
}
|
||||
},
|
||||
"metric":{
|
||||
"tests":{
|
||||
"elo":{
|
||||
"score":1500,
|
||||
"N":400,
|
||||
"K":24
|
||||
},
|
||||
"gl2":{
|
||||
"score":1500,
|
||||
"rd":250,
|
||||
"vol":0.06
|
||||
},
|
||||
"ts":{
|
||||
"mu":25,
|
||||
"sigma":8.33
|
||||
}
|
||||
}
|
||||
},
|
||||
"pit":{
|
||||
"tests":{
|
||||
"wheel-mechanism":true,
|
||||
"low-balls":true,
|
||||
"high-balls":true,
|
||||
"wheel-success":true,
|
||||
"strategic-focus":true,
|
||||
"climb-mechanism":true,
|
||||
"attitude":true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
def load_config(path, config_vector):
|
||||
try:
|
||||
f = open(path, "r")
|
||||
config_vector.update(json.load(f))
|
||||
f.close()
|
||||
return 0
|
||||
except:
|
||||
f = open(path, "w")
|
||||
f.write(sample_json)
|
||||
f.close()
|
||||
return 1
|
||||
@ -1,11 +1,7 @@
|
||||
class APIError(Exception):
|
||||
code = None
|
||||
def __init__(self, str, endpoint):
|
||||
def __init__(self, str):
|
||||
super().__init__(str)
|
||||
self.endpoint = endpoint
|
||||
|
||||
class ConfigurationError (Exception):
|
||||
code = None
|
||||
def __init__(self, str, code):
|
||||
def __init__(self, str):
|
||||
super().__init__(str)
|
||||
self.code = code
|
||||
@ -22,7 +22,7 @@ class Module(metaclass = abc.ABCMeta):
|
||||
def validate_config(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
@abc.abstractmethod
|
||||
def run(self, exec_threads, *args, **kwargs):
|
||||
def run(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
class Match (Module):
|
||||
@ -46,9 +46,9 @@ class Match (Module):
|
||||
def validate_config(self):
|
||||
return True, ""
|
||||
|
||||
def run(self, exec_threads):
|
||||
def run(self):
|
||||
self._load_data()
|
||||
self._process_data(exec_threads)
|
||||
self._process_data()
|
||||
self._push_results()
|
||||
|
||||
def _load_data(self):
|
||||
@ -85,7 +85,7 @@ class Match (Module):
|
||||
if test == "regression_sigmoidal":
|
||||
return an.regression(ranges, data, ['sig'])
|
||||
|
||||
def _process_data(self, exec_threads):
|
||||
def _process_data(self):
|
||||
|
||||
tests = self.config["tests"]
|
||||
data = self.data
|
||||
@ -103,7 +103,6 @@ class Match (Module):
|
||||
input_vector.append((team, variable, test, data[team][variable]))
|
||||
|
||||
self.data = input_vector
|
||||
#self.results = list(exec_threads.map(self._simplestats, self.data))
|
||||
self.results = []
|
||||
for test_var_data in self.data:
|
||||
self.results.append(self._simplestats(test_var_data))
|
||||
@ -164,15 +163,15 @@ class Metric (Module):
|
||||
def validate_config(self):
|
||||
return True, ""
|
||||
|
||||
def run(self, exec_threads):
|
||||
def run(self):
|
||||
self._load_data()
|
||||
self._process_data(exec_threads)
|
||||
self._process_data()
|
||||
self._push_results()
|
||||
|
||||
def _load_data(self):
|
||||
self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp)
|
||||
|
||||
def _process_data(self, exec_threads):
|
||||
def _process_data(self):
|
||||
|
||||
elo_N = self.config["tests"]["elo"]["N"]
|
||||
elo_K = self.config["tests"]["elo"]["K"]
|
||||
@ -289,15 +288,15 @@ class Pit (Module):
|
||||
def validate_config(self):
|
||||
return True, ""
|
||||
|
||||
def run(self, exec_threads):
|
||||
def run(self):
|
||||
self._load_data()
|
||||
self._process_data(exec_threads)
|
||||
self._process_data()
|
||||
self._push_results()
|
||||
|
||||
def _load_data(self):
|
||||
self.data = d.load_pit(self.apikey, self.competition)
|
||||
|
||||
def _process_data(self, exec_threads):
|
||||
def _process_data(self):
|
||||
tests = self.config["tests"]
|
||||
return_vector = {}
|
||||
for team in self.data:
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import requests
|
||||
import json
|
||||
from exceptions import APIError
|
||||
from config import load_config
|
||||
from dep import load_config
|
||||
|
||||
url = "https://titanscouting.epochml.org"
|
||||
config_tra = {}
|
||||
|
||||
@ -23,6 +23,9 @@ __changelog__ = """changelog:
|
||||
- config-preference option selects between prioritizing local config and prioritizing database config
|
||||
- synchronize-config option selects whether to update the non prioritized config with the prioritized one
|
||||
- divided config options between persistent ones (keys), and variable ones (everything else)
|
||||
- generalized behavior of various core components by collecting loose functions in several dependencies into classes
|
||||
- module.py contains classes, each one represents a single data analysis routine
|
||||
- config.py contains the Configuration class, which stores the configuration information and abstracts the getter methods
|
||||
0.9.3:
|
||||
- improved data loading performance by removing redundant PyMongo client creation (120s to 14s)
|
||||
- passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions
|
||||
@ -152,16 +155,12 @@ __all__ = [
|
||||
# imports:
|
||||
|
||||
import json
|
||||
from multiprocessing import freeze_support
|
||||
import os
|
||||
import pymongo
|
||||
import sys
|
||||
import time
|
||||
import os, sys, time
|
||||
import pymongo # soon to be deprecated
|
||||
import traceback
|
||||
import warnings
|
||||
import zmq
|
||||
import pull
|
||||
from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError
|
||||
from config import Configuration, ConfigurationError
|
||||
from data import get_previous_time, set_current_time, check_new_database_matches
|
||||
from interface import splash, log, ERR, INF, stdout, stderr
|
||||
from module import Match, Metric, Pit
|
||||
@ -171,10 +170,6 @@ config_path = "config.json"
|
||||
def main(send, verbose = False, profile = False, debug = False):
|
||||
|
||||
def close_all():
|
||||
if "exec_threads" in locals():
|
||||
exec_threads.terminate()
|
||||
exec_threads.join()
|
||||
exec_threads.close()
|
||||
if "client" in locals():
|
||||
client.close()
|
||||
if "f" in locals():
|
||||
@ -196,14 +191,11 @@ def main(send, verbose = False, profile = False, debug = False):
|
||||
|
||||
send(stdout, INF, "current time: " + str(loop_start))
|
||||
|
||||
config = {}
|
||||
|
||||
if load_config(config_path, config):
|
||||
raise ConfigurationError("could not find config at <" + config_path + ">, generating blank config and exiting", 110)
|
||||
config = Configuration(config_path)
|
||||
|
||||
send(stdout, INF, "found and loaded config at <" + config_path + ">")
|
||||
|
||||
apikey, tbakey, preference, sync = parse_config_persistent(send, config)
|
||||
apikey, tbakey = config.database, config.tba
|
||||
|
||||
send(stdout, INF, "found and loaded database and tba keys")
|
||||
|
||||
@ -213,13 +205,10 @@ def main(send, verbose = False, profile = False, debug = False):
|
||||
previous_time = get_previous_time(client)
|
||||
send(stdout, INF, "analysis backtimed to: " + str(previous_time))
|
||||
|
||||
config = resolve_config_conflicts(send, client, config, preference, sync)
|
||||
config.resolve_config_conflicts(send, client)
|
||||
|
||||
config_modules, competition = config.modules, config.competition
|
||||
|
||||
exec_threads, config_modules = parse_config_variable(send, config)
|
||||
if 'competition' in config['variable']:
|
||||
competition = config['variable']['competition']
|
||||
else:
|
||||
competition = pull.get_team_competition()
|
||||
for m in config_modules:
|
||||
if m in modules:
|
||||
start = time.time()
|
||||
@ -227,7 +216,7 @@ def main(send, verbose = False, profile = False, debug = False):
|
||||
valid = current_module.validate_config()
|
||||
if not valid:
|
||||
continue
|
||||
current_module.run(exec_threads)
|
||||
current_module.run()
|
||||
send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds")
|
||||
if debug:
|
||||
f = open(m + ".log", "w+")
|
||||
@ -360,7 +349,6 @@ def restart(pid_path):
|
||||
if __name__ == "__main__":
|
||||
|
||||
if sys.platform.startswith("win"):
|
||||
freeze_support()
|
||||
start(None, verbose = True)
|
||||
|
||||
else:
|
||||
|
||||
@ -1,27 +0,0 @@
|
||||
{
|
||||
"persistent": {
|
||||
"type": "dict",
|
||||
"require_all": true,
|
||||
"schema": {
|
||||
"key": {
|
||||
"type": "dict",
|
||||
"require_all":true,
|
||||
"schema": {
|
||||
"database": {"type":"string"},
|
||||
"tba": {"type": "string"},
|
||||
"tra": {
|
||||
"type": "dict",
|
||||
"require_all": true,
|
||||
"schema": {
|
||||
"CLIENT_ID": {"type": "string"},
|
||||
"CLIENT_SECRET": {"type": "string"},
|
||||
"url": {"type": "string"}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"config-preference": {"type": "string", "required": true},
|
||||
"synchronize-config": {"type": "boolean", "required": true}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user