From b4c9ef22b6d957119f39699b694a1511acfb84ff Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 9 Jan 2022 22:19:50 -0600 Subject: [PATCH] pull some data from API, fix some datatype compats Former-commit-id: 03ec0dd8fd4d1540a5061b00b3758a866d5a24ce --- src/cli/config.py | 202 +++++++++++++++++++++++++---------------- src/cli/data.py | 22 ++--- src/cli/exceptions.py | 5 + src/cli/pull.py | 66 ++++++++++++++ src/cli/superscript.py | 6 +- 5 files changed, 209 insertions(+), 92 deletions(-) create mode 100644 src/cli/exceptions.py create mode 100644 src/cli/pull.py diff --git a/src/cli/config.py b/src/cli/config.py index ee2b122..f837133 100644 --- a/src/cli/config.py +++ b/src/cli/config.py @@ -8,76 +8,130 @@ from interface import stderr, stdout, INF, ERR config_path = "config.json" -sample_json = """{ - "persistent":{ - "key":{ - "database":"", - "tba":"" - }, - "config-preference":"local", - "synchronize-config":false - }, - "variable":{ - - "max-threads":0.5, - - "competition":"", - "team":"", - - "event-delay":false, - "loop-delay":0, - "reportable":true, - - "teams":[], - - "modules":{ - - "match":{ - "tests":{ - "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] - } - - }, - - "metric":{ - "tests":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 - }, - "ts":{ - "mu":25, - "sigma":8.33 - } - } - }, - - "pit":{ - "tests":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true - } - } - } - } -}""" +sample_json = """ +{ + "persistent":{ + "key":{ + "database":"", + "tba":"", + "tra":{ + "CLIENT_ID":"", + "CLIENT_SECRET":"" + } + }, + "config-preference":"local", + "synchronize-config":false + }, + "variable":{ + "max-threads":0.5, + "team":"", + "event-delay":false, + "loop-delay":0, + "reportable":true, + "teams":[ + + ], + "modules":{ + "match":{ + "tests":{ + "balls-blocked":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-collected":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-started":[ + "basic_stats", + "historical_analyss", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ] + } + }, + "metric":{ + "tests":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + } + }, + "pit":{ + "tests":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true + } + } + } + } +} +""" class ConfigurationError (Exception): code = None @@ -139,23 +193,17 @@ def parse_config_variable(send, config): raise ConfigurationError("unable to start threads", 200) send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") - try: - competition = config["variable"]["competition"] - except: - raise ConfigurationError("variable/competition field is invalid or missing", 101) try: modules = config["variable"]["modules"] except: raise ConfigurationError("variable/modules field is invalid or missing", 102) - if competition == None or competition == "": - raise ConfigurationError("variable/competition field is empty", 105) if modules == None: raise ConfigurationError("variable/modules field is empty", 106) send(stdout, INF, "found and loaded competition, match, metrics, pit from config") - return exec_threads, competition, modules + return exec_threads, modules def resolve_config_conflicts(send, client, config, preference, sync): diff --git a/src/cli/data.py b/src/cli/data.py index 1d9c056..d262bd3 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -1,4 +1,6 @@ import requests +import pull +import pandas as pd def pull_new_tba_matches(apikey, competition, cutoff): api_key= apikey @@ -13,7 +15,7 @@ def get_team_match_data(client, competition, team_num): db = client.data_scouting mdata = db.matchdata out = {} - for i in mdata.find({"competition" : competition, "team_scouted": team_num}): + for i in mdata.find({"competition" : competition, "team_scouted": str(team_num)}): out[i['match']] = i['data'] return pd.DataFrame(out) @@ -21,7 +23,7 @@ def get_team_pit_data(client, competition, team_num): db = client.data_scouting mdata = db.pitdata out = {} - return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"] + return mdata.find_one({"competition" : competition, "team_scouted": str(team_num)})["data"] def get_team_metrics_data(client, competition, team_num): db = client.data_processing @@ -29,25 +31,21 @@ def get_team_metrics_data(client, competition, team_num): return mdata.find_one({"competition" : competition, "team": team_num}) def get_match_data_formatted(client, competition): - db = client.data_scouting - mdata = db.teamlist - x=mdata.find_one({"competition":competition}) + teams_at_comp = pull.get_teams_at_competition(competition) out = {} - for i in x: + for team in teams_at_comp: try: - out[int(i)] = unkeyify_2l(get_team_match_data(client, competition, int(i)).transpose().to_dict()) + out[int(team)] = unkeyify_2l(get_team_match_data(client, competition, team).transpose().to_dict()) except: pass return out def get_metrics_data_formatted(client, competition): - db = client.data_scouting - mdata = db.teamlist - x=mdata.find_one({"competition":competition}) + teams_at_comp = pull.get_teams_at_competition(competition) out = {} - for i in x: + for team in teams_at_comp: try: - out[int(i)] = get_team_metrics_data(client, competition, int(i)) + out[int(team)] = get_team_metrics_data(client, competition, int(team)) except: pass return out diff --git a/src/cli/exceptions.py b/src/cli/exceptions.py new file mode 100644 index 0000000..891668b --- /dev/null +++ b/src/cli/exceptions.py @@ -0,0 +1,5 @@ +class APIError(Exception): + code = None + def __init__(self, str, endpoint): + super().__init__(str) + self.endpoint = endpoint \ No newline at end of file diff --git a/src/cli/pull.py b/src/cli/pull.py new file mode 100644 index 0000000..db2772f --- /dev/null +++ b/src/cli/pull.py @@ -0,0 +1,66 @@ +import requests +import json +from exceptions import APIError + +def load_config(path): + with open(path, "r") as f: + return json.load(f) + +url = "https://titanscouting.epochml.org" +config_tra = load_config("config.json") +trakey = config_tra['persistent']['key']['tra'] + +def get_team_competition(): + endpoint = '/api/fetchTeamCompetition' + params = { + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return json['competition'] + else: + raise APIError(json, endpoint) + +def get_team(): + endpoint = '/api/fetchTeamCompetition' + params = { + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return json['team'] + else: + raise APIError(json, endpoint) + +def get_team_match_data(competition, team_num): + endpoint = '/api/fetchAllTeamMatchData' + params = { + "competition": competition, + "teamScouted": team_num, + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return json['data'][team_num] + else: + raise APIError(json, endpoint) + +def get_teams_at_competition(competition): + endpoint = '/api/fetchAllTeamNicknamesAtCompetition' + params = { + "competition": competition, + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return list(json['data'].keys()) + else: + raise APIError(json, endpoint) \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 064a371..a362e4a 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -160,7 +160,7 @@ import time import traceback import warnings import zmq - +import pull from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError from data import get_previous_time, set_current_time, check_new_database_matches from interface import splash, log, ERR, INF, stdout, stderr @@ -214,8 +214,8 @@ def main(send, verbose = False, profile = False, debug = False): config = resolve_config_conflicts(send, client, config, preference, sync) - exec_threads, competition, config_modules = parse_config_variable(send, config) - + exec_threads, config_modules = parse_config_variable(send, config) + competition = pull.get_team_competition() for m in config_modules: if m in modules: start = time.time()