pull some data from API, fix some datatype compats

This commit is contained in:
Dev Singh 2022-01-09 22:19:50 -06:00
parent 3669712bec
commit 03ec0dd8fd
5 changed files with 209 additions and 92 deletions

View File

@ -8,43 +8,97 @@ from interface import stderr, stdout, INF, ERR
config_path = "config.json" config_path = "config.json"
sample_json = """{ sample_json = """
{
"persistent":{ "persistent":{
"key":{ "key":{
"database":"", "database":"",
"tba":"" "tba":"",
"tra":{
"CLIENT_ID":"",
"CLIENT_SECRET":""
}
}, },
"config-preference":"local", "config-preference":"local",
"synchronize-config":false "synchronize-config":false
}, },
"variable":{ "variable":{
"max-threads":0.5, "max-threads":0.5,
"competition":"",
"team":"", "team":"",
"event-delay":false, "event-delay":false,
"loop-delay":0, "loop-delay":0,
"reportable":true, "reportable":true,
"teams":[
"teams":[], ],
"modules":{ "modules":{
"match":{ "match":{
"tests":{ "tests":{
"balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], "balls-blocked":[
"balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], "basic_stats",
"balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], "historical_analysis",
"balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], "regression_linear",
"balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], "regression_logarithmic",
"balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], "regression_exponential",
"balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] "regression_polynomial",
"regression_sigmoidal"
],
"balls-collected":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-lower-teleop":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-lower-auto":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-started":[
"basic_stats",
"historical_analyss",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-upper-teleop":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-upper-auto":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
]
} }
}, },
"metric":{ "metric":{
"tests":{ "tests":{
"elo":{ "elo":{
@ -63,7 +117,6 @@ sample_json = """{
} }
} }
}, },
"pit":{ "pit":{
"tests":{ "tests":{
"wheel-mechanism":true, "wheel-mechanism":true,
@ -77,7 +130,8 @@ sample_json = """{
} }
} }
} }
}""" }
"""
class ConfigurationError (Exception): class ConfigurationError (Exception):
code = None code = None
@ -139,23 +193,17 @@ def parse_config_variable(send, config):
raise ConfigurationError("unable to start threads", 200) raise ConfigurationError("unable to start threads", 200)
send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads")
try:
competition = config["variable"]["competition"]
except:
raise ConfigurationError("variable/competition field is invalid or missing", 101)
try: try:
modules = config["variable"]["modules"] modules = config["variable"]["modules"]
except: except:
raise ConfigurationError("variable/modules field is invalid or missing", 102) raise ConfigurationError("variable/modules field is invalid or missing", 102)
if competition == None or competition == "":
raise ConfigurationError("variable/competition field is empty", 105)
if modules == None: if modules == None:
raise ConfigurationError("variable/modules field is empty", 106) raise ConfigurationError("variable/modules field is empty", 106)
send(stdout, INF, "found and loaded competition, match, metrics, pit from config") send(stdout, INF, "found and loaded competition, match, metrics, pit from config")
return exec_threads, competition, modules return exec_threads, modules
def resolve_config_conflicts(send, client, config, preference, sync): def resolve_config_conflicts(send, client, config, preference, sync):

View File

@ -1,4 +1,6 @@
import requests import requests
import pull
import pandas as pd
def pull_new_tba_matches(apikey, competition, cutoff): def pull_new_tba_matches(apikey, competition, cutoff):
api_key= apikey api_key= apikey
@ -13,7 +15,7 @@ def get_team_match_data(client, competition, team_num):
db = client.data_scouting db = client.data_scouting
mdata = db.matchdata mdata = db.matchdata
out = {} out = {}
for i in mdata.find({"competition" : competition, "team_scouted": team_num}): for i in mdata.find({"competition" : competition, "team_scouted": str(team_num)}):
out[i['match']] = i['data'] out[i['match']] = i['data']
return pd.DataFrame(out) return pd.DataFrame(out)
@ -21,7 +23,7 @@ def get_team_pit_data(client, competition, team_num):
db = client.data_scouting db = client.data_scouting
mdata = db.pitdata mdata = db.pitdata
out = {} out = {}
return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"] return mdata.find_one({"competition" : competition, "team_scouted": str(team_num)})["data"]
def get_team_metrics_data(client, competition, team_num): def get_team_metrics_data(client, competition, team_num):
db = client.data_processing db = client.data_processing
@ -29,25 +31,21 @@ def get_team_metrics_data(client, competition, team_num):
return mdata.find_one({"competition" : competition, "team": team_num}) return mdata.find_one({"competition" : competition, "team": team_num})
def get_match_data_formatted(client, competition): def get_match_data_formatted(client, competition):
db = client.data_scouting teams_at_comp = pull.get_teams_at_competition(competition)
mdata = db.teamlist
x=mdata.find_one({"competition":competition})
out = {} out = {}
for i in x: for team in teams_at_comp:
try: try:
out[int(i)] = unkeyify_2l(get_team_match_data(client, competition, int(i)).transpose().to_dict()) out[int(team)] = unkeyify_2l(get_team_match_data(client, competition, team).transpose().to_dict())
except: except:
pass pass
return out return out
def get_metrics_data_formatted(client, competition): def get_metrics_data_formatted(client, competition):
db = client.data_scouting teams_at_comp = pull.get_teams_at_competition(competition)
mdata = db.teamlist
x=mdata.find_one({"competition":competition})
out = {} out = {}
for i in x: for team in teams_at_comp:
try: try:
out[int(i)] = get_team_metrics_data(client, competition, int(i)) out[int(team)] = get_team_metrics_data(client, competition, int(team))
except: except:
pass pass
return out return out

5
src/cli/exceptions.py Normal file
View File

@ -0,0 +1,5 @@
class APIError(Exception):
code = None
def __init__(self, str, endpoint):
super().__init__(str)
self.endpoint = endpoint

66
src/cli/pull.py Normal file
View File

@ -0,0 +1,66 @@
import requests
import json
from exceptions import APIError
def load_config(path):
with open(path, "r") as f:
return json.load(f)
url = "https://titanscouting.epochml.org"
config_tra = load_config("config.json")
trakey = config_tra['persistent']['key']['tra']
def get_team_competition():
endpoint = '/api/fetchTeamCompetition'
params = {
"CLIENT_ID": trakey['CLIENT_ID'],
"CLIENT_SECRET": trakey['CLIENT_SECRET']
}
response = requests.request("GET", url + endpoint, params=params)
json = response.json()
if json['success']:
return json['competition']
else:
raise APIError(json, endpoint)
def get_team():
endpoint = '/api/fetchTeamCompetition'
params = {
"CLIENT_ID": trakey['CLIENT_ID'],
"CLIENT_SECRET": trakey['CLIENT_SECRET']
}
response = requests.request("GET", url + endpoint, params=params)
json = response.json()
if json['success']:
return json['team']
else:
raise APIError(json, endpoint)
def get_team_match_data(competition, team_num):
endpoint = '/api/fetchAllTeamMatchData'
params = {
"competition": competition,
"teamScouted": team_num,
"CLIENT_ID": trakey['CLIENT_ID'],
"CLIENT_SECRET": trakey['CLIENT_SECRET']
}
response = requests.request("GET", url + endpoint, params=params)
json = response.json()
if json['success']:
return json['data'][team_num]
else:
raise APIError(json, endpoint)
def get_teams_at_competition(competition):
endpoint = '/api/fetchAllTeamNicknamesAtCompetition'
params = {
"competition": competition,
"CLIENT_ID": trakey['CLIENT_ID'],
"CLIENT_SECRET": trakey['CLIENT_SECRET']
}
response = requests.request("GET", url + endpoint, params=params)
json = response.json()
if json['success']:
return list(json['data'].keys())
else:
raise APIError(json, endpoint)

View File

@ -160,7 +160,7 @@ import time
import traceback import traceback
import warnings import warnings
import zmq import zmq
import pull
from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError
from data import get_previous_time, set_current_time, check_new_database_matches from data import get_previous_time, set_current_time, check_new_database_matches
from interface import splash, log, ERR, INF, stdout, stderr from interface import splash, log, ERR, INF, stdout, stderr
@ -214,8 +214,8 @@ def main(send, verbose = False, profile = False, debug = False):
config = resolve_config_conflicts(send, client, config, preference, sync) config = resolve_config_conflicts(send, client, config, preference, sync)
exec_threads, competition, config_modules = parse_config_variable(send, config) exec_threads, config_modules = parse_config_variable(send, config)
competition = pull.get_team_competition()
for m in config_modules: for m in config_modules:
if m in modules: if m in modules:
start = time.time() start = time.time()