tra-superscript/competition/data.py

219 lines
6.0 KiB
Python
Raw Normal View History

from calendar import c
import requests
import pull
import pandas as pd
import json
2022-04-08 20:20:41 +00:00
def pull_new_tba_matches(apikey, competition, last_match):
api_key= apikey
x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth-Key":api_key})
2022-03-25 18:39:24 +00:00
json = x.json()
out = []
2022-03-25 18:39:24 +00:00
for i in json:
2022-04-08 20:20:41 +00:00
if i["actual_time"] != None and i["comp_level"] == "qm" and i["match_number"] > last_match :
out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]})
2022-03-28 19:43:24 +00:00
out.sort(key=lambda x: x['match'])
return out
def pull_new_tba_matches_manual(apikey, competition, cutoff):
filename = competition+"-wins.json"
with open(filename, 'r') as f:
2022-03-29 23:43:28 +00:00
data = json.load(f)
return data
def get_team_match_data(client, competition, team_num):
db = client.data_scouting
mdata = db.matchdata
out = {}
for i in mdata.find({"competition" : competition, "team_scouted": str(team_num)}):
out[i['match']] = i['data']
return pd.DataFrame(out)
2022-03-28 19:43:24 +00:00
def clear_metrics(client, competition):
db = client.data_processing
data = db.team_metrics
data.delete_many({competition: competition})
return True
def get_team_pit_data(client, competition, team_num):
db = client.data_scouting
mdata = db.pitdata
out = {}
return mdata.find_one({"competition" : competition, "team_scouted": str(team_num)})["data"]
def get_team_metrics_data(client, competition, team_num):
db = client.data_processing
mdata = db.team_metrics
temp = mdata.find_one({"team": team_num})
if temp != None:
if competition in temp['metrics'].keys():
temp = temp['metrics'][competition]
else :
temp = None
else:
temp = None
return temp
def get_match_data_formatted(client, competition):
teams_at_comp = pull.get_teams_at_competition(competition)
out = {}
for team in teams_at_comp:
Merge service-dev changes with master (#24) * added config.json removed old config files Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.000 Signed-off-by: Arthur <learthurgo@gmail.com> * changed data.py Signed-off-by: Arthur <learthurgo@gmail.com> * changes to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * removed cells from visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * more changes to visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * added analysis-master/metrics/__pycache__ to git ignore moved pit configs in config.json to the borrom superscript.py v 0.0.6.001 Signed-off-by: Arthur <learthurgo@gmail.com> * removed old database key Signed-off-by: Arthur <learthurgo@gmail.com> * adjusted config files Signed-off-by: Arthur <learthurgo@gmail.com> * Delete config-pop.json * fixed .gitignore Signed-off-by: Arthur <learthurgo@gmail.com> * analysis.py 1.2.1.003 added team kv pair to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.002 Signed-off-by: Arthur <learthurgo@gmail.com> * finished app.py API made minute changes to parentheses use in various packages Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * deleted a __pycache__ folder from metrics Signed-off-by: Arthur Lu <learthurgo@gmail.com> * more changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * additions to app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed app.py to api.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed extranneous files Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed api.py to tra.py removed rest api calls from tra.py * renamed api.py to tra.py removed rest api calls from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed flask import from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * changes to devcontainer.json Signed-off-by: Arthur Lu <learthurgo@gmail.com> * fixed unit tests to be correct removed some tests regressions because of potential function overflow removed trueskill unit test because of slight deviation chance Signed-off-by: Arthur Lu <learthurgo@gmail.com>
2020-05-20 13:52:38 +00:00
try:
out[int(team)] = unkeyify_2l(get_team_match_data(client, competition, team).transpose().to_dict())
Merge service-dev changes with master (#24) * added config.json removed old config files Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.000 Signed-off-by: Arthur <learthurgo@gmail.com> * changed data.py Signed-off-by: Arthur <learthurgo@gmail.com> * changes to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * removed cells from visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * more changes to visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * added analysis-master/metrics/__pycache__ to git ignore moved pit configs in config.json to the borrom superscript.py v 0.0.6.001 Signed-off-by: Arthur <learthurgo@gmail.com> * removed old database key Signed-off-by: Arthur <learthurgo@gmail.com> * adjusted config files Signed-off-by: Arthur <learthurgo@gmail.com> * Delete config-pop.json * fixed .gitignore Signed-off-by: Arthur <learthurgo@gmail.com> * analysis.py 1.2.1.003 added team kv pair to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.002 Signed-off-by: Arthur <learthurgo@gmail.com> * finished app.py API made minute changes to parentheses use in various packages Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * deleted a __pycache__ folder from metrics Signed-off-by: Arthur Lu <learthurgo@gmail.com> * more changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * additions to app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed app.py to api.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed extranneous files Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed api.py to tra.py removed rest api calls from tra.py * renamed api.py to tra.py removed rest api calls from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed flask import from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * changes to devcontainer.json Signed-off-by: Arthur Lu <learthurgo@gmail.com> * fixed unit tests to be correct removed some tests regressions because of potential function overflow removed trueskill unit test because of slight deviation chance Signed-off-by: Arthur Lu <learthurgo@gmail.com>
2020-05-20 13:52:38 +00:00
except:
pass
return out
def get_metrics_data_formatted(client, competition):
teams_at_comp = pull.get_teams_at_competition(competition)
out = {}
for team in teams_at_comp:
try:
out[int(team)] = get_team_metrics_data(client, competition, int(team))
except:
pass
return out
def get_pit_data_formatted(client, competition):
2023-03-17 15:09:48 +00:00
x=requests.get("https://scouting.titanrobotics2022.com/api/fetchAllTeamNicknamesAtCompetition?competition="+competition)
x = x.json()
x = x['data']
x = x.keys()
out = {}
for i in x:
try:
out[int(i)] = get_team_pit_data(client, competition, int(i))
except:
pass
return out
def get_pit_variable_data(client, competition):
Merge service-dev changes with master (#24) * added config.json removed old config files Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.000 Signed-off-by: Arthur <learthurgo@gmail.com> * changed data.py Signed-off-by: Arthur <learthurgo@gmail.com> * changes to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * removed cells from visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * more changes to visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * added analysis-master/metrics/__pycache__ to git ignore moved pit configs in config.json to the borrom superscript.py v 0.0.6.001 Signed-off-by: Arthur <learthurgo@gmail.com> * removed old database key Signed-off-by: Arthur <learthurgo@gmail.com> * adjusted config files Signed-off-by: Arthur <learthurgo@gmail.com> * Delete config-pop.json * fixed .gitignore Signed-off-by: Arthur <learthurgo@gmail.com> * analysis.py 1.2.1.003 added team kv pair to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.002 Signed-off-by: Arthur <learthurgo@gmail.com> * finished app.py API made minute changes to parentheses use in various packages Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * deleted a __pycache__ folder from metrics Signed-off-by: Arthur Lu <learthurgo@gmail.com> * more changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * additions to app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed app.py to api.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed extranneous files Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed api.py to tra.py removed rest api calls from tra.py * renamed api.py to tra.py removed rest api calls from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed flask import from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * changes to devcontainer.json Signed-off-by: Arthur Lu <learthurgo@gmail.com> * fixed unit tests to be correct removed some tests regressions because of potential function overflow removed trueskill unit test because of slight deviation chance Signed-off-by: Arthur Lu <learthurgo@gmail.com>
2020-05-20 13:52:38 +00:00
db = client.data_processing
mdata = db.team_pit
out = {}
return mdata.find()
def get_pit_variable_formatted(client, competition):
temp = get_pit_variable_data(client, competition)
Merge service-dev changes with master (#24) * added config.json removed old config files Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.000 Signed-off-by: Arthur <learthurgo@gmail.com> * changed data.py Signed-off-by: Arthur <learthurgo@gmail.com> * changes to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * removed cells from visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * more changes to visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * added analysis-master/metrics/__pycache__ to git ignore moved pit configs in config.json to the borrom superscript.py v 0.0.6.001 Signed-off-by: Arthur <learthurgo@gmail.com> * removed old database key Signed-off-by: Arthur <learthurgo@gmail.com> * adjusted config files Signed-off-by: Arthur <learthurgo@gmail.com> * Delete config-pop.json * fixed .gitignore Signed-off-by: Arthur <learthurgo@gmail.com> * analysis.py 1.2.1.003 added team kv pair to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.002 Signed-off-by: Arthur <learthurgo@gmail.com> * finished app.py API made minute changes to parentheses use in various packages Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * deleted a __pycache__ folder from metrics Signed-off-by: Arthur Lu <learthurgo@gmail.com> * more changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * additions to app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed app.py to api.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed extranneous files Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed api.py to tra.py removed rest api calls from tra.py * renamed api.py to tra.py removed rest api calls from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed flask import from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * changes to devcontainer.json Signed-off-by: Arthur Lu <learthurgo@gmail.com> * fixed unit tests to be correct removed some tests regressions because of potential function overflow removed trueskill unit test because of slight deviation chance Signed-off-by: Arthur Lu <learthurgo@gmail.com>
2020-05-20 13:52:38 +00:00
out = {}
for i in temp:
out[i["variable"]] = i["data"]
return out
def push_team_tests_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_tests"):
db = client[dbname]
mdata = db[colname]
mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "data" : data}, True)
def push_team_metrics_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"):
db = client[dbname]
mdata = db[colname]
mdata.update_one({"team": team_num}, {"$set": {"metrics.{}".format(competition): data}}, upsert=True)
def push_team_pit_data(client, competition, variable, data, dbname = "data_processing", colname = "team_pit"):
db = client[dbname]
mdata = db[colname]
mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True)
def get_analysis_flags(client, flag):
db = client.data_processing
mdata = db.flags
2022-04-08 20:40:33 +00:00
return mdata.find_one({"_id": "2022"})
def set_analysis_flags(client, flag, data):
db = client.data_processing
mdata = db.flags
2022-04-08 20:40:33 +00:00
return mdata.update_one({"_id": "2022"}, {"$set": data})
Merge service-dev changes with master (#24) * added config.json removed old config files Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.000 Signed-off-by: Arthur <learthurgo@gmail.com> * changed data.py Signed-off-by: Arthur <learthurgo@gmail.com> * changes to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * removed cells from visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * more changes to visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * added analysis-master/metrics/__pycache__ to git ignore moved pit configs in config.json to the borrom superscript.py v 0.0.6.001 Signed-off-by: Arthur <learthurgo@gmail.com> * removed old database key Signed-off-by: Arthur <learthurgo@gmail.com> * adjusted config files Signed-off-by: Arthur <learthurgo@gmail.com> * Delete config-pop.json * fixed .gitignore Signed-off-by: Arthur <learthurgo@gmail.com> * analysis.py 1.2.1.003 added team kv pair to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.002 Signed-off-by: Arthur <learthurgo@gmail.com> * finished app.py API made minute changes to parentheses use in various packages Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * deleted a __pycache__ folder from metrics Signed-off-by: Arthur Lu <learthurgo@gmail.com> * more changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * additions to app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed app.py to api.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed extranneous files Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed api.py to tra.py removed rest api calls from tra.py * renamed api.py to tra.py removed rest api calls from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed flask import from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * changes to devcontainer.json Signed-off-by: Arthur Lu <learthurgo@gmail.com> * fixed unit tests to be correct removed some tests regressions because of potential function overflow removed trueskill unit test because of slight deviation chance Signed-off-by: Arthur Lu <learthurgo@gmail.com>
2020-05-20 13:52:38 +00:00
def unkeyify_2l(layered_dict):
out = {}
for i in layered_dict.keys():
add = []
sortkey = []
for j in layered_dict[i].keys():
add.append([j,layered_dict[i][j]])
add.sort(key = lambda x: x[0])
Merge service-dev changes with master (#24) * added config.json removed old config files Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.000 Signed-off-by: Arthur <learthurgo@gmail.com> * changed data.py Signed-off-by: Arthur <learthurgo@gmail.com> * changes to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * removed cells from visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * more changes to visualize_pit.py Signed-off-by: Arthur <learthurgo@gmail.com> * added analysis-master/metrics/__pycache__ to git ignore moved pit configs in config.json to the borrom superscript.py v 0.0.6.001 Signed-off-by: Arthur <learthurgo@gmail.com> * removed old database key Signed-off-by: Arthur <learthurgo@gmail.com> * adjusted config files Signed-off-by: Arthur <learthurgo@gmail.com> * Delete config-pop.json * fixed .gitignore Signed-off-by: Arthur <learthurgo@gmail.com> * analysis.py 1.2.1.003 added team kv pair to config.json Signed-off-by: Arthur <learthurgo@gmail.com> * superscript.py v 0.0.6.002 Signed-off-by: Arthur <learthurgo@gmail.com> * finished app.py API made minute changes to parentheses use in various packages Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * bug fixes in app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * made changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * deleted a __pycache__ folder from metrics Signed-off-by: Arthur Lu <learthurgo@gmail.com> * more changes to .gitignore Signed-off-by: Arthur Lu <learthurgo@gmail.com> * additions to app.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed app.py to api.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed extranneous files Signed-off-by: Arthur Lu <learthurgo@gmail.com> * renamed api.py to tra.py removed rest api calls from tra.py * renamed api.py to tra.py removed rest api calls from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * removed flask import from tra.py Signed-off-by: Arthur Lu <learthurgo@gmail.com> * changes to devcontainer.json Signed-off-by: Arthur Lu <learthurgo@gmail.com> * fixed unit tests to be correct removed some tests regressions because of potential function overflow removed trueskill unit test because of slight deviation chance Signed-off-by: Arthur Lu <learthurgo@gmail.com>
2020-05-20 13:52:38 +00:00
out[i] = list(map(lambda x: x[1], add))
return out
def get_previous_time(client):
previous_time = get_analysis_flags(client, "latest_update")
if previous_time == None:
set_analysis_flags(client, "latest_update", 0)
previous_time = 0
else:
previous_time = previous_time["latest_update"]
return previous_time
def set_current_time(client, current_time):
set_analysis_flags(client, "latest_update", {"latest_update":current_time})
def get_database_config(client):
remote_config = get_analysis_flags(client, "config")
return remote_config["config"] if remote_config != None else None
def set_database_config(client, config):
set_analysis_flags(client, "config", {"config": config})
def load_match(client, competition):
return get_match_data_formatted(client, competition)
def load_metric(client, competition, match, group_name, metrics):
group = {}
for team in match[group_name]:
db_data = get_team_metrics_data(client, competition, team)
if db_data == None:
gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]}
2022-03-29 04:17:25 +00:00
group[team] = {"gl2": gl2}
else:
metrics = db_data
gl2 = metrics["gl2"]
2022-03-29 04:17:25 +00:00
group[team] = {"gl2": gl2}
return group
def load_pit(client, competition):
return get_pit_data_formatted(client, competition)
def push_match(client, competition, results):
for team in results:
push_team_tests_data(client, competition, team, results[team])
def push_metric(client, competition, metric):
for team in metric:
push_team_metrics_data(client, competition, team, metric[team])
def push_pit(client, competition, pit):
for variable in pit:
push_team_pit_data(client, competition, variable, pit[variable])
def check_new_database_matches(client, competition):
2022-03-25 18:39:24 +00:00
return True