diff --git a/.gitignore b/.gitignore index 89255a74..1b800b48 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,6 @@ data analysis/.ipynb_checkpoints/test-checkpoint.ipynb .vscode data analysis/arthur_pull.ipynb data analysis/keys.txt -data analysis/check_for_new_matches.ipynb \ No newline at end of file +data analysis/check_for_new_matches.ipynb +data analysis/test.ipynb +data analysis/visualize_pit.ipynb \ No newline at end of file diff --git a/data analysis/__pycache__/data.cpython-37.pyc b/data analysis/__pycache__/data.cpython-37.pyc index e331827b..6b91e2a6 100644 Binary files a/data analysis/__pycache__/data.cpython-37.pyc and b/data analysis/__pycache__/data.cpython-37.pyc differ diff --git a/data analysis/config.csv b/data analysis/config.csv index 1022fc2f..d1975c6a 100644 --- a/data analysis/config.csv +++ b/data analysis/config.csv @@ -3,4 +3,11 @@ balls-blocked,basic_stats,historical_analysis,regression_linear,regression_logar balls-collected,basic_stats,historical_analysis,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal balls-lower,basic_stats,historical_analysis,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal balls-started,basic_stats,historical_analyss,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal -balls-upper,basic_stats,historical_analysis,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal \ No newline at end of file +balls-upper,basic_stats,historical_analysis,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal +wheel-mechanism +low-balls +high-balls +wheel-success +strategic-focus +climb-mechanism +attitude \ No newline at end of file diff --git a/data analysis/data.py b/data analysis/data.py index 9d51455d..b7fec8b5 100644 --- a/data analysis/data.py +++ b/data analysis/data.py @@ -8,7 +8,7 @@ def pull_new_tba_matches(apikey, competition, cutoff): x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth_Key":api_key}) out = [] for i in x.json(): - if (i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm"): + if (i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm"): out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) return out @@ -21,6 +21,13 @@ def get_team_match_data(apikey, competition, team_num): out[i['match']] = i['data'] return pd.DataFrame(out) +def get_team_pit_data(apikey, competition, team_num): + client = pymongo.MongoClient(apikey) + db = client.data_scouting + mdata = db.pitdata + out = {} + return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"] + def get_team_metrics_data(apikey, competition, team_num): client = pymongo.MongoClient(apikey) db = client.data_processing @@ -38,7 +45,7 @@ def unkeyify_2l(layered_dict): out[i] = list(map(lambda x: x[1], add)) return out -def get_data_formatted(apikey, competition): +def get_match_data_formatted(apikey, competition): client = pymongo.MongoClient(apikey) db = client.data_scouting mdata = db.teamlist @@ -51,6 +58,19 @@ def get_data_formatted(apikey, competition): pass return out +def get_pit_data_formatted(apikey, competition): + client = pymongo.MongoClient(apikey) + db = client.data_scouting + mdata = db.teamlist + x=mdata.find_one({"competition":competition}) + out = {} + for i in x: + try: + out[int(i)] = get_team_pit_data(apikey, competition, int(i)) + except: + pass + return out + def push_team_tests_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_tests"): client = pymongo.MongoClient(apikey) db = client[dbname] @@ -63,6 +83,12 @@ def push_team_metrics_data(apikey, competition, team_num, data, dbname = "data_p mdata = db[colname] mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True) +def push_team_pit_data(apikey, competition, variable, data, dbname = "data_processing", colname = "team_pit"): + client = pymongo.MongoClient(apikey) + db = client[dbname] + mdata = db[colname] + mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True) + def get_analysis_flags(apikey, flag): client = pymongo.MongoClient(apikey) db = client.data_processing diff --git a/data analysis/superscript.py b/data analysis/superscript.py index a2bece23..3a3c590c 100644 --- a/data analysis/superscript.py +++ b/data analysis/superscript.py @@ -3,10 +3,12 @@ # Notes: # setup: -__version__ = "0.0.2.001" +__version__ = "0.0.3.000" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 0.0.3.00: + - added analysis to pit data 0.0.2.001: - minor stability patches - implemented db syncing for timestamps @@ -69,6 +71,7 @@ __all__ = [ from analysis import analysis as an import data as d +import matplotlib.pyplot as plt import time import warnings @@ -102,7 +105,8 @@ def main(): print(" analysis backtimed to: " + str(previous_time)) print(" loading data") - data = d.get_data_formatted(apikey, competition) + data = d.get_match_data_formatted(apikey, competition) + pit_data = d.pit = d.get_pit_data_formatted(apikey, competition) print(" loaded data") print(" running tests") @@ -113,10 +117,14 @@ def main(): metrics = metricsloop(tbakey, apikey, competition, previous_time) print(" finished metrics") + print(" running pit analysis") + pit = pitloop(pit_data, config) + print(" finished pit analysis") + d.set_analysis_flags(apikey, "latest_update", {"latest_update":current_time}) print(" pushing to database") - push_to_database(apikey, competition, results, metrics) + push_to_database(apikey, competition, results, metrics, pit) print(" pushed to database") def load_config(file): @@ -168,7 +176,7 @@ def simplestats(data, test): if(test == "regression_sigmoidal"): return an.regression(list(range(len(data))), data, ['sig']) -def push_to_database(apikey, competition, results, metrics): +def push_to_database(apikey, competition, results, metrics, pit): for team in results: @@ -178,6 +186,10 @@ def push_to_database(apikey, competition, results, metrics): d.push_team_metrics_data(apikey, competition, team, metrics[team]) + for variable in pit: + + d.push_team_pit_data(apikey, competition, variable, pit[variable]) + def metricsloop(tbakey, apikey, competition, timestamp): # listener based metrics update elo_N = 400 @@ -327,6 +339,18 @@ def load_metrics(apikey, competition, match, group_name): return group +def pitloop(pit, tests): + + return_vector = {} + for team in pit: + for variable in pit[team]: + if(variable in tests): + if(not variable in return_vector): + return_vector[variable] = [] + return_vector[variable].append(pit[team][variable]) + + return return_vector + main() """ diff --git a/data analysis/visualize_pit.py b/data analysis/visualize_pit.py new file mode 100644 index 00000000..3305ee4d --- /dev/null +++ b/data analysis/visualize_pit.py @@ -0,0 +1,59 @@ +# To add a new cell, type '# %%' +# To add a new markdown cell, type '# %% [markdown]' +# %% +import matplotlib.pyplot as plt +import data as d +import pymongo + + +# %% +def get_pit_variable_data(apikey, competition): + client = pymongo.MongoClient(apikey) + db = client.data_processing + mdata = db.team_pit + out = {} + return mdata.find() + + +# %% +def get_pit_variable_formatted(apikey, competition): + temp = get_pit_variable_data(apikey, competition) + out = {} + for i in temp: + out[i["variable"]] = i["data"] + return out + + +# %% +pit = get_pit_variable_formatted("mongodb+srv://api-user-new:titanscout2022@2022-scouting-4vfuu.mongodb.net/test?authSource=admin&replicaSet=2022-scouting-shard-0&readPreference=primary&appname=MongoDB%20Compass&ssl=true", "2020ilch") + + +# %% +import matplotlib.pyplot as plt +import numpy as np + + +# %% +fig, ax = plt.subplots(1, len(pit), sharey=True, figsize=(20,10)) + +i = 0 + +for variable in pit: + + ax[i].hist(pit[variable]) + ax[i].invert_xaxis() + + ax[i].set_xlabel('') + ax[i].set_ylabel('Frequency') + ax[i].set_title(variable) + + plt.yticks(np.arange(len(pit[variable]))) + + i+=1 + +plt.show() + + +# %% + +