From 25a05beca22d3144b997feefca767fc209b76bb5 Mon Sep 17 00:00:00 2001 From: art Date: Thu, 5 Mar 2020 22:52:02 -0600 Subject: [PATCH] superscript.py v 0.0.3.000 --- .gitignore | 4 +- data analysis/__pycache__/data.cpython-37.pyc | Bin 3415 -> 4251 bytes data analysis/config.csv | 9 ++- data analysis/data.py | 30 ++++++++- data analysis/superscript.py | 32 ++++++++-- data analysis/visualize_pit.py | 59 ++++++++++++++++++ 6 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 data analysis/visualize_pit.py diff --git a/.gitignore b/.gitignore index 89255a74..1b800b48 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,6 @@ data analysis/.ipynb_checkpoints/test-checkpoint.ipynb .vscode data analysis/arthur_pull.ipynb data analysis/keys.txt -data analysis/check_for_new_matches.ipynb \ No newline at end of file +data analysis/check_for_new_matches.ipynb +data analysis/test.ipynb +data analysis/visualize_pit.ipynb \ No newline at end of file diff --git a/data analysis/__pycache__/data.cpython-37.pyc b/data analysis/__pycache__/data.cpython-37.pyc index e331827b5a2fdf92843f071ffaa2e677bebcc0ca..6b91e2a6c1a813ac165256db5b99aff44299069a 100644 GIT binary patch delta 1561 zcmaJ>%}*0S6yMp;=_gRM($a!7NLek!#E3xw5x>zG6F(04(JY;!)fUL?YSe7QK@P?k z4Vjpjh!HNHr$wZiSvh6GmT7>OnZ7cy=lqFd^ ztZHH@mWFRLBNrsIYhf`(-Y%^6Bp$iylDsS(ldic{-PM_vkeGB^;klKcZ)DOdgz|uR zBC39z>jfYM1>wMBz|deEg<7UgBa zATjX!o~vqqYl0 z4~i^`gjfl+W?OC)oxD@DDtT~`M;~oqw0$y3km&HLgzpw#)ysMQxMXcKi)6iYu`vtywdNHJ`p#{X@q9S{52|xLPV9sNRl+e-~-~d@uIu& zz#c`&<&y3AK==S`<-OtxHA8i$6whcT*@g$KSWe-lf4cnC42M6b$o7g4w37^i_XJ^L z)ySm>Tn)0@Up4Nh9L<$n;^?mNkUUge-O)vqW`~R|#Pb6f7`foPQRpB{TF1&3G{bR3 zv(EG;mSNauW4hKkLJopNCMn=QNccgp@SONT-QGQ*da_k9RZq#6SS`Oe>6gYuQ(c27 zo(3{Ke%`TUJe^HWTa}_`6lTh()K`O_s_%jA?c=a=BUO}8!pG2Y9|-Y_R;K(gL+}St z5b|bl&!`j~UMkp|0MI^y;xKLq)RD4*ijSiG2#R{5uyTF}cN{nmhDJEYV5XLPB(Z>W z;}d{^edhPrLD3x8Gv~^#!sJDTDa#6hMukpN*E`9hcP(DBCd)-rsU6`2Y+mC}sY@}! zZ$Q4W?%&AK3syR(;$0}XYxa(Pv$3}>gR${MCxschv~RPzbFjkGrpne0yVhfxPn@UX z)E1U{TCdjo!(Qd%(5VTbgh%%``sAk&mw%wOu04$tSQ}fsZWB+!vAM~=JfDFXxQY-U zKUAk9f`}vuvKhAJr*A;kBa?ugF@<;FVbG-$B&;N*|Np65^O_ mei8-NCCB(cEEY#!B$`3@SfcRCGytzbbsC}(8VTz7QvDAym?Pf+ delta 914 zcmZWo&1(}u6yHfUlg+NXiAhY;Hf?BZH(Bk?_-#vVi+Ctr)PpG+*36~^vPqcTgGfY> z{sUnyLJ<|jyXeJ>co)1>5cc3*@ITOl?@g^ubcXrO+c)#x?|sbfEqtA^)=g7aVC>uO zy&uNNDw4sdx^-=eSuDLv8agYo<17o3!E!9Wt2Bgs1w6=r%`{7kISIWXb#VwF~^ zk=E|?ShL^r9Lfu*pF%i^u!yjPV2dZz$rV8iGysL@gu>*@;sgCgmc+B<17eGU)}5Qj zJ6U;0B^FU(X2}f6z~XiBTzk5BKn^!}8K9wsp6hozLC8;|u8vR@+bPEwr+=eY^4T3P zY%(`=o9#aL-7xf+BR-|d^c>naB4Z3|H4vki>pN-`Z+6&0GN|MwfLo5r|K$G8sMs<| z)JK%v@43DgC0qS23KOaSGHSC(@i%piXoRn#(^CMVZ1`MD#~b*z8CUN zD>x#@^GgVq(IEA2*)bx11z`nYybK@pys*te4o@e;Src2K#nXm{P6Ko-_X zNSa7R)r8M4mYrK4_a{5M0gvU**-3x=D#nA6AHEf|la-zDxRhL$<$iX!Lv7MPO;d{ki R!Hj7sm{c<~Q>QxB=x?Frqm}>w diff --git a/data analysis/config.csv b/data analysis/config.csv index 1022fc2f..d1975c6a 100644 --- a/data analysis/config.csv +++ b/data analysis/config.csv @@ -3,4 +3,11 @@ balls-blocked,basic_stats,historical_analysis,regression_linear,regression_logar balls-collected,basic_stats,historical_analysis,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal balls-lower,basic_stats,historical_analysis,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal balls-started,basic_stats,historical_analyss,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal -balls-upper,basic_stats,historical_analysis,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal \ No newline at end of file +balls-upper,basic_stats,historical_analysis,regression_linear,regression_logarithmic,regression_exponential,regression_polynomial,regression_sigmoidal +wheel-mechanism +low-balls +high-balls +wheel-success +strategic-focus +climb-mechanism +attitude \ No newline at end of file diff --git a/data analysis/data.py b/data analysis/data.py index 9d51455d..b7fec8b5 100644 --- a/data analysis/data.py +++ b/data analysis/data.py @@ -8,7 +8,7 @@ def pull_new_tba_matches(apikey, competition, cutoff): x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth_Key":api_key}) out = [] for i in x.json(): - if (i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm"): + if (i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm"): out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) return out @@ -21,6 +21,13 @@ def get_team_match_data(apikey, competition, team_num): out[i['match']] = i['data'] return pd.DataFrame(out) +def get_team_pit_data(apikey, competition, team_num): + client = pymongo.MongoClient(apikey) + db = client.data_scouting + mdata = db.pitdata + out = {} + return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"] + def get_team_metrics_data(apikey, competition, team_num): client = pymongo.MongoClient(apikey) db = client.data_processing @@ -38,7 +45,7 @@ def unkeyify_2l(layered_dict): out[i] = list(map(lambda x: x[1], add)) return out -def get_data_formatted(apikey, competition): +def get_match_data_formatted(apikey, competition): client = pymongo.MongoClient(apikey) db = client.data_scouting mdata = db.teamlist @@ -51,6 +58,19 @@ def get_data_formatted(apikey, competition): pass return out +def get_pit_data_formatted(apikey, competition): + client = pymongo.MongoClient(apikey) + db = client.data_scouting + mdata = db.teamlist + x=mdata.find_one({"competition":competition}) + out = {} + for i in x: + try: + out[int(i)] = get_team_pit_data(apikey, competition, int(i)) + except: + pass + return out + def push_team_tests_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_tests"): client = pymongo.MongoClient(apikey) db = client[dbname] @@ -63,6 +83,12 @@ def push_team_metrics_data(apikey, competition, team_num, data, dbname = "data_p mdata = db[colname] mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True) +def push_team_pit_data(apikey, competition, variable, data, dbname = "data_processing", colname = "team_pit"): + client = pymongo.MongoClient(apikey) + db = client[dbname] + mdata = db[colname] + mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True) + def get_analysis_flags(apikey, flag): client = pymongo.MongoClient(apikey) db = client.data_processing diff --git a/data analysis/superscript.py b/data analysis/superscript.py index a2bece23..3a3c590c 100644 --- a/data analysis/superscript.py +++ b/data analysis/superscript.py @@ -3,10 +3,12 @@ # Notes: # setup: -__version__ = "0.0.2.001" +__version__ = "0.0.3.000" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 0.0.3.00: + - added analysis to pit data 0.0.2.001: - minor stability patches - implemented db syncing for timestamps @@ -69,6 +71,7 @@ __all__ = [ from analysis import analysis as an import data as d +import matplotlib.pyplot as plt import time import warnings @@ -102,7 +105,8 @@ def main(): print(" analysis backtimed to: " + str(previous_time)) print(" loading data") - data = d.get_data_formatted(apikey, competition) + data = d.get_match_data_formatted(apikey, competition) + pit_data = d.pit = d.get_pit_data_formatted(apikey, competition) print(" loaded data") print(" running tests") @@ -113,10 +117,14 @@ def main(): metrics = metricsloop(tbakey, apikey, competition, previous_time) print(" finished metrics") + print(" running pit analysis") + pit = pitloop(pit_data, config) + print(" finished pit analysis") + d.set_analysis_flags(apikey, "latest_update", {"latest_update":current_time}) print(" pushing to database") - push_to_database(apikey, competition, results, metrics) + push_to_database(apikey, competition, results, metrics, pit) print(" pushed to database") def load_config(file): @@ -168,7 +176,7 @@ def simplestats(data, test): if(test == "regression_sigmoidal"): return an.regression(list(range(len(data))), data, ['sig']) -def push_to_database(apikey, competition, results, metrics): +def push_to_database(apikey, competition, results, metrics, pit): for team in results: @@ -178,6 +186,10 @@ def push_to_database(apikey, competition, results, metrics): d.push_team_metrics_data(apikey, competition, team, metrics[team]) + for variable in pit: + + d.push_team_pit_data(apikey, competition, variable, pit[variable]) + def metricsloop(tbakey, apikey, competition, timestamp): # listener based metrics update elo_N = 400 @@ -327,6 +339,18 @@ def load_metrics(apikey, competition, match, group_name): return group +def pitloop(pit, tests): + + return_vector = {} + for team in pit: + for variable in pit[team]: + if(variable in tests): + if(not variable in return_vector): + return_vector[variable] = [] + return_vector[variable].append(pit[team][variable]) + + return return_vector + main() """ diff --git a/data analysis/visualize_pit.py b/data analysis/visualize_pit.py new file mode 100644 index 00000000..3305ee4d --- /dev/null +++ b/data analysis/visualize_pit.py @@ -0,0 +1,59 @@ +# To add a new cell, type '# %%' +# To add a new markdown cell, type '# %% [markdown]' +# %% +import matplotlib.pyplot as plt +import data as d +import pymongo + + +# %% +def get_pit_variable_data(apikey, competition): + client = pymongo.MongoClient(apikey) + db = client.data_processing + mdata = db.team_pit + out = {} + return mdata.find() + + +# %% +def get_pit_variable_formatted(apikey, competition): + temp = get_pit_variable_data(apikey, competition) + out = {} + for i in temp: + out[i["variable"]] = i["data"] + return out + + +# %% +pit = get_pit_variable_formatted("mongodb+srv://api-user-new:titanscout2022@2022-scouting-4vfuu.mongodb.net/test?authSource=admin&replicaSet=2022-scouting-shard-0&readPreference=primary&appname=MongoDB%20Compass&ssl=true", "2020ilch") + + +# %% +import matplotlib.pyplot as plt +import numpy as np + + +# %% +fig, ax = plt.subplots(1, len(pit), sharey=True, figsize=(20,10)) + +i = 0 + +for variable in pit: + + ax[i].hist(pit[variable]) + ax[i].invert_xaxis() + + ax[i].set_xlabel('') + ax[i].set_ylabel('Frequency') + ax[i].set_title(variable) + + plt.yticks(np.arange(len(pit[variable]))) + + i+=1 + +plt.show() + + +# %% + +