alu/tra-analysis
1
0
Fork 0
mirror of https://github.com/titanscouting/tra-analysis.git synced 2024-09-20 08:57:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

beautified

Browse Source
This commit is contained in:
ltcptgeneral 2019-04-03 13:34:31 -05:00
parent ef81273d4a
commit 20f082b760
2 changed files with 234 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
data analysis/superscript.py
View File

@ -60,6 +60,7 @@ import time
import tbarequest as tba import tbarequest as tba
import csv import csv
def titanservice(): def titanservice():
print("[OK] loading data") print("[OK] loading data")
@ -67,7 +68,8 @@ def titanservice():
start = time.time() start = time.time()
source_dir = 'data' source_dir = 'data'
file_list = glob.glob(source_dir + '/*.csv') #supposedly sorts by alphabetical order, skips reading teams.csv because of redundancy # supposedly sorts by alphabetical order, skips reading teams.csv because of redundancy
file_list = glob.glob(source_dir + '/*.csv')
data = [] data = []
files = [fn for fn in glob.glob('data/*.csv') files = [fn for fn in glob.glob('data/*.csv')
if not (os.path.basename(fn).startswith('scores') or os.path.basename(fn).startswith('teams') or os.path.basename(fn).startswith('match') or os.path.basename(fn).startswith('notes') or os.path.basename(fn).startswith('observationType') or os.path.basename(fn).startswith('teamDBRef'))] # scores will be handled sperately if not (os.path.basename(fn).startswith('scores') or os.path.basename(fn).startswith('teams') or os.path.basename(fn).startswith('match') or os.path.basename(fn).startswith('notes') or os.path.basename(fn).startswith('observationType') or os.path.basename(fn).startswith('teamDBRef'))] # scores will be handled sperately
@ -126,8 +128,8 @@ def titanservice():
# print(r2best_curve) # print(r2best_curve)
measure_stats.append(teams[i] + list(analysis.basic_stats(
measure_stats.append(teams[i] + list(analysis.basic_stats(line, 0, 0)) + list(analysis.histo_analysis(line, 1, -3, 3))) line, 0, 0)) + list(analysis.histo_analysis(line, 1, -3, 3)))
stats.append(list(measure_stats)) stats.append(list(measure_stats))
nishant = [] nishant = []
@ -154,7 +156,8 @@ def titanservice():
x = list(range(len(line))) x = list(range(len(line)))
eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1) eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1)
beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "min_overfit") beqs, brmss, br2s, boverfit = analysis.select_best_regression(
eqs, rmss, r2s, overfit, "min_overfit")
#print(eqs, rmss, r2s, overfit) #print(eqs, rmss, r2s, overfit)
@ -166,7 +169,8 @@ def titanservice():
# print(ofbest_curve) # print(ofbest_curve)
beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "max_r2s") beqs, brmss, br2s, boverfit = analysis.select_best_regression(
eqs, rmss, r2s, overfit, "max_r2s")
r2best_curve.append(beqs) r2best_curve.append(beqs)
r2best_curve.append(brmss) r2best_curve.append(brmss)
@ -199,10 +203,13 @@ def titanservice():
for j in range(len(files)): for j in range(len(files)):
json_out[str(teams[i][0])] = (stats[j][i]) json_out[str(teams[i][0])] = (stats[j][i])
name = os.path.basename(files[j]) name = os.path.basename(files[j])
general_general_stats.document(name).set({'stats':json_out.get(teams[i][0])}) general_general_stats.document(name).set(
{'stats': json_out.get(teams[i][0])})
for i in range(len(teams)): for i in range(len(teams)):
nnum = location.collection(teams[i][0]).document(u'nishant_number').set({'nishant':score_out.get(teams[i][0])}) nnum = location.collection(teams[i][0]).document(
u'nishant_number').set({'nishant': score_out.get(teams[i][0])})
def pulldata(): def pulldata():
teams = analysis.load_csv('data/teams.csv') teams = analysis.load_csv('data/teams.csv')
@ -210,7 +217,8 @@ def pulldata():
for i in range(len(teams)): for i in range(len(teams)):
team_scores = [] team_scores = []
# print(teams[i][0]) # print(teams[i][0])
request_data_object = tba.req_team_matches(teams[i][0], 2019, "UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5") request_data_object = tba.req_team_matches(
teams[i][0], 2019, "UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5")
json_data = request_data_object.json() json_data = request_data_object.json()
for match in range(len(json_data) - 1, -1, -1): for match in range(len(json_data) - 1, -1, -1):
@ -218,12 +226,15 @@ def pulldata():
# print(json_data[match]) # print(json_data[match])
json_data.remove(json_data[match]) json_data.remove(json_data[match])
json_data = sorted(json_data, key=lambda k: k.get('actual_time', 0), reverse=False) json_data = sorted(json_data, key=lambda k: k.get(
'actual_time', 0), reverse=False)
for j in range(len(json_data)): for j in range(len(json_data)):
if "frc" + teams[i][0] in json_data[j].get('alliances').get('blue').get('team_keys'): if "frc" + teams[i][0] in json_data[j].get('alliances').get('blue').get('team_keys'):
team_scores.append(json_data[j].get('alliances').get('blue').get('score')) team_scores.append(json_data[j].get(
'alliances').get('blue').get('score'))
elif "frc" + teams[i][0] in json_data[j].get('alliances').get('red').get('team_keys'): elif "frc" + teams[i][0] in json_data[j].get('alliances').get('red').get('team_keys'):
team_scores.append(json_data[j].get('alliances').get('red').get('score')) team_scores.append(json_data[j].get(
'alliances').get('red').get('score'))
scores.append(team_scores) scores.append(team_scores)
with open("data/scores.csv", "w+", newline='') as file: with open("data/scores.csv", "w+", newline='') as file:
@ -231,17 +242,20 @@ def pulldata():
writer.writerows(scores) writer.writerows(scores)
list_teams = teams list_teams = teams
teams=db.collection('data').document('team-2022').collection("Central 2019").get() teams = db.collection('data').document(
'team-2022').collection("Central 2019").get()
full = [] full = []
tms = [] tms = []
for team in teams: for team in teams:
tms.append(team.id) tms.append(team.id)
reports=db.collection('data').document('team-2022').collection("Central 2019").document(team.id).collection("matches").get() reports = db.collection('data').document(
'team-2022').collection("Central 2019").document(team.id).collection("matches").get()
for report in reports: for report in reports:
data = [] data = []
data.append(db.collection('data').document('team-2022').collection("Central 2019").document(team.id).collection("matches").document(report.id).get().to_dict()) data.append(db.collection('data').document('team-2022').collection("Central 2019").document(
team.id).collection("matches").document(report.id).get().to_dict())
full.append(data) full.append(data)
quant_keys = [] quant_keys = []
@ -282,7 +296,8 @@ def pulldata():
individual_keys = list(full[i][j].get(key).keys()) individual_keys = list(full[i][j].get(key).keys())
var[individual_keys[k]] = full[i][j].get(key).get(individual_keys[k]) var[individual_keys[k]] = full[i][j].get(
key).get(individual_keys[k])
out.append(var) out.append(var)
@ -316,7 +331,8 @@ def pulldata():
for i in sorted_out: for i in sorted_out:
team_index = list_teams.index(sorted_out[sorted_out.index(i)][j_list.index('teamDBRef')][5:]) team_index = list_teams.index(
sorted_out[sorted_out.index(i)][j_list.index('teamDBRef')][5:])
for j in range(len(i)): for j in range(len(i)):
@ -329,6 +345,7 @@ def pulldata():
writer = csv.writer(file, delimiter=',') writer = csv.writer(file, delimiter=',')
writer.writerows(big_out[i]) writer.writerows(big_out[i])
def service(): def service():
while True: while True:
@ -347,7 +364,8 @@ def service():
break break
except: except:
if (i != 4): if (i != 4):
print("[WARNING] failed, trying " + str(5 - i - 1) + " more times") print("[WARNING] failed, trying " +
str(5 - i - 1) + " more times")
else: else:
print("[ERROR] failed to compute data, skipping") print("[ERROR] failed to compute data, skipping")
fucked = True fucked = True
@ -365,6 +383,7 @@ def service():
time.sleep(300 - (end - start)) # executes once every 5 minutes time.sleep(300 - (end - start)) # executes once every 5 minutes
warnings.simplefilter("ignore") warnings.simplefilter("ignore")
# Use a service account # Use a service account
try: try:

32
data analysis/superscript_nishant_only.py
View File

@ -60,6 +60,7 @@ import time
import tbarequest as tba import tbarequest as tba
import csv import csv
def titanservice(): def titanservice():
print("[OK] loading data") print("[OK] loading data")
@ -67,7 +68,8 @@ def titanservice():
start = time.time() start = time.time()
source_dir = 'data' source_dir = 'data'
file_list = glob.glob(source_dir + '/*.csv') #supposedly sorts by alphabetical order, skips reading teams.csv because of redundancy # supposedly sorts by alphabetical order, skips reading teams.csv because of redundancy
file_list = glob.glob(source_dir + '/*.csv')
data = [] data = []
files = [fn for fn in glob.glob('data/*.csv') files = [fn for fn in glob.glob('data/*.csv')
if not (os.path.basename(fn).startswith('scores') or os.path.basename(fn).startswith('teams') or os.path.basename(fn).startswith('match') or os.path.basename(fn).startswith('notes') or os.path.basename(fn).startswith('observationType') or os.path.basename(fn).startswith('teamDBRef'))] # scores will be handled sperately if not (os.path.basename(fn).startswith('scores') or os.path.basename(fn).startswith('teams') or os.path.basename(fn).startswith('match') or os.path.basename(fn).startswith('notes') or os.path.basename(fn).startswith('observationType') or os.path.basename(fn).startswith('teamDBRef'))] # scores will be handled sperately
@ -126,7 +128,6 @@ def titanservice():
# print(r2best_curve) # print(r2best_curve)
#measure_stats.append(teams[i] + list(analysis.basic_stats(line, 0, 0)) + list(analysis.histo_analysis(line, 1, -3, 3))) #measure_stats.append(teams[i] + list(analysis.basic_stats(line, 0, 0)) + list(analysis.histo_analysis(line, 1, -3, 3)))
# stats.append(list(measure_stats)) # stats.append(list(measure_stats))
@ -154,7 +155,8 @@ def titanservice():
x = list(range(len(line))) x = list(range(len(line)))
eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1) eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1)
beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "min_overfit") beqs, brmss, br2s, boverfit = analysis.select_best_regression(
eqs, rmss, r2s, overfit, "min_overfit")
#print(eqs, rmss, r2s, overfit) #print(eqs, rmss, r2s, overfit)
@ -166,7 +168,8 @@ def titanservice():
# print(ofbest_curve) # print(ofbest_curve)
beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "max_r2s") beqs, brmss, br2s, boverfit = analysis.select_best_regression(
eqs, rmss, r2s, overfit, "max_r2s")
r2best_curve.append(beqs) r2best_curve.append(beqs)
r2best_curve.append(brmss) r2best_curve.append(brmss)
@ -202,7 +205,9 @@ def titanservice():
# general_general_stats.document(name).set({'stats':json_out.get(teams[i][0])}) # general_general_stats.document(name).set({'stats':json_out.get(teams[i][0])})
for i in range(len(teams)): for i in range(len(teams)):
nnum = location.collection(teams[i][0]).document(u'nishant_number').set({'nishant':score_out.get(teams[i][0])}) nnum = location.collection(teams[i][0]).document(
u'nishant_number').set({'nishant': score_out.get(teams[i][0])})
def pulldata(): def pulldata():
teams = analysis.load_csv('data/teams.csv') teams = analysis.load_csv('data/teams.csv')
@ -210,7 +215,8 @@ def pulldata():
for i in range(len(teams)): for i in range(len(teams)):
team_scores = [] team_scores = []
# print(teams[i][0]) # print(teams[i][0])
request_data_object = tba.req_team_matches(teams[i][0], 2019, "UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5") request_data_object = tba.req_team_matches(
teams[i][0], 2019, "UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5")
json_data = request_data_object.json() json_data = request_data_object.json()
for match in range(len(json_data) - 1, -1, -1): for match in range(len(json_data) - 1, -1, -1):
@ -218,18 +224,22 @@ def pulldata():
# print(json_data[match]) # print(json_data[match])
json_data.remove(json_data[match]) json_data.remove(json_data[match])
json_data = sorted(json_data, key=lambda k: k.get('actual_time', 0), reverse=False) json_data = sorted(json_data, key=lambda k: k.get(
'actual_time', 0), reverse=False)
for j in range(len(json_data)): for j in range(len(json_data)):
if "frc" + teams[i][0] in json_data[j].get('alliances').get('blue').get('team_keys'): if "frc" + teams[i][0] in json_data[j].get('alliances').get('blue').get('team_keys'):
team_scores.append(json_data[j].get('alliances').get('blue').get('score')) team_scores.append(json_data[j].get(
'alliances').get('blue').get('score'))
elif "frc" + teams[i][0] in json_data[j].get('alliances').get('red').get('team_keys'): elif "frc" + teams[i][0] in json_data[j].get('alliances').get('red').get('team_keys'):
team_scores.append(json_data[j].get('alliances').get('red').get('score')) team_scores.append(json_data[j].get(
'alliances').get('red').get('score'))
scores.append(team_scores) scores.append(team_scores)
with open("data/scores.csv", "w+", newline='') as file: with open("data/scores.csv", "w+", newline='') as file:
writer = csv.writer(file, delimiter=',') writer = csv.writer(file, delimiter=',')
writer.writerows(scores) writer.writerows(scores)
def service(): def service():
while True: while True:
@ -248,7 +258,8 @@ def service():
break break
except: except:
if (i != 4): if (i != 4):
print("[WARNING] failed, trying " + str(5 - i - 1) + " more times") print("[WARNING] failed, trying " +
str(5 - i - 1) + " more times")
else: else:
print("[ERROR] failed to compute data, skipping") print("[ERROR] failed to compute data, skipping")
fucked = True fucked = True
@ -266,6 +277,7 @@ def service():
time.sleep(300 - (end - start)) # executes once every 5 minutes time.sleep(300 - (end - start)) # executes once every 5 minutes
warnings.simplefilter("ignore") warnings.simplefilter("ignore")
# Use a service account # Use a service account
try: try: