beautified

data analysis/.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,88 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tbarequest"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2022', 'frc2358', 'frc7417']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2040', 'frc2481', 'frc81']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm70', 'match_number': 70, 'post_result_time': None, 'predicted_time': 1553357160, 'score_breakdown': None, 'set_number': 1, 'time': 1553357160, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc6237', 'frc4156', 'frc4646']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc4241', 'frc1781', 'frc2022']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm61', 'match_number': 61, 'post_result_time': None, 'predicted_time': 1553352300, 'score_breakdown': None, 'set_number': 1, 'time': 1553352300, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc1736', 'frc2022', 'frc5126']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc5690', 'frc1329', 'frc4241']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm57', 'match_number': 57, 'post_result_time': None, 'predicted_time': 1553350140, 'score_breakdown': None, 'set_number': 1, 'time': 1553350140, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc6237', 'frc2358', 'frc2039']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2022', 'frc7848', 'frc4212']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm53', 'match_number': 53, 'post_result_time': None, 'predicted_time': 1553292960, 'score_breakdown': None, 'set_number': 1, 'time': 1553292960, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc7465', 'frc4213', 'frc2704']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc1329', 'frc6055', 'frc2022']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm41', 'match_number': 41, 'post_result_time': None, 'predicted_time': 1553287200, 'score_breakdown': None, 'set_number': 1, 'time': 1553287200, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2022', 'frc4156', 'frc5442']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc5822', 'frc4143', 'frc7848']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm4', 'match_number': 4, 'post_result_time': None, 'predicted_time': 1553264820, 'score_breakdown': None, 'set_number': 1, 'time': 1553264820, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc1094', 'frc4314', 'frc6651']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc6237', 'frc2081', 'frc2022']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm38', 'match_number': 38, 'post_result_time': None, 'predicted_time': 1553285760, 'score_breakdown': None, 'set_number': 1, 'time': 1553285760, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2013', 'frc2081', 'frc6055']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc81', 'frc2022', 'frc3695']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm31', 'match_number': 31, 'post_result_time': None, 'predicted_time': 1553282400, 'score_breakdown': None, 'set_number': 1, 'time': 1553282400, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc4655', 'frc5822', 'frc2022']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc4256', 'frc323', 'frc2709']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm22', 'match_number': 22, 'post_result_time': None, 'predicted_time': 1553278080, 'score_breakdown': None, 'set_number': 1, 'time': 1553278080, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc1756', 'frc1329', 'frc1288']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc4096', 'frc2022', 'frc323']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm14', 'match_number': 14, 'post_result_time': None, 'predicted_time': 1553269860, 'score_breakdown': None, 'set_number': 1, 'time': 1553269860, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2039', 'frc2022', 'frc1288']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2013', 'frc4096', 'frc1781']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm10', 'match_number': 10, 'post_result_time': None, 'predicted_time': 1553267940, 'score_breakdown': None, 'set_number': 1, 'time': 1553267940, 'videos': [], 'winning_alliance': ''}\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Response [404]>"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tbarequest.req_team_matches('frc16', '2019','UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

data analysis/.ipynb_checkpoints/superscript-checkpoint.py

@@ -0,0 +1,265 @@
+#Titan Robotics Team 2022: Super Script
+#Written by Arthur Lu & Jacob Levine
+#Notes:
+#setup:
+
+__version__ = "1.0.6.000"
+
+__changelog__ = """changelog:
+1.0.6.000:
+    - added pulldata function
+    - service now pulls in, computes data, and outputs data as planned
+1.0.5.003:
+    - hotfix: actually pushes data correctly now
+1.0.5.002:
+    - more information given
+    - performance improvements
+1.0.5.001:
+    - grammar
+1.0.5.000:
+    - service now iterates forever
+    - ready for production other than pulling json data
+1.0.4.001:
+    - grammar fixes
+1.0.4.000:
+    - actually pushes to firebase
+1.0.3.001:
+    - processes data more efficiently
+1.0.3.000:
+    - actually processes data
+1.0.2.000:
+    - added data reading from folder
+    - nearly crashed computer reading from 20 GiB of data
+1.0.1.000:
+    - added data reading from file
+    - added superstructure to code
+1.0.0.000:
+    - added import statements (revolutionary)
+""" 
+
+__author__ = (
+    "Arthur Lu <arthurlu@ttic.edu>, "
+    "Jacob Levine <jlevine@ttic.edu>,"
+    )
+
+import firebase_admin
+from firebase_admin import credentials
+from firebase_admin import firestore
+import analysis
+#import titanlearn
+import visualization
+import os
+import sys
+import warnings
+import glob
+import numpy as np
+import time
+import tbarequest as tba
+import csv
+
+def titanservice():
+    
+    print("[OK] loading data")
+
+    start = time.time()
+    
+    source_dir = 'data'
+    file_list = glob.glob(source_dir + '/*.csv') #supposedly sorts by alphabetical order, skips reading teams.csv because of redundancy
+    data = []
+    files = [fn for fn in glob.glob('data/*.csv') 
+             if not (os.path.basename(fn).startswith('teams'))] #scores will be handled sperately
+
+    for i in files:
+            data.append(analysis.load_csv(i))
+
+    stats = []
+    measure_stats = []
+    teams = analysis.load_csv("data/teams.csv")
+    scores = analysis.load_csv("data/scores.csv")
+
+    end = time.time()
+
+    print("[OK] loaded data in " + str(end - start) + " seconds")
+
+    #assumes that team number is in the first column, and that the order of teams is the same across all files
+    #unhelpful comment
+    for measure in data: #unpacks 3d array into 2ds
+
+        measure_stats = []
+
+        for i in range(len(measure)): #unpacks into specific teams
+
+                    ofbest_curve = [None]
+                    r2best_curve = [None]
+
+                    line = measure[i]
+
+                    #print(line)
+
+                    x = list(range(len(line)))
+                    eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1)
+
+                    beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "min_overfit")
+
+                    #print(eqs, rmss, r2s, overfit)
+                    
+                    ofbest_curve.append(beqs)
+                    ofbest_curve.append(brmss)
+                    ofbest_curve.append(br2s)
+                    ofbest_curve.append(boverfit)
+                    ofbest_curve.pop(0)
+
+                    #print(ofbest_curve)
+
+                    beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "max_r2s")
+
+                    r2best_curve.append(beqs)
+                    r2best_curve.append(brmss)
+                    r2best_curve.append(br2s)
+                    r2best_curve.append(boverfit)
+                    r2best_curve.pop(0)
+
+                    #print(r2best_curve)
+
+                    
+                    measure_stats.append(teams[i] + list(analysis.basic_stats(line, 0, 0)) + list(analysis.histo_analysis(line, 1, -3, 3)) + ofbest_curve + r2best_curve)
+
+        stats.append(list(measure_stats))
+        nishant = []
+            
+        for i in range(len(scores)):
+
+                    ofbest_curve = [None]
+                    r2best_curve = [None]
+
+                    line = measure[i]
+
+                    #print(line)
+
+                    x = list(range(len(line)))
+                    eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1)
+
+                    beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "min_overfit")
+
+                    #print(eqs, rmss, r2s, overfit)
+                    
+                    ofbest_curve.append(beqs)
+                    ofbest_curve.append(brmss)
+                    ofbest_curve.append(br2s)
+                    ofbest_curve.append(boverfit)
+                    ofbest_curve.pop(0)
+
+                    #print(ofbest_curve)
+
+                    beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "max_r2s")
+
+                    r2best_curve.append(beqs)
+                    r2best_curve.append(brmss)
+                    r2best_curve.append(br2s)
+                    r2best_curve.append(boverfit)
+                    r2best_curve.pop(0)
+
+                    #print(r2best_curve)
+                    
+                    z = len(scores[0]) + 1
+                    nis_num = []
+
+                    nis_num.append(eval(str(ofbest_curve[0])))
+                    nis_num.append(eval(str(r2best_curve[0])))
+
+                    nis_num.append((eval(ofbest_curve[0]) + eval(r2best_curve[0])) / 2)
+
+                    nishant.append(teams[i] + nis_num)
+                
+    json_out = {}
+    score_out = {}
+
+    for i in range(len(teams)):
+            score_out[str(teams[i][0])] = (nishant[i])
+
+    location = db.collection(u'stats').document(u'stats-noNN')
+    for i in range(len(teams)):
+        general_general_stats = location.collection(teams[i][0])
+        
+        for j in range(len(files)):
+            json_out[str(teams[i][0])] = (stats[j][i])
+            name = os.path.basename(files[j])
+            general_general_stats.document(name).set({'stats':json_out.get(teams[i][0])})
+
+    for i in range(len(teams)):
+        nnum = location.collection(teams[i][0]).document(u'nishant_number').set({'nishant':score_out.get(teams[i][0])})
+
+def pulldata():
+    teams = analysis.load_csv('data/teams.csv')
+    scores = []
+    for i in range(len(teams)):
+        team_scores = []
+        #print(teams[i][0])
+        request_data_object = tba.req_team_matches(teams[i][0], 2019, "UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5")
+        json_data = request_data_object.json()
+        for match in range(len(json_data) - 1, -1, -1):
+            if json_data[match].get('winning_alliance') == "":
+                print(json_data[match])
+                json_data.remove(json_data[match])
+
+            
+        json_data = sorted(json_data, key=lambda k: k.get('actual_time', 0), reverse=False)
+        for j in range(len(json_data)):
+            if "frc" + teams[i][0] in json_data[j].get('alliances').get('blue').get('team_keys'):
+                team_scores.append(json_data[j].get('alliances').get('blue').get('score'))
+            elif "frc" + teams[i][0] in json_data[j].get('alliances').get('red').get('team_keys'):
+                team_scores.append(json_data[j].get('alliances').get('red').get('score'))
+        scores.append(team_scores)
+
+    with open("data/scores.csv", "w+", newline = '') as file:
+        writer = csv.writer(file, delimiter = ',')
+        writer.writerows(scores)
+            
+def service():
+
+    while True:
+
+        pulldata()
+
+        start = time.time()
+
+        print("[OK] starting calculations")
+
+        fucked = False
+        
+        for i in range(0, 5):
+            try:
+                titanservice()
+                break
+            except:
+                if (i != 4):
+                    print("[WARNING] failed, trying " + str(5 - i - 1) + " more times")
+                else:
+                    print("[ERROR] failed to compute data, skipping")
+                    fucked = True
+
+        end = time.time()
+        if (fucked == True):
+
+            break
+
+        else:
+            
+            print("[OK] finished calculations")
+
+        print("[OK] waiting: " + str(300 - (end - start)) + " seconds" + "\n")
+
+        time.sleep(300 - (end - start)) #executes once every 5 minutes
+
+warnings.simplefilter("ignore")
+#Use a service account
+try:
+    cred = credentials.Certificate('keys/firebasekey.json')
+except:
+    cred = credentials.Certificate('keys/keytemp.json')
+firebase_admin.initialize_app(cred)
+
+db = firestore.client()
+
+service() #finally we write something that isn't a function definition
+#titanservice()

data analysis/.ipynb_checkpoints/tbarequest-checkpoint.py

@@ -0,0 +1,104 @@
+#Titan Robotics Team 2022: TBA Requests Module
+#Written by Arthur Lu & Jacob Levine
+#Notes:
+#   this should be imported as a python module using 'import tbarequest'
+#   this should be included in the local directory or environment variable
+#   this module has not been optimized for multhreaded computing
+#Number of easter eggs: none yet
+#setup:
+
+__version__ = "1.0.0.001"
+
+#changelog should be viewed using print(tbarequest.__changelog__)
+__changelog__ = """changelog:
+1.0.1.000:
+    - fixed a simple error
+1.0.0.xxx:
+    -added common requests and JSON processing"""
+__author__ = (
+    "Arthur Lu <arthurlu@ttic.edu>, "
+    "Jacob Levine <jlevine@ttic.edu>,"
+    )
+__all__ = [
+    'process_json_ret',
+    'req_all_events',
+    'req_event_matches',
+    'req_event_insights',
+    'req_event_elim_alli'
+    'req_team_events',
+    'req_team_matches'
+    ]
+#imports
+import requests
+
+#as this code is public, i'm not putting 2022's API key in here. just add it as a var in your script and go
+#requests a list of events that a team went to
+def req_team_events(team,year,apikey):
+    headers={'X-TBA-Auth-Key':apikey}
+    r=requests.get('https://www.thebluealliance.com/api/v3/team/frc'+str(team)+'/events/'+str(year),headers=headers)
+    return r
+
+#gets every match that a team played in
+def req_team_matches(team,year,apikey):
+    headers={'X-TBA-Auth-Key':apikey}
+    r=requests.get('https://www.thebluealliance.com/api/v3/team/frc'+str(team)+'/matches/'+str(year), headers=headers)
+    return r
+
+#gets all events in a certain year
+def req_all_events(year, apikey):
+    headers={'X-TBA-Auth-Key':apikey}
+    r=requests.get('https://www.thebluealliance.com/api/v3/events/'+str(year), headers=headers)
+    return r
+
+#gets all matches for an event
+def req_event_matches(event_key,apikey):
+    headers={'X-TBA-Auth-Key':apikey}
+    r=requests.get('https://www.thebluealliance.com/api/v3/event/'+str(event_key)+'/matches', headers=headers)
+    return r
+
+#gets elimination alliances from a event
+def req_event_elim_alli(event_key, apikey):
+    headers={'X-TBA-Auth-Key':apikey}
+    r=requests.get('https://www.thebluealliance.com/api/v3/event/'+str(event_key)+'/alliances', headers=headers)
+    return r
+
+#requests oprs and dprs
+def req_event_opr(event_key, apikey):
+    headers={'X-TBA-Auth-Key':apikey}
+    r=requests.get('https://www.thebluealliance.com/api/v3//event/'+str(event_key)+'/oprs', headers=headers)
+    return r
+
+
+
+#gets TBA's insights from an event
+def req_event_insights(event_key, apikey):
+    headers={'X-TBA-Auth-Key':apikey}
+    r=requests.get('https://www.thebluealliance.com/api/v3/event/'+str(event_key)+'/insights', headers=headers)
+    return r
+
+#processes the json return. right now, it's slow and not great. will throw an exception if it doesn't get a good status code
+def process_json_ret(req):
+    if req.status_code == 200:
+        keys=[]
+        for i in req.json():
+            for j in i.keys():
+                read=False
+                for k in keys:
+                    if k==j:
+                        read=True
+                        break
+                if not read:
+                    keys.append(j)
+        out=[]
+        out.append(keys)
+        for i in req.json():
+            buf=[]
+            for j in keys:
+                try:
+                    buf.append(i[j])
+                except:
+                    buf.append("")
+            out.append(buf)
+        return out
+    else:
+        raise ValueError('Status code is: '+req.status_code+', not 200')

data analysis/.ipynb_checkpoints/test-checkpoint.py

@@ -0,0 +1,97 @@
+import firebase_admin
+from firebase_admin import credentials
+from firebase_admin import firestore
+import csv
+import numpy as np
+
+# Use a service account
+cred = credentials.Certificate('keys/keytemp.json')
+#add your own key as this is public. email me for details
+firebase_admin.initialize_app(cred)
+
+db = firestore.client()
+
+teams=db.collection('data').document('team-2022').collection("Central 2019").get()
+full=[]
+tms=[]
+for team in teams:
+    
+    tms.append(team.id)
+    reports=db.collection('data').document('team-2022').collection("Central 2019").document(team.id).collection("matches").get()
+
+    for report in reports:
+        data=[]
+        data.append(db.collection('data').document('team-2022').collection("Central 2019").document(team.id).collection("matches").document(report.id).get().to_dict())
+        full.append(data)
+
+quant_keys = []
+
+list_teams = ["2022", "16", "2451"]
+
+out = []
+var = {}
+
+for i in range(len(full)):
+    for j in range(len(full[i])):
+        for key in list(full[i][j].keys()):
+            
+            if "Quantitative" in key:
+                
+                quant_keys.append(key)
+                
+                if full[i][j].get(key).get('teamDBRef')[5:] in list_teams:
+                    
+                    var = {}
+                    measured_vars = []
+                    
+                    for k in range(len(list(full[i][j].get(key).keys()))):
+
+                        individual_keys = list(full[i][j].get(key).keys())
+                        
+                        var[individual_keys[k]] = full[i][j].get(key).get(individual_keys[k])
+
+                    out.append(var)
+
+sorted_out = []
+
+for i in out:
+
+    j_list = []
+
+    key_list = []
+
+    sorted_keys = sorted(i.keys())
+
+    for j in sorted_keys:
+
+        key_list.append(i[j])
+
+        j_list.append(j)
+
+    sorted_out.append(key_list)
+
+var_index = 0
+team_index = 0
+
+big_out = []
+
+for j in range(len(i)):
+    big_out.append([])
+    for t in range(len(list_teams)):
+        big_out[j].append([])
+
+for i in sorted_out:
+
+    team_index = list_teams.index(sorted_out[sorted_out.index(i)][j_list.index('teamDBRef')][5:])
+
+    for j in range(len(i)):
+
+        big_out[j][team_index].append(i[j])
+
+for i in range(len(big_out)):
+
+    with open('data/' + j_list[i] + '.csv', "w+", newline = '') as file:
+
+        writer = csv.writer(file, delimiter = ',')
+        writer.writerows(big_out[i])
+

data analysis/.ipynb_checkpoints/titanlearn-checkpoint.py

@@ -0,0 +1,206 @@
+#Titan Robotics Team 2022: ML Module
+#Written by Arthur Lu & Jacob Levine
+#Notes:
+#   this should be imported as a python module using 'import titanlearn'
+#   this should be included in the local directory or environment variable
+#   this module has not been optimized for multhreaded computing
+#   this module learns from its mistakes far faster than 2022's captains
+#setup:
+
+__version__ = "1.0.0.001"
+
+#changelog should be viewed using print(analysis.__changelog__)
+__changelog__ = """changelog:
+1.0.0.xxx:
+    -added generation of ANNS, basic SGD training"""
+__author__ = (
+    "Arthur Lu <arthurlu@ttic.edu>, "
+    "Jacob Levine <jlevine@ttic.edu>,"
+    )
+__all__ = [
+    'linear_nn',
+    'train_sgd_minibatch',
+    'train_sgd_simple'
+    ]
+#imports
+import torch
+import warnings
+from collections import OrderedDict
+from sklearn import metrics, datasets
+import numpy as np
+import matplotlib.pyplot as plt
+import math
+import time
+
+#enable CUDA if possible
+device = torch.device("cpu")
+
+#linear_nn: creates a fully connected network given params
+def linear_nn(in_dim, hidden_dim, out_dim, num_hidden, act_fn="tanh", end="none"):
+    if act_fn.lower()=="tanh":
+        k=OrderedDict([("in", torch.nn.Linear(in_dim,hidden_dim))])
+        for i in range(num_hidden):
+            k.update({"lin"+str(i+1): torch.nn.Linear(hidden_dim,hidden_dim), "tanh"+str(i+1):torch.nn.Tanh()})
+
+    elif act_fn.lower()=="sigmoid":
+        k=OrderedDict([("in", torch.nn.Linear(in_dim,hidden_dim))])
+        for i in range(num_hidden):
+            k.update({"lin"+str(i+1): torch.nn.Linear(hidden_dim,hidden_dim), "sig"+str(i+1):torch.nn.Sigmoid()})
+
+    elif act_fn.lower()=="relu":
+        k=OrderedDict([("in", torch.nn.Linear(in_dim,hidden_dim))])
+        for i in range(num_hidden):
+            k.update({"lin"+str(i+1): torch.nn.Linear(hidden_dim,hidden_dim), "relu"+str(i+1):torch.nn.ReLU()})
+
+    elif act_fn.lower()=="leaky relu":
+        k=OrderedDict([("in", torch.nn.Linear(in_dim,hidden_dim))])
+        for i in range(num_hidden):
+            k.update({"lin"+str(i+1): torch.nn.Linear(hidden_dim,hidden_dim), "lre"+str(i+1):torch.nn.LeakyReLU()})
+    else:
+        warnings.warn("Did not specify a valid inner activation function. Returning nothing.")
+        return None
+
+    if end.lower()=="softmax":
+        k.update({"out": torch.nn.Linear(hidden_dim,out_dim), "softmax": torch.nn.Softmax()})
+    elif end.lower()=="none":
+        k.update({"out": torch.nn.Linear(hidden_dim,out_dim)})
+    elif end.lower()=="sigmoid":
+        k.update({"out": torch.nn.Linear(hidden_dim,out_dim), "sigmoid": torch.nn.Sigmoid()})
+    else:
+        warnings.warn("Did not specify a valid final activation function. Returning nothing.")
+        return None
+
+    return torch.nn.Sequential(k)
+
+#train_sgd_simple: trains network using SGD
+def train_sgd_simple(net, evalType, data, ground, dev=None, devg=None, iters=1000, learnrate=1e-4, testevery=1, graphsaveloc=None, modelsaveloc=None, loss="mse"):
+    model=net.to(device)
+    data=data.to(device)
+    ground=ground.to(device)
+    if dev != None:
+        dev=dev.to(device)
+    losses=[]
+    dev_losses=[]
+    if loss.lower()=="mse":
+        loss_fn = torch.nn.MSELoss()
+    elif loss.lower()=="cross entropy":
+        loss_fn = torch.nn.CrossEntropyLoss()
+    elif loss.lower()=="nll":
+        loss_fn = torch.nn.NLLLoss()
+    elif loss.lower()=="poisson nll":
+        loss_fn = torch.nn.PoissonNLLLoss()
+    else:
+        warnings.warn("Did not specify a valid loss function. Returning nothing.")
+        return None
+    optimizer=torch.optim.SGD(model.parameters(), lr=learnrate)
+    for i in range(iters):
+        if i%testevery==0:
+            with torch.no_grad():
+                output = model(data)
+                if evalType == "ap":
+                    ap = metrics.average_precision_score(ground.cpu().numpy(), output.cpu().numpy())
+                if evalType == "regression":
+                    ap = metrics.explained_variance_score(ground.cpu().numpy(), output.cpu().numpy())
+                losses.append(ap)
+                print(str(i)+": "+str(ap))
+                plt.plot(np.array(range(0,i+1,testevery)),np.array(losses), label="train AP")
+                if dev != None:
+                    output = model(dev)
+                    print(evalType)
+                    if evalType == "ap":
+
+                        ap = metrics.average_precision_score(devg.numpy(), output.numpy())
+                        dev_losses.append(ap)
+                        plt.plot(np.array(range(0,i+1,testevery)),np.array(losses), label="dev AP")
+                    elif evalType == "regression":
+                        ev = metrics.explained_variance_score(devg.numpy(), output.numpy())
+                        dev_losses.append(ev)
+                        plt.plot(np.array(range(0,i+1,testevery)),np.array(losses), label="dev EV")
+
+
+                if graphsaveloc != None:
+                    plt.savefig(graphsaveloc+".pdf")
+        with torch.enable_grad():
+            optimizer.zero_grad()
+            output = model(data)
+            loss = loss_fn(output, ground)
+            print(loss.item())
+            loss.backward()
+            optimizer.step()
+    if modelsaveloc != None:
+        torch.save(model, modelsaveloc)
+    plt.show()
+    return model
+
+#train_sgd_minibatch: same as above, but with minibatches
+def train_sgd_minibatch(net, data, ground, dev=None, devg=None, epoch=100, batchsize=20, learnrate=1e-4, testevery=20, graphsaveloc=None, modelsaveloc=None, loss="mse"):
+    model=net.to(device)
+    data=data.to(device)
+    ground=ground.to(device)
+    if dev != None:
+        dev=dev.to(device)
+    losses=[]
+    dev_losses=[]
+    if loss.lower()=="mse":
+        loss_fn = torch.nn.MSELoss()
+    elif loss.lower()=="cross entropy":
+        loss_fn = torch.nn.CrossEntropyLoss()
+    elif loss.lower()=="nll":
+        loss_fn = torch.nn.NLLLoss()
+    elif loss.lower()=="poisson nll":
+        loss_fn = torch.nn.PoissonNLLLoss()
+    else:
+        warnings.warn("Did not specify a valid loss function. Returning nothing.")
+        return None
+    optimizer=torch.optim.LBFGS(model.parameters(), lr=learnrate)
+    itercount=0
+    for i in range(epoch):
+        print("EPOCH "+str(i)+" OF "+str(epoch-1))
+        batches=math.ceil(data.size()[0].item()/batchsize)
+        for j in range(batches):
+            batchdata=[]
+            batchground=[]
+            for k in range(j*batchsize, min((j+1)*batchsize, data.size()[0].item()),1):
+                batchdata.append(data[k])
+                batchground.append(ground[k])
+            batchdata=torch.stack(batchdata)
+            batchground=torch.stack(batchground)
+            if itercount%testevery==0:
+                with torch.no_grad():
+                    output = model(data)
+                    ap = metrics.average_precision_score(ground.numpy(), output.numpy())
+                    losses.append(ap)
+                    print(str(i)+": "+str(ap))
+                    plt.plot(np.array(range(0,i+1,testevery)),np.array(losses))
+                    if dev != None:
+                        output = model(dev)
+                        ap = metrics.average_precision_score(devg.numpy(), output.numpy())
+                        dev_losses.append(ap)
+                        plt.plot(np.array(range(0,i+1,testevery)),np.array(losses), label="dev AP")
+                    if graphsaveloc != None:
+                        plt.savefig(graphsaveloc+".pdf")
+            with torch.enable_grad():
+                optimizer.zero_grad()
+                output = model(batchdata)
+                loss = loss_fn(output, ground)
+                loss.backward()
+                optimizer.step()
+            itercount +=1
+    if modelsaveloc != None:
+        torch.save(model, modelsaveloc)
+    plt.show()
+    return model
+
+def retyuoipufdyu():
+
+    data = torch.tensor(datasets.fetch_california_housing()['data']).to(torch.float)
+    ground = datasets.fetch_california_housing()['target']
+    ground = torch.tensor(ground).to(torch.float)
+    model = linear_nn(8, 100, 1, 20, act_fn = "relu")
+    print(model)
+    return train_sgd_simple(model,"regression", data, ground, learnrate=1e-4, iters=1000)
+
+start = time.time()
+retyuoipufdyu()
+end = time.time()
+print(end-start)

data analysis/.ipynb_checkpoints/visualization-checkpoint.py

@@ -0,0 +1,130 @@
+#Titan Robotics Team 2022: Visualization Module
+#Written by Arthur Lu & Jacob Levine
+#Notes:
+#   this should be imported as a python module using 'import visualization'
+#   this should be included in the local directory or environment variable
+#   this module has not been optimized for multhreaded computing
+#Number of easter eggs: Jake is Jewish and does not observe easter.
+#setup:
+
+__version__ = "1.0.0.001"
+
+#changelog should be viewed using print(analysis.__changelog__)
+__changelog__ = """changelog:
+1.0.0.xxx:
+    -added basic plotting, clustering, and regression comparisons"""
+__author__ = (
+    "Arthur Lu <arthurlu@ttic.edu>, "
+    "Jacob Levine <jlevine@ttic.edu>,"
+    )
+__all__ = [
+    'affinity_prop',
+    'bar_graph',
+    'dbscan',
+    'kmeans',
+    'line_plot',
+    'pca_comp',
+    'regression_comp',
+    'scatter_plot',
+    'spectral',
+    'vis_2d'
+    ]
+#imports
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.decomposition import PCA, KernelPCA, IncrementalPCA
+from sklearn.preprocessing import StandardScaler
+from sklearn.cluster import AffinityPropagation, DBSCAN, KMeans, SpectralClustering
+
+#bar of x,y
+def bar_graph(x,y):
+    x=np.asarray(x)
+    y=np.asarray(y)
+    plt.bar(x,y)
+    plt.show()
+
+#scatter of x,y
+def scatter_plot(x,y):
+    x=np.asarray(x)
+    y=np.asarray(y)
+    plt.scatter(x,y)
+    plt.show()
+
+#line of x,y
+def line_plot(x,y):
+    x=np.asarray(x)
+    y=np.asarray(y)
+    plt.scatter(x,y)
+    plt.show()
+
+#plot data + regression fit
+def regression_comp(x,y,reg):
+    x=np.asarray(x)
+    y=np.asarray(y)
+    regx=np.arange(x.min(),x.max(),(x.max()-x.min())/1000)
+    regy=[]
+    for i in regx:
+        regy.append(eval(reg[0].replace("z",str(i))))
+    regy=np.asarray(regy)
+    plt.scatter(x,y)
+    plt.plot(regx,regy,color="orange",linewidth=3)
+    plt.text(.85*max([x.max(),regx.max()]),.95*max([y.max(),regy.max()]),
+            u"R\u00b2="+str(round(reg[2],5)),
+            horizontalalignment='center', verticalalignment='center')
+    plt.text(.85*max([x.max(),regx.max()]),.85*max([y.max(),regy.max()]),
+            "MSE="+str(round(reg[1],5)),
+             horizontalalignment='center', verticalalignment='center')
+    plt.show()
+
+#PCA to compress down to 2d
+def pca_comp(big_multidim):
+    pca=PCA(n_components=2)
+    td_norm=StandardScaler().fit_transform(big_multidim)
+    td_pca=pca.fit_transform(td_norm)
+    return td_pca
+
+#one-stop visualization of multidim datasets
+def vis_2d(big_multidim):
+    td_pca=pca_comp(big_multidim)
+    plt.scatter(td_pca[:,0], td_pca[:,1])
+
+def cluster_vis(data, cluster_assign):
+    pca=PCA(n_components=2)
+    td_norm=StandardScaler().fit_transform(data)
+    td_pca=pca.fit_transform(td_norm)
+    colors = np.array(list(islice(cycle(['#377eb8', '#ff7f00', '#4daf4a',
+                                             '#f781bf', '#a65628', '#984ea3',
+                                             '#999999', '#e41a1c', '#dede00']),
+                                      int(max(clu) + 1))))
+    colors = np.append(colors, ["#000000"])
+    plt.figure(figsize=(8, 8))
+    plt.scatter(td_norm[:, 0], td_norm[:, 1], s=10, color=colors[cluster_assign])
+    plt.show()
+
+#affinity prop- slow, but ok if you don't have any idea how many you want
+def affinity_prop(data, damping=.77, preference=-70):
+    td_norm=StandardScaler().fit_transform(data)
+    db = AffinityPropagation(damping=damping,preference=preference).fit(td)
+    y=db.predict(td_norm)
+    return y
+
+#DBSCAN- slightly faster but can label your dataset as all outliers
+def dbscan(data, eps=.3):
+    td_norm=StandardScaler().fit_transform(data)
+    db = DBSCAN(eps=eps).fit(td)
+    y=db.labels_.astype(np.int)
+    return y
+
+#K-means clustering- the classic
+def kmeans(data, num_clusters):
+    td_norm=StandardScaler().fit_transform(data)
+    db = KMeans(n_clusters=num_clusters).fit(td)
+    y=db.labels_.astype(np.int)
+    return y
+
+#Spectral Clustering- Seems to work really well
+def spectral(data, num_clusters):
+    td_norm=StandardScaler().fit_transform(data)
+    db = SpectralClustering(n_clusters=num_clusters).fit(td)
+    y=db.labels_.astype(np.int)
+    return y

data analysis/Untitled.ipynb

@@ -0,0 +1,88 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tbarequest"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2022', 'frc2358', 'frc7417']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2040', 'frc2481', 'frc81']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm70', 'match_number': 70, 'post_result_time': None, 'predicted_time': 1553357160, 'score_breakdown': None, 'set_number': 1, 'time': 1553357160, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc6237', 'frc4156', 'frc4646']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc4241', 'frc1781', 'frc2022']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm61', 'match_number': 61, 'post_result_time': None, 'predicted_time': 1553352300, 'score_breakdown': None, 'set_number': 1, 'time': 1553352300, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc1736', 'frc2022', 'frc5126']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc5690', 'frc1329', 'frc4241']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm57', 'match_number': 57, 'post_result_time': None, 'predicted_time': 1553350140, 'score_breakdown': None, 'set_number': 1, 'time': 1553350140, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc6237', 'frc2358', 'frc2039']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2022', 'frc7848', 'frc4212']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm53', 'match_number': 53, 'post_result_time': None, 'predicted_time': 1553292960, 'score_breakdown': None, 'set_number': 1, 'time': 1553292960, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc7465', 'frc4213', 'frc2704']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc1329', 'frc6055', 'frc2022']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm41', 'match_number': 41, 'post_result_time': None, 'predicted_time': 1553287200, 'score_breakdown': None, 'set_number': 1, 'time': 1553287200, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2022', 'frc4156', 'frc5442']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc5822', 'frc4143', 'frc7848']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm4', 'match_number': 4, 'post_result_time': None, 'predicted_time': 1553264820, 'score_breakdown': None, 'set_number': 1, 'time': 1553264820, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc1094', 'frc4314', 'frc6651']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc6237', 'frc2081', 'frc2022']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm38', 'match_number': 38, 'post_result_time': None, 'predicted_time': 1553285760, 'score_breakdown': None, 'set_number': 1, 'time': 1553285760, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2013', 'frc2081', 'frc6055']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc81', 'frc2022', 'frc3695']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm31', 'match_number': 31, 'post_result_time': None, 'predicted_time': 1553282400, 'score_breakdown': None, 'set_number': 1, 'time': 1553282400, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc4655', 'frc5822', 'frc2022']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc4256', 'frc323', 'frc2709']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm22', 'match_number': 22, 'post_result_time': None, 'predicted_time': 1553278080, 'score_breakdown': None, 'set_number': 1, 'time': 1553278080, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc1756', 'frc1329', 'frc1288']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc4096', 'frc2022', 'frc323']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm14', 'match_number': 14, 'post_result_time': None, 'predicted_time': 1553269860, 'score_breakdown': None, 'set_number': 1, 'time': 1553269860, 'videos': [], 'winning_alliance': ''}\n",
+      "{'actual_time': None, 'alliances': {'blue': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2039', 'frc2022', 'frc1288']}, 'red': {'dq_team_keys': [], 'score': -1, 'surrogate_team_keys': [], 'team_keys': ['frc2013', 'frc4096', 'frc1781']}}, 'comp_level': 'qm', 'event_key': '2019ilpe', 'key': '2019ilpe_qm10', 'match_number': 10, 'post_result_time': None, 'predicted_time': 1553267940, 'score_breakdown': None, 'set_number': 1, 'time': 1553267940, 'videos': [], 'winning_alliance': ''}\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Response [404]>"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tbarequest.req_team_matches('frc16', '2019','UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

data analysis/superscript.py

@@ -1,7 +1,7 @@
-#Titan Robotics Team 2022: Super Script
+# Titan Robotics Team 2022: Super Script
-#Written by Arthur Lu & Jacob Levine
+# Written by Arthur Lu & Jacob Levine
-#Notes:
+# Notes:
-#setup:
+# setup:
 
 __version__ = "1.0.6.000"
 
@@ -35,12 +35,12 @@ __changelog__ = """changelog:
     - added superstructure to code
 1.0.0.000:
     - added import statements (revolutionary)
-""" 
+"""
 
 __author__ = (
     "Arthur Lu <arthurlu@ttic.edu>, "
     "Jacob Levine <jlevine@ttic.edu>,"
-    )
+)
 
 import firebase_admin
 from firebase_admin import credentials
@@ -57,22 +57,24 @@ import time
 import tbarequest as tba
 import csv
 
+
 def titanservice():
-    
+
     print("[OK] loading data")
 
     start = time.time()
-    
+
     source_dir = 'data'
-    file_list = glob.glob(source_dir + '/*.csv') #supposedly sorts by alphabetical order, skips reading teams.csv because of redundancy
+    # supposedly sorts by alphabetical order, skips reading teams.csv because of redundancy
+    file_list = glob.glob(source_dir + '/*.csv')
     data = []
-    files = [fn for fn in glob.glob('data/*.csv') 
+    files = [fn for fn in glob.glob('data/*.csv')
-             if not (os.path.basename(fn).startswith('scores') or os.path.basename(fn).startswith('teams') or os.path.basename(fn).startswith('match') or os.path.basename(fn).startswith('notes') or os.path.basename(fn).startswith('observationType') or os.path.basename(fn).startswith('teamDBRef'))] #scores will be handled sperately
+             if not (os.path.basename(fn).startswith('scores') or os.path.basename(fn).startswith('teams') or os.path.basename(fn).startswith('match') or os.path.basename(fn).startswith('notes') or os.path.basename(fn).startswith('observationType') or os.path.basename(fn).startswith('teamDBRef'))]  # scores will be handled sperately
 
     for i in files:
-            data.append(analysis.load_csv(i))
+        data.append(analysis.load_csv(i))
 
-    #print(files)
+    # print(files)
 
     stats = []
     measure_stats = []
@@ -83,156 +85,168 @@ def titanservice():
 
     print("[OK] loaded data in " + str(end - start) + " seconds")
 
-    #assumes that team number is in the first column, and that the order of teams is the same across all files
+    # assumes that team number is in the first column, and that the order of teams is the same across all files
-    #unhelpful comment
+    # unhelpful comment
-    for measure in data: #unpacks 3d array into 2ds
+    for measure in data:  # unpacks 3d array into 2ds
 
         measure_stats = []
 
-        for i in range(len(measure)): #unpacks into specific teams
+        for i in range(len(measure)):  # unpacks into specific teams
 
-                    #ofbest_curve = [None]
+            #ofbest_curve = [None]
-                    #r2best_curve = [None]
+            #r2best_curve = [None]
 
-                    line = measure[i]
+            line = measure[i]
 
-                    #print(line)
+            # print(line)
 
-                    #x = list(range(len(line)))
+            #x = list(range(len(line)))
-                    #eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1)
+            #eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1)
 
-                    #beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "min_overfit")
+            #beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "min_overfit")
 
-                    #print(eqs, rmss, r2s, overfit)
+            #print(eqs, rmss, r2s, overfit)
-                    
-                    #ofbest_curve.append(beqs)
-                    #ofbest_curve.append(brmss)
-                    #ofbest_curve.append(br2s)
-                    #ofbest_curve.append(boverfit)
-                    #ofbest_curve.pop(0)
 
-                    #print(ofbest_curve)
+            # ofbest_curve.append(beqs)
+            # ofbest_curve.append(brmss)
+            # ofbest_curve.append(br2s)
+            # ofbest_curve.append(boverfit)
+            # ofbest_curve.pop(0)
 
-                    #beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "max_r2s")
+            # print(ofbest_curve)
 
-                    #r2best_curve.append(beqs)
+            #beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "max_r2s")
-                    #r2best_curve.append(brmss)
-                    #r2best_curve.append(br2s)
-                    #r2best_curve.append(boverfit)
-                    #r2best_curve.pop(0)
 
-                    #print(r2best_curve)
+            # r2best_curve.append(beqs)
+            # r2best_curve.append(brmss)
+            # r2best_curve.append(br2s)
+            # r2best_curve.append(boverfit)
+            # r2best_curve.pop(0)
 
-                    
+            # print(r2best_curve)
-                    measure_stats.append(teams[i] + list(analysis.basic_stats(line, 0, 0)) + list(analysis.histo_analysis(line, 1, -3, 3)))
+
+            measure_stats.append(teams[i] + list(analysis.basic_stats(
+                line, 0, 0)) + list(analysis.histo_analysis(line, 1, -3, 3)))
 
         stats.append(list(measure_stats))
         nishant = []
-            
+
     for i in range(len(scores)):
 
-                #print(scores)
+        # print(scores)
 
-                ofbest_curve = [None]
+        ofbest_curve = [None]
-                r2best_curve = [None]
+        r2best_curve = [None]
 
-                line = scores[i]
+        line = scores[i]
 
-                #print(line)
+        # print(line)
 
-                #print(line)
+        # print(line)
 
-                x = list(range(len(line)))
+        x = list(range(len(line)))
-                eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1)
+        eqs, rmss, r2s, overfit = analysis.optimize_regression(x, line, 10, 1)
 
-                beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "min_overfit")
+        beqs, brmss, br2s, boverfit = analysis.select_best_regression(
+            eqs, rmss, r2s, overfit, "min_overfit")
 
-                #print(eqs, rmss, r2s, overfit)
+        #print(eqs, rmss, r2s, overfit)
-                    
-                ofbest_curve.append(beqs)
-                ofbest_curve.append(brmss)
-                ofbest_curve.append(br2s)
-                ofbest_curve.append(boverfit)
-                ofbest_curve.pop(0)
 
-                #print(ofbest_curve)
+        ofbest_curve.append(beqs)
+        ofbest_curve.append(brmss)
+        ofbest_curve.append(br2s)
+        ofbest_curve.append(boverfit)
+        ofbest_curve.pop(0)
 
-                beqs, brmss, br2s, boverfit = analysis.select_best_regression(eqs, rmss, r2s, overfit, "max_r2s")
+        # print(ofbest_curve)
 
-                r2best_curve.append(beqs)
+        beqs, brmss, br2s, boverfit = analysis.select_best_regression(
-                r2best_curve.append(brmss)
+            eqs, rmss, r2s, overfit, "max_r2s")
-                r2best_curve.append(br2s)
-                r2best_curve.append(boverfit)
-                r2best_curve.pop(0)
 
-                #print(r2best_curve)
+        r2best_curve.append(beqs)
-                    
+        r2best_curve.append(brmss)
-                z = len(scores[0]) + 1
+        r2best_curve.append(br2s)
-                nis_num = []
+        r2best_curve.append(boverfit)
+        r2best_curve.pop(0)
 
-                nis_num.append(eval(str(ofbest_curve[0])))
+        # print(r2best_curve)
-                nis_num.append(eval(str(r2best_curve[0])))
 
-                nis_num.append((eval(ofbest_curve[0]) + eval(r2best_curve[0])) / 2)
+        z = len(scores[0]) + 1
+        nis_num = []
+
+        nis_num.append(eval(str(ofbest_curve[0])))
+        nis_num.append(eval(str(r2best_curve[0])))
+
+        nis_num.append((eval(ofbest_curve[0]) + eval(r2best_curve[0])) / 2)
+
+        nishant.append(teams[i] + nis_num)
 
-                nishant.append(teams[i] + nis_num)
-                
     json_out = {}
     score_out = {}
 
     for i in range(len(teams)):
-            score_out[str(teams[i][0])] = (nishant[i])
+        score_out[str(teams[i][0])] = (nishant[i])
 
     location = db.collection(u'stats').document(u'stats-noNN')
     for i in range(len(teams)):
         general_general_stats = location.collection(teams[i][0])
-        
+
         for j in range(len(files)):
             json_out[str(teams[i][0])] = (stats[j][i])
             name = os.path.basename(files[j])
-            general_general_stats.document(name).set({'stats':json_out.get(teams[i][0])})
+            general_general_stats.document(name).set(
+                {'stats': json_out.get(teams[i][0])})
 
     for i in range(len(teams)):
-        nnum = location.collection(teams[i][0]).document(u'nishant_number').set({'nishant':score_out.get(teams[i][0])})
+        nnum = location.collection(teams[i][0]).document(
+            u'nishant_number').set({'nishant': score_out.get(teams[i][0])})
+
 
 def pulldata():
     teams = analysis.load_csv('data/teams.csv')
     scores = []
     for i in range(len(teams)):
         team_scores = []
-        #print(teams[i][0])
+        # print(teams[i][0])
-        request_data_object = tba.req_team_matches(teams[i][0], 2019, "UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5")
+        request_data_object = tba.req_team_matches(
+            teams[i][0], 2019, "UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5")
         json_data = request_data_object.json()
 
         for match in range(len(json_data) - 1, -1, -1):
             if json_data[match].get('winning_alliance') == "":
-                #print(json_data[match])
+                # print(json_data[match])
                 json_data.remove(json_data[match])
-            
+
-        json_data = sorted(json_data, key=lambda k: k.get('actual_time', 0), reverse=False)
+        json_data = sorted(json_data, key=lambda k: k.get(
+            'actual_time', 0), reverse=False)
         for j in range(len(json_data)):
             if "frc" + teams[i][0] in json_data[j].get('alliances').get('blue').get('team_keys'):
-                team_scores.append(json_data[j].get('alliances').get('blue').get('score'))
+                team_scores.append(json_data[j].get(
+                    'alliances').get('blue').get('score'))
             elif "frc" + teams[i][0] in json_data[j].get('alliances').get('red').get('team_keys'):
-                team_scores.append(json_data[j].get('alliances').get('red').get('score'))
+                team_scores.append(json_data[j].get(
+                    'alliances').get('red').get('score'))
         scores.append(team_scores)
 
-    with open("data/scores.csv", "w+", newline = '') as file:
+    with open("data/scores.csv", "w+", newline='') as file:
-        writer = csv.writer(file, delimiter = ',')
+        writer = csv.writer(file, delimiter=',')
         writer.writerows(scores)
 
     list_teams = teams
-    teams=db.collection('data').document('team-2022').collection("Central 2019").get()
+    teams = db.collection('data').document(
-    full=[]
+        'team-2022').collection("Central 2019").get()
-    tms=[]
+    full = []
+    tms = []
     for team in teams:
-        
+
         tms.append(team.id)
-        reports=db.collection('data').document('team-2022').collection("Central 2019").document(team.id).collection("matches").get()
+        reports = db.collection('data').document(
+            'team-2022').collection("Central 2019").document(team.id).collection("matches").get()
 
         for report in reports:
-            data=[]
+            data = []
-            data.append(db.collection('data').document('team-2022').collection("Central 2019").document(team.id).collection("matches").document(report.id).get().to_dict())
+            data.append(db.collection('data').document('team-2022').collection("Central 2019").document(
+                team.id).collection("matches").document(report.id).get().to_dict())
             full.append(data)
 
     quant_keys = []
@@ -243,21 +257,22 @@ def pulldata():
     for i in range(len(full)):
         for j in range(len(full[i])):
             for key in list(full[i][j].keys()):
-                
+
                 if "Quantitative" in key:
-                    
+
                     quant_keys.append(key)
-                    
+
                     if full[i][j].get(key).get('teamDBRef')[5:] in list_teams:
-                        
+
                         var = {}
                         measured_vars = []
-                        
+
                         for k in range(len(list(full[i][j].get(key).keys()))):
 
                             individual_keys = list(full[i][j].get(key).keys())
-                            
+
-                            var[individual_keys[k]] = full[i][j].get(key).get(individual_keys[k])
+                            var[individual_keys[k]] = full[i][j].get(
+                                key).get(individual_keys[k])
 
                         out.append(var)
 
@@ -291,7 +306,8 @@ def pulldata():
 
         for i in sorted_out:
 
-            team_index = list_teams.index(sorted_out[sorted_out.index(i)][j_list.index('teamDBRef')][5:])
+            team_index = list_teams.index(
+                sorted_out[sorted_out.index(i)][j_list.index('teamDBRef')][5:])
 
             for j in range(len(i)):
 
@@ -299,11 +315,12 @@ def pulldata():
 
         for i in range(len(big_out)):
 
-            with open('data/' + j_list[i] + '.csv', "w+", newline = '') as file:
+            with open('data/' + j_list[i] + '.csv', "w+", newline='') as file:
 
-                writer = csv.writer(file, delimiter = ',')
+                writer = csv.writer(file, delimiter=',')
                 writer.writerows(big_out[i])
-            
+
+
 def service():
 
     while True:
@@ -315,17 +332,18 @@ def service():
         print("[OK] starting calculations")
 
         fucked = False
-        
+
         for i in range(0, 5):
-            #try:
+            # try:
-                titanservice()
+            titanservice()
-                break
+            break
-            #except:
+            # except:
-                if (i != 4):
+            if (i != 4):
-                    print("[WARNING] failed, trying " + str(5 - i - 1) + " more times")
+                print("[WARNING] failed, trying " +
-                else:
+                      str(5 - i - 1) + " more times")
-                    print("[ERROR] failed to compute data, skipping")
+            else:
-                    fucked = True
+                print("[ERROR] failed to compute data, skipping")
+                fucked = True
 
         end = time.time()
         if (fucked == True):
@@ -333,15 +351,16 @@ def service():
             break
 
         else:
-            
+
             print("[OK] finished calculations")
 
         print("[OK] waiting: " + str(300 - (end - start)) + " seconds" + "\n")
 
-        time.sleep(300 - (end - start)) #executes once every 5 minutes
+        time.sleep(300 - (end - start))  # executes once every 5 minutes
+
 
 warnings.simplefilter("ignore")
-#Use a service account
+# Use a service account
 try:
     cred = credentials.Certificate('keys/firebasekey.json')
 except:
@@ -350,5 +369,5 @@ firebase_admin.initialize_app(cred)
 
 db = firestore.client()
 
-service() #finally we write something that isn't a function definition
+service()  # finally we write something that isn't a function definition
-#titanservice()
+# titanservice()