Merge branch 'c'

data analysis/analysis/analysis.py

@@ -1,15 +1,15 @@
-#Titan Robotics Team 2022: Data Analysis Module
-#Written by Arthur Lu & Jacob Levine
-#Notes:
+# Titan Robotics Team 2022: Data Analysis Module
+# Written by Arthur Lu & Jacob Levine
+# Notes:
 #   this should be imported as a python module using 'import analysis'
 #   this should be included in the local directory or environment variable
 #   this module has not been optimized for multhreaded computing
-#number of easter eggs: 2
-#setup:
+# number of easter eggs: 2
+# setup:
 
 __version__ = "1.0.8.005"
 
-#changelog should be viewed using print(analysis.__changelog__)
+# changelog should be viewed using print(analysis.__changelog__)
 __changelog__ = """changelog:
 1.0.8.005:
     - minor fixes
@@ -101,7 +101,7 @@ __changelog__ = """changelog:
 __author__ = (
     "Arthur Lu <arthurlu@ttic.edu>, "
     "Jacob Levine <jlevine@ttic.edu>,"
-    )
+)
 
 __all__ = [
     '_init_device',
@@ -125,12 +125,12 @@ __all__ = [
     'optimize_regression',
     'select_best_regression',
     'basic_analysis',
-    #all statistics functions left out due to integration in other functions
-    ]
+    # all statistics functions left out due to integration in other functions
+]
 
-#now back to your regularly scheduled programming:
+# now back to your regularly scheduled programming:
 
-#imports (now in alphabetical order! v 1.0.3.006):
+# imports (now in alphabetical order! v 1.0.3.006):
 
 from bisect import bisect_left, bisect_right
 import collections
@@ -149,14 +149,16 @@ import scipy
 from scipy.optimize import curve_fit
 from scipy import stats
 from sklearn import *
-#import statistics <-- statistics.py functions have been integrated into analysis.py as of v 1.0.3.002
+# import statistics <-- statistics.py functions have been integrated into analysis.py as of v 1.0.3.002
 import time
 import torch
 
+
 class error(ValueError):
     pass
 
-def _init_device (setting, arg): #initiates computation device for ANNs
+
+def _init_device(setting, arg):  # initiates computation device for ANNs
     if setting == "cuda":
         try:
             return torch.device(setting + ":" + str(arg) if torch.cuda.is_available() else "cpu")
@@ -170,6 +172,7 @@ def _init_device (setting, arg): #initiates computation device for ANNs
     else:
         raise error("specified device does not exist")
 
+
 class c_entities:
 
     c_names = []
@@ -190,7 +193,6 @@ class c_entities:
         self.c_logic = logic
         return None
 
-
     def append(self, n_name, n_id, n_pos, n_property, n_logic):
         self.c_names.append(n_name)
         self.c_ids.append(n_id)
@@ -232,6 +234,7 @@ class c_entities:
     def regurgitate(self):
         return[self.c_names, self.c_ids, self.c_pos, self.c_properties, self.c_logic]
 
+
 class nc_entities:
 
     c_names = []
@@ -241,7 +244,7 @@ class nc_entities:
     c_effects = []
 
     def debug(self):
-        print ("nc_entities (non-controlable entities) has attributes names, ids, positions, properties, and effects. __init__ takes self, 1d array of names, 1d array of ids, 2d array of positions, 2d array of properties, and 2d array of effects.")
+        print("nc_entities (non-controlable entities) has attributes names, ids, positions, properties, and effects. __init__ takes self, 1d array of names, 1d array of ids, 2d array of positions, 2d array of properties, and 2d array of effects.")
         return[self.c_names, self.c_ids, self.c_pos, self.c_properties, self.c_effects]
 
     def __init__(self, names, ids, pos, properties, effects):
@@ -295,6 +298,7 @@ class nc_entities:
 
         return[self.c_names, self.c_ids, self.c_pos, self.c_properties, self.c_effects]
 
+
 class obstacles:
 
     c_names = []
@@ -351,6 +355,7 @@ class obstacles:
     def regurgitate(self):
         return[self.c_names, self.c_ids, self.c_perim, self.c_effects]
 
+
 class objectives:
 
     c_names = []
@@ -408,13 +413,16 @@ class objectives:
     def regurgitate(self):
         return[self.c_names, self.c_ids, self.c_pos, self.c_effects]
 
+
 def load_csv(filepath):
-    with open(filepath, newline = '') as csvfile:
+    with open(filepath, newline='') as csvfile:
         file_array = list(csv.reader(csvfile))
         csvfile.close()
     return file_array
 
-def basic_stats(data, method, arg): # data=array, mode = ['1d':1d_basic_stats, 'column':c_basic_stats, 'row':r_basic_stats], arg for mode 1 or mode 2 for column or row
+
+# data=array, mode = ['1d':1d_basic_stats, 'column':c_basic_stats, 'row':r_basic_stats], arg for mode 1 or mode 2 for column or row
+def basic_stats(data, method, arg):
 
     if method == 'debug':
         return "basic_stats requires 3 args: data, mode, arg; where data is data to be analyzed, mode is an int from 0 - 2 depending on type of analysis (by column or by row) and is only applicable to 2d arrays (for 1d arrays use mode 1), and arg is row/column number for mode 1 or mode 2; function returns: [mean, median, mode, stdev, variance]"
@@ -423,7 +431,7 @@ def basic_stats(data, method, arg): # data=array, mode = ['1d':1d_basic_stats, '
 
         data_t = []
 
-        for i in range (0, len(data), 1):
+        for i in range(0, len(data), 1):
             data_t.append(float(data[i]))
 
         _mean = mean(data_t)
@@ -498,64 +506,72 @@ def basic_stats(data, method, arg): # data=array, mode = ['1d':1d_basic_stats, '
     else:
         raise error("method error")
 
-def z_score(point, mean, stdev): #returns z score with inputs of point, mean and standard deviation of spread
-    score = (point - mean)/stdev
+
+# returns z score with inputs of point, mean and standard deviation of spread
+def z_score(point, mean, stdev):
+    score = (point - mean) / stdev
     return score
 
-def z_normalize(x, y, mode): #mode is either 'x' or 'y' or 'both' depending on the variable(s) to be normalized
 
-	x_norm = []
-	y_norm = []
+# mode is either 'x' or 'y' or 'both' depending on the variable(s) to be normalized
+def z_normalize(x, y, mode):
 
-	mean = 0
-	stdev = 0
+    x_norm = []
+    y_norm = []
 
-	if mode == 'x':
-		_mean, _median, _mode, _stdev, _variance = basic_stats(x, "1d", 0)
+    mean = 0
+    stdev = 0
 
-		for i in range (0, len(x), 1):
-			x_norm.append(z_score(x[i], _mean, _stdev))
+    if mode == 'x':
+        _mean, _median, _mode, _stdev, _variance = basic_stats(x, "1d", 0)
 
-		return x_norm, y
+        for i in range(0, len(x), 1):
+            x_norm.append(z_score(x[i], _mean, _stdev))
 
-	if mode == 'y': 
-		_mean, _median, _mode, _stdev, _variance = basic_stats(y, "1d", 0)
+        return x_norm, y
 
-		for i in range (0, len(y), 1):
-			y_norm.append(z_score(y[i], _mean, _stdev))
+    if mode == 'y':
+        _mean, _median, _mode, _stdev, _variance = basic_stats(y, "1d", 0)
 
-		return x, y_norm
+        for i in range(0, len(y), 1):
+            y_norm.append(z_score(y[i], _mean, _stdev))
 
-	if mode == 'both':
-		_mean, _median, _mode, _stdev, _variance = basic_stats(x, "1d", 0)
+        return x, y_norm
 
-		for i in range (0, len(x), 1):
-			x_norm.append(z_score(x[i], _mean, _stdev))
+    if mode == 'both':
+        _mean, _median, _mode, _stdev, _variance = basic_stats(x, "1d", 0)
 
-		_mean, _median, _mode, _stdev, _variance = basic_stats(y, "1d", 0)
+        for i in range(0, len(x), 1):
+            x_norm.append(z_score(x[i], _mean, _stdev))
 
-		for i in range (0, len(y), 1):
-			y_norm.append(z_score(y[i], _mean, _stdev))
+        _mean, _median, _mode, _stdev, _variance = basic_stats(y, "1d", 0)
 
-		return x_norm, y_norm
+        for i in range(0, len(y), 1):
+            y_norm.append(z_score(y[i], _mean, _stdev))
 
-	else:
+        return x_norm, y_norm
 
-		return error('method error')
+    else:
 
-def stdev_z_split(mean, stdev, delta, low_bound, high_bound): #returns n-th percentile of spread given mean, standard deviation, lower z-score, and upper z-score
+        return error('method error')
+
+
+# returns n-th percentile of spread given mean, standard deviation, lower z-score, and upper z-score
+def stdev_z_split(mean, stdev, delta, low_bound, high_bound):
 
     z_split = []
     i = low_bound
 
     while True:
-        z_split.append(float((1 / (stdev * math.sqrt(2 * math.pi))) * math.e ** (-0.5 * (((i - mean) / stdev) ** 2))))
+        z_split.append(float((1 / (stdev * math.sqrt(2 * math.pi))) *
+                             math.e ** (-0.5 * (((i - mean) / stdev) ** 2))))
         i = i + delta
         if i > high_bound:
             break
 
     return z_split
 
+
 def histo_analysis(hist_data, delta, low_bound, high_bound):
 
     if hist_data == 'debug':
@@ -565,12 +581,12 @@ def histo_analysis(hist_data, delta, low_bound, high_bound):
 
     for i in range(0, len(hist_data), 1):
         try:
-            derivative.append(float(hist_data[i - 1]) - float(hist_data [i]))
+            derivative.append(float(hist_data[i - 1]) - float(hist_data[i]))
         except:
             pass
 
     derivative_sorted = sorted(derivative, key=int)
-    mean_derivative = basic_stats(derivative_sorted,"1d", 0)[0]
+    mean_derivative = basic_stats(derivative_sorted, "1d", 0)[0]
     stdev_derivative = basic_stats(derivative_sorted, "1d", 0)[3]
 
     predictions = []
@@ -593,23 +609,26 @@ def histo_analysis(hist_data, delta, low_bound, high_bound):
 
     return predictions
 
+
 def poly_regression(x, y, power):
 
-    if x == "null": #if x is 'null', then x will be filled with integer points between 1 and the size of y
+    if x == "null":  # if x is 'null', then x will be filled with integer points between 1 and the size of y
         x = []
 
         for i in range(len(y)):
             print(i)
-            x.append(i+1)
+            x.append(i + 1)
 
-    reg_eq = scipy.polyfit(x, y, deg = power)
+    reg_eq = scipy.polyfit(x, y, deg=power)
     eq_str = ""
 
     for i in range(0, len(reg_eq), 1):
-        if i < len(reg_eq)- 1:
-            eq_str = eq_str + str(reg_eq[i]) + "*(z**" + str(len(reg_eq) - i - 1) + ")+"
+        if i < len(reg_eq) - 1:
+            eq_str = eq_str + str(reg_eq[i]) + \
+                "*(z**" + str(len(reg_eq) - i - 1) + ")+"
         else:
-            eq_str = eq_str + str(reg_eq[i]) + "*(z**" + str(len(reg_eq) - i - 1) + ")"
+            eq_str = eq_str + str(reg_eq[i]) + \
+                "*(z**" + str(len(reg_eq) - i - 1) + ")"
 
     vals = []
 
@@ -617,108 +636,121 @@ def poly_regression(x, y, power):
         z = x[i]
 
         try:
-        	exec("vals.append(" + eq_str + ")")
+            exec("vals.append(" + eq_str + ")")
         except:
-        	pass
+            pass
 
     _rms = rms(vals, y)
     r2_d2 = r_squared(vals, y)
 
     return [eq_str, _rms, r2_d2]
 
+
 def log_regression(x, y, base):
 
-	x_fit = []
+    x_fit = []
 
-	for i in range(len(x)):
-		try:
-			x_fit.append(np.log(x[i]) / np.log(base)) #change of base for logs
-		except:
-			pass
+    for i in range(len(x)):
+        try:
+            # change of base for logs
+            x_fit.append(np.log(x[i]) / np.log(base))
+        except:
+            pass
 
-	reg_eq = np.polyfit(x_fit, y, 1) # y = reg_eq[0] * log(x, base) + reg_eq[1]
-	q_str = str(reg_eq[0]) + "* (np.log(z) / np.log(" + str(base) +"))+" + str(reg_eq[1])
-	vals = []
+    # y = reg_eq[0] * log(x, base) + reg_eq[1]
+    reg_eq = np.polyfit(x_fit, y, 1)
+    q_str = str(reg_eq[0]) + "* (np.log(z) / np.log(" + \
+        str(base) + "))+" + str(reg_eq[1])
+    vals = []
 
-	for i in range(len(x)):
-		z = x[i]
+    for i in range(len(x)):
+        z = x[i]
 
-		try:
-			exec("vals.append(" + eq_str + ")")
-		except:
-			pass
+        try:
+            exec("vals.append(" + eq_str + ")")
+        except:
+            pass
 
-	_rms = rms(vals, y)
-	r2_d2 = r_squared(vals, y)
+    _rms = rms(vals, y)
+    r2_d2 = r_squared(vals, y)
+
+    return eq_str, _rms, r2_d2
 
-	return eq_str, _rms, r2_d2
 
 def exp_regression(x, y, base):
 
-	y_fit = []
+    y_fit = []
 
-	for i in range(len(y)):
-		try:       
-			y_fit.append(np.log(y[i]) / np.log(base)) #change of base for logs
-		except:
-			pass
+    for i in range(len(y)):
+        try:
+            # change of base for logs
+            y_fit.append(np.log(y[i]) / np.log(base))
+        except:
+            pass
 
-	reg_eq = np.polyfit(x, y_fit, 1, w=np.sqrt(y_fit)) # y = base ^ (reg_eq[0] * x) * base ^ (reg_eq[1])
-	eq_str = "(" + str(base) + "**(" + str(reg_eq[0]) + "*z))*(" + str(base) + "**(" + str(reg_eq[1]) + "))"
-	vals = []
+    # y = base ^ (reg_eq[0] * x) * base ^ (reg_eq[1])
+    reg_eq = np.polyfit(x, y_fit, 1, w=np.sqrt(y_fit))
+    eq_str = "(" + str(base) + "**(" + \
+        str(reg_eq[0]) + "*z))*(" + str(base) + "**(" + str(reg_eq[1]) + "))"
+    vals = []
 
-	for i in range(len(x)):
-		z = x[i]
+    for i in range(len(x)):
+        z = x[i]
 
-		try:
-			exec("vals.append(" + eq_str + ")")
-		except:
-			pass
+        try:
+            exec("vals.append(" + eq_str + ")")
+        except:
+            pass
 
-	_rms = rms(vals, y)
-	r2_d2 = r_squared(vals, y)
+    _rms = rms(vals, y)
+    r2_d2 = r_squared(vals, y)
+
+    return eq_str, _rms, r2_d2
 
-	return eq_str, _rms, r2_d2
 
 def tanh_regression(x, y):
 
-	def tanh (x, a, b, c, d):
+    def tanh(x, a, b, c, d):
 
-		return a * np.tanh(b * (x - c)) + d
+        return a * np.tanh(b * (x - c)) + d
 
-	reg_eq = np.float64(curve_fit(tanh, np.array(x), np.array(y))[0]).tolist()
-	eq_str = str(reg_eq[0]) + " * np.tanh(" + str(reg_eq[1]) + "*(z - " + str(reg_eq[2]) + ")) + " + str(reg_eq[3])
-	vals = []
+    reg_eq = np.float64(curve_fit(tanh, np.array(x), np.array(y))[0]).tolist()
+    eq_str = str(reg_eq[0]) + " * np.tanh(" + str(reg_eq[1]) + \
+        "*(z - " + str(reg_eq[2]) + ")) + " + str(reg_eq[3])
+    vals = []
 
-	for i in range(len(x)):
-		z = x[i]
-		try:
-			exec("vals.append(" + eq_str + ")")
-		except:
-			pass
+    for i in range(len(x)):
+        z = x[i]
+        try:
+            exec("vals.append(" + eq_str + ")")
+        except:
+            pass
 
-	_rms = rms(vals, y)
-	r2_d2 = r_squared(vals, y)
+    _rms = rms(vals, y)
+    r2_d2 = r_squared(vals, y)
 
-	return eq_str, _rms, r2_d2
+    return eq_str, _rms, r2_d2
 
-def r_squared(predictions, targets): # assumes equal size inputs
+
+def r_squared(predictions, targets):  # assumes equal size inputs
 
     return metrics.r2_score(np.array(targets), np.array(predictions))
 
-def rms(predictions, targets): # assumes equal size inputs
+
+def rms(predictions, targets):  # assumes equal size inputs
 
     _sum = 0
 
     for i in range(0, len(targets), 1):
         _sum = (targets[i] - predictions[i]) ** 2
 
-    return float(math.sqrt(_sum/len(targets)))
+    return float(math.sqrt(_sum / len(targets)))
+
 
 def calc_overfit(equation, rms_train, r2_train, x_test, y_test):
 
-    #performance overfit = performance(train) - performance(test) where performance is r^2
-    #error overfit = error(train) - error(test) where error is rms; biased towards smaller values
+    # performance overfit = performance(train) - performance(test) where performance is r^2
+    # error overfit = error(train) - error(test) where error is rms; biased towards smaller values
 
     vals = []
 
@@ -733,19 +765,22 @@ def calc_overfit(equation, rms_train, r2_train, x_test, y_test):
 
     return r2_train - r2_test
 
+
 def strip_data(data, mode):
 
-    if mode == "adam": #x is the row number, y are the data
+    if mode == "adam":  # x is the row number, y are the data
         pass
 
-    if mode == "eve": #x are the data, y is the column number
+    if mode == "eve":  # x are the data, y is the column number
         pass
 
     else:
         raise error("mode error")
 
-def optimize_regression(x, y, _range, resolution):#_range in poly regression is the range of powers tried, and in log/exp it is the inverse of the stepsize taken from -1000 to 1000
-#usage not: for demonstration purpose only, performance is shit
+
+# _range in poly regression is the range of powers tried, and in log/exp it is the inverse of the stepsize taken from -1000 to 1000
+def optimize_regression(x, y, _range, resolution):
+    # usage not: for demonstration purpose only, performance is shit
     if type(resolution) != int:
         raise error("resolution must be int")
 
@@ -758,7 +793,7 @@ def optimize_regression(x, y, _range, resolution):#_range in poly regression is
     x_test = []
     y_test = []
 
-    for i in range (0, math.floor(len(x) * 0.5), 1):
+    for i in range(0, math.floor(len(x) * 0.5), 1):
         index = random.randint(0, len(x) - 1)
 
         x_test.append(x[index])
@@ -774,7 +809,7 @@ def optimize_regression(x, y, _range, resolution):#_range in poly regression is
     rmss = []
     r2s = []
 
-    for i in range (0, _range + 1, 1):
+    for i in range(0, _range + 1, 1):
         try:
             x, y, z = poly_regression(x_train, y_train, i)
             eqs.append(x)
@@ -783,21 +818,21 @@ def optimize_regression(x, y, _range, resolution):#_range in poly regression is
         except:
             pass
 
-    for i in range (1, 100 * resolution + 1):
+    for i in range(1, 100 * resolution + 1):
         try:
-        	x, y, z = exp_regression(x_train, y_train, float(i / resolution))
-        	eqs.append(x)
-        	rmss.append(y)
-        	r2s.append(z)
+            x, y, z = exp_regression(x_train, y_train, float(i / resolution))
+            eqs.append(x)
+            rmss.append(y)
+            r2s.append(z)
         except:
             pass
 
-    for i in range (1, 100 * resolution + 1):
+    for i in range(1, 100 * resolution + 1):
         try:
-        	x, y, z = log_regression(x_train, y_train, float(i / resolution))
-        	eqs.append(x)
-        	rmss.append(y)
-        	r2s.append(z)
+            x, y, z = log_regression(x_train, y_train, float(i / resolution))
+            eqs.append(x)
+            rmss.append(y)
+            r2s.append(z)
         except:
             pass
 
@@ -810,13 +845,14 @@ def optimize_regression(x, y, _range, resolution):#_range in poly regression is
     except:
         pass
 
-    for i in range (0, len(eqs), 1): #marks all equations where r2 = 1 as they 95% of the time overfit the data
+    # marks all equations where r2 = 1 as they 95% of the time overfit the data
+    for i in range(0, len(eqs), 1):
         if r2s[i] == 1:
             eqs[i] = ""
             rmss[i] = ""
             r2s[i] = ""
 
-    while True: #removes all equations marked for removal
+    while True:  # removes all equations marked for removal
         try:
             eqs.remove('')
             rmss.remove('')
@@ -826,12 +862,13 @@ def optimize_regression(x, y, _range, resolution):#_range in poly regression is
 
     overfit = []
 
-    for i in range (0, len(eqs), 1):
+    for i in range(0, len(eqs), 1):
 
         overfit.append(calc_overfit(eqs[i], rmss[i], r2s[i], x_test, y_test))
 
     return eqs, rmss, r2s, overfit
 
+
 def select_best_regression(eqs, rmss, r2s, overfit, selector):
 
     b_eq = ""
@@ -860,32 +897,35 @@ def select_best_regression(eqs, rmss, r2s, overfit, selector):
 
     return b_eq, b_rms, b_r2, b_overfit
 
-def p_value(x, y): #takes 2 1d arrays
 
-	return stats.ttest_ind(x, y)[1]
+def p_value(x, y):  # takes 2 1d arrays
 
-def basic_analysis(data): #assumes that rows are the independent variable and columns are the dependant. also assumes that time flows from lowest column to highest column.
+    return stats.ttest_ind(x, y)[1]
 
-	row = len(data)
-	column = []
 
-	for i in range(0, row, 1):        
-		column.append(len(data[i]))
+# assumes that rows are the independent variable and columns are the dependant. also assumes that time flows from lowest column to highest column.
+def basic_analysis(data):
 
-	column_max = max(column)
-	row_b_stats = []
-	row_histo = []
+    row = len(data)
+    column = []
 
-	for i in range(0, row, 1):
-		row_b_stats.append(basic_stats(data, "row", i))
-		row_histo.append(histo_analysis(data[i], 0.67449, -0.67449, 0.67449))
+    for i in range(0, row, 1):
+        column.append(len(data[i]))
 
-	column_b_stats = []
+    column_max = max(column)
+    row_b_stats = []
+    row_histo = []
 
-	for i in range(0, column_max, 1):
-		column_b_stats.append(basic_stats(data, "column", i))
+    for i in range(0, row, 1):
+        row_b_stats.append(basic_stats(data, "row", i))
+        row_histo.append(histo_analysis(data[i], 0.67449, -0.67449, 0.67449))
 
-	return[row_b_stats, column_b_stats, row_histo]
+    column_b_stats = []
+
+    for i in range(0, column_max, 1):
+        column_b_stats.append(basic_stats(data, "column", i))
+
+    return[row_b_stats, column_b_stats, row_histo]
 
 
 def benchmark(x, y):
@@ -900,22 +940,25 @@ def benchmark(x, y):
 
     return [(end_g - start_g), (end_a - start_a)]
 
+
 def generate_data(filename, x, y, low, high):
 
     file = open(filename, "w")
 
-    for i in range (0, y, 1):
+    for i in range(0, y, 1):
         temp = ""
 
-        for j in range (0, x - 1, 1):
-            temp = str(random.uniform(low, high)) +  ","  + temp
+        for j in range(0, x - 1, 1):
+            temp = str(random.uniform(low, high)) + "," + temp
 
         temp = temp + str(random.uniform(low, high))
         file.write(temp + "\n")
 
+
 class StatisticsError(ValueError):
     pass
 
+
 def _sum(data, start=0):
     count = 0
     n, d = _exact_ratio(start)
@@ -924,7 +967,7 @@ def _sum(data, start=0):
     T = _coerce(int, type(start))
     for typ, values in groupby(data, type):
         T = _coerce(T, typ)  # or raise TypeError
-        for n,d in map(_exact_ratio, values):
+        for n, d in map(_exact_ratio, values):
             count += 1
             partials[d] = partials_get(d, 0) + n
     if None in partials:
@@ -936,26 +979,35 @@ def _sum(data, start=0):
         total = sum(Fraction(n, d) for d, n in sorted(partials.items()))
     return (T, total, count)
 
+
 def _isfinite(x):
     try:
         return x.is_finite()  # Likely a Decimal.
     except AttributeError:
         return math.isfinite(x)  # Coerces to float first.
 
+
 def _coerce(T, S):
 
     assert T is not bool, "initial type T is bool"
 
-    if T is S:  return T
+    if T is S:
+        return T
 
-    if S is int or S is bool:  return T
-    if T is int:  return S
+    if S is int or S is bool:
+        return T
+    if T is int:
+        return S
 
-    if issubclass(S, T):  return S
-    if issubclass(T, S):  return T
+    if issubclass(S, T):
+        return S
+    if issubclass(T, S):
+        return T
 
-    if issubclass(T, int):  return S
-    if issubclass(S, int):  return T
+    if issubclass(T, int):
+        return S
+    if issubclass(S, int):
+        return T
 
     if issubclass(T, Fraction) and issubclass(S, float):
         return S
@@ -965,6 +1017,7 @@ def _coerce(T, S):
     msg = "don't know how to coerce %s and %s"
     raise TypeError(msg % (T.__name__, S.__name__))
 
+
 def _exact_ratio(x):
 
     try:
@@ -988,6 +1041,7 @@ def _exact_ratio(x):
     msg = "can't convert type '{}' to numerator/denominator"
     raise TypeError(msg.format(type(x).__name__))
 
+
 def _convert(value, T):
 
     if type(value) is T:
@@ -1000,10 +1054,11 @@ def _convert(value, T):
         return T(value)
     except TypeError:
         if issubclass(T, Decimal):
-            return T(value.numerator)/T(value.denominator)
+            return T(value.numerator) / T(value.denominator)
         else:
             raise
 
+
 def _counts(data):
 
     table = collections.Counter(iter(data)).most_common()
@@ -1029,8 +1084,8 @@ def _find_lteq(a, x):
 def _find_rteq(a, l, x):
 
     i = bisect_right(a, x, lo=l)
-    if i != (len(a)+1) and a[i-1] == x:
-        return i-1
+    if i != (len(a) + 1) and a[i - 1] == x:
+        return i - 1
     raise ValueError
 
 
@@ -1041,6 +1096,7 @@ def _fail_neg(values, errmsg='negative value'):
             raise StatisticsError(errmsg)
         yield x
 
+
 def mean(data):
 
     if iter(data) is data:
@@ -1050,7 +1106,8 @@ def mean(data):
         raise StatisticsError('mean requires at least one data point')
     T, total, count = _sum(data)
     assert count == n
-    return _convert(total/n, T)
+    return _convert(total / n, T)
+
 
 def median(data):
 
@@ -1058,11 +1115,12 @@ def median(data):
     n = len(data)
     if n == 0:
         raise StatisticsError("no median for empty data")
-    if n%2 == 1:
-        return data[n//2]
+    if n % 2 == 1:
+        return data[n // 2]
     else:
-        i = n//2
-        return (data[i - 1] + data[i])/2
+        i = n // 2
+        return (data[i - 1] + data[i]) / 2
+
 
 def mode(data):
 
@@ -1071,23 +1129,25 @@ def mode(data):
         return table[0][0]
     elif table:
         raise StatisticsError(
-                'no unique mode; found %d equally common values' % len(table)
-                )
+            'no unique mode; found %d equally common values' % len(table)
+        )
     else:
         raise StatisticsError('no mode for empty data')
 
+
 def _ss(data, c=None):
 
     if c is None:
         c = mean(data)
-    T, total, count = _sum((x-c)**2 for x in data)
+    T, total, count = _sum((x - c)**2 for x in data)
 
-    U, total2, count2 = _sum((x-c) for x in data)
+    U, total2, count2 = _sum((x - c) for x in data)
     assert T == U and count == count2
-    total -=  total2**2/len(data)
+    total -= total2**2 / len(data)
     assert not total < 0, 'negative sum of square deviations: %f' % total
     return (T, total)
 
+
 def variance(data, xbar=None):
 
     if iter(data) is data:
@@ -1096,7 +1156,8 @@ def variance(data, xbar=None):
     if n < 2:
         raise StatisticsError('variance requires at least two data points')
     T, ss = _ss(data, xbar)
-    return _convert(ss/(n-1), T)
+    return _convert(ss / (n - 1), T)
+
 
 def stdev(data, xbar=None):
 

data analysis/analysis/compile.bat

@@ -0,0 +1,2 @@
+python setup.py build_ext --inplace
+pause

data analysis/analysis/setup.py

@@ -0,0 +1,5 @@
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(name='analysis',
+      ext_modules=cythonize("analysis.py"))

data analysis/data/scores.csv

@@ -1,36 +1,36 @@
-23,57,61,28,62,31,58,42
-35,58,61,39,51,51,42,31,33,70
-14,6,9,54,48,36,59,46,30,17,68,38
-46,71,40,52,68,57,60,57,60,60,65
-65,43,74,59,68,59,75,62,67,55,60,79,86,67,66,77,71,75
-56,44,26,50,49,41,33,40,45,44,39,53,74,63,65,70,71
-15,49,53,18,53,45,20,55,36,54,49,53,64,71,82,78,67,60,67
-27,16,41,64,48,21,65,61,46,68,46,72,67,61
-33,51,63,85,39,59,44,45,34,89,55,34,46
-42,56,53,50,37,39,52,59,38,43,56,38,42,53
-21,22,59,27,33,32,14,36,53,42,58,58
-10,54,36,44,40,63,41,31,46
-50,45,49,19,55,35,15,33
-26,62,45,37,42,29,59,44,40,47,67,42,64,63,54,60,88,76,80,86,78,78,76
-14,12,61,24,53,39,32,15,44
-47,53,34,48,42,61,34,51,34,45,50,56,46,59,54,47,53,43,62,40,80,66,94,58,64,71,82,78,67,60,67
-42,55,36,34,56,46,26,35,52,70
-50,17,66,53,32,60,32,39,53
-36,45,44,49,49,37,49,36,46
-
-32,45,61,52,34,47,59,62,48,57,71,49,79,50,48,51,55,54,42,55,47,76,43,65,55,104,57,85,55
-10,44,49,20,45,32,38,41
-58,41,38,64,61,39,42,40,54,66,69,63,34
-42,37,62,38,51,44,41,70,41,28
-
-19,55,58,42,22,34,44,45,43,33,45
-26,31,41,26,69,34,12,25
-39,45,49,44,48,49,37,39
-55,45,48,19,53,32,59,56,58,79,50
-33,68,12,18,49,36,39,35,49,35,21,38
-47,51,64,50,102,40,47,31,57,15,54,50,56,64,51,37,62,60,77,62,44,52,52,72,67
-
-9,55,26,45,33,18,52,30,53,42,58,58
-
-
-
+23,57,61,28,62,31,58,42,47,61,52,54,50,68,54,44,53,26,28,68
+35,58,61,39,51,51,42,31,33,70,47,55,44,55,62,43,69,69,65,51,46,68,60,78,96,58,63
+14,6,9,54,48,36,59,46,30,17,68,38,35,39,48,33,43,68,60,51,53,51,67,59
+46,71,40,52,68,57,60,57,60,60,65,79,55,54,47,75,80,72,45,59,64,67,57,63,77,71,72,72,77
+65,43,74,59,68,59,75,62,67,55,60,79,86,67,66,77,71,75,68,67,65,41,75,68,86,92,74,64,65,29,60,78,96,58,63
+56,44,26,50,49,41,33,40,45,44,39,53,74,63,65,70,71,52,71,54,75,52,61,46,53,53,51,48,55,67,46,58
+15,49,53,18,53,45,20,55,36,54,49,53,64,71,82,78,67,60,67,52,52,52,57,55,64,86,71,59,79,84,52,71,85,84,66,63,64
+27,16,41,64,48,21,65,61,46,68,46,72,67,61,51,52,65,55,75,54,60,56,75,55,70,55,63,77,71,72,72,77
+33,51,63,85,39,59,44,45,34,89,55,34,46,47,74,54,57,52,80,42,92,60,45,81,64,63,77,71,72,72,77
+42,56,53,50,37,39,52,59,38,43,56,38,42,53,52,67,52,47,45,57,69,51,63,64,48,30,58,48
+21,22,59,27,33,32,14,36,53,42,58,58,67,48,43,38,62,61,42,60,59,26,70,30,46,50
+10,54,36,44,40,63,41,31,46,79,40,43,55,55,65,52,74,46,48,41,81,70,70,64
+50,45,49,19,55,35,15,33,68,36,48,49,66,61,69,44,60,55,46,49,58,48
+26,62,45,37,42,29,59,44,40,47,67,42,64,63,54,60,88,76,80,86,78,78,76,79,69,66,57,57,43,60,60,63,64,70,64,63,76
+14,12,61,24,53,39,32,15,44,47,67,48,38,41,57,52,52,53,61,46,52
+47,53,34,48,42,61,34,51,34,45,50,56,46,59,54,47,53,43,62,40,80,66,94,58,64,71,82,78,67,60,67,67,58,65,75,66,74,92,65,79,55,83,60,78,96,58,63
+42,55,36,34,56,46,26,35,52,70,51,71,54,33,46,57,49,71,60,46,70,30
+50,17,66,53,32,60,32,39,53,68,39,43,61,41,64,49,69,52,45,28,64
+36,45,44,49,49,37,49,36,46,53,36,66,61,76,80,74,53,60,61,84,68,71,85,84,66,63,64
+53,48,48,41,43,54,46,49,65,46,28,57
+32,45,61,52,34,47,59,62,48,57,71,49,79,50,48,51,55,54,42,55,47,76,43,65,55,104,57,85,55,75,48,44,49,50,72,71,75,55,81,83,81,70,70,64
+10,44,49,20,45,32,38,41,35,69,65,69,46,57,65,53,65,64,70,83,46,50
+58,41,38,64,61,39,42,40,54,66,69,63,34,47,74,65,47,43,61,52,56,62,79,63,64,46,50
+42,37,62,38,51,44,41,70,41,28,32,61,54,61,66,57,60,52,75,51,65,57
+35,71,43,39,76,52,45,63,55,65,41,67,46,58
+19,55,58,42,22,34,44,45,43,33,45,75,39,48,39,57,61,86,46,62,64,55,64,58,48
+26,31,41,26,69,34,12,25,67,52,44,69,45,61,60,45,53,61,70,49
+39,45,49,44,48,49,37,39,75,40,43,41,46,64,42,44,53,64,46,55,71,85,84,66,63,64
+55,45,48,19,53,32,59,56,58,79,50,52,58,45,54,76,60,60,52,64,67,64,67,46,58
+33,68,12,18,49,36,39,35,49,35,21,38,51,55,48,38,57,54,49,52,51,81,29
+47,51,64,50,102,40,47,31,57,15,54,50,56,64,51,37,62,60,77,62,44,52,52,72,67,52,61,43,75,62,47,72,49,62,64,84,49,81,70,70,64
+53,40,45,69,50,43,49,56,60,51,55,59
+9,55,26,45,33,18,52,30,53,42,58,58,52,36,65,39,55,57,54,69,65,55,63,59,63,76
+32,69,45,49,52,54,74,46,74,26,63,52,63,76
+47,58,66,33,45,47,46,46,65,45,46,41
+32,74,54,54,41,47,65,51,60,55,46,29