Merge branch 'c'

data analysis/analysis/analysis.py

@@ -153,9 +153,11 @@ from sklearn import *
 import time
 import torch
 
+
 class error(ValueError):
     pass
 
+
 def _init_device(setting, arg):  # initiates computation device for ANNs
     if setting == "cuda":
         try:
@@ -170,6 +172,7 @@ def _init_device (setting, arg): #initiates computation device for ANNs
     else:
         raise error("specified device does not exist")
 
+
 class c_entities:
 
     c_names = []
@@ -190,7 +193,6 @@ class c_entities:
         self.c_logic = logic
         return None
 
-
     def append(self, n_name, n_id, n_pos, n_property, n_logic):
         self.c_names.append(n_name)
         self.c_ids.append(n_id)
@@ -232,6 +234,7 @@ class c_entities:
     def regurgitate(self):
         return[self.c_names, self.c_ids, self.c_pos, self.c_properties, self.c_logic]
 
+
 class nc_entities:
 
     c_names = []
@@ -295,6 +298,7 @@ class nc_entities:
 
         return[self.c_names, self.c_ids, self.c_pos, self.c_properties, self.c_effects]
 
+
 class obstacles:
 
     c_names = []
@@ -351,6 +355,7 @@ class obstacles:
     def regurgitate(self):
         return[self.c_names, self.c_ids, self.c_perim, self.c_effects]
 
+
 class objectives:
 
     c_names = []
@@ -408,13 +413,16 @@ class objectives:
     def regurgitate(self):
         return[self.c_names, self.c_ids, self.c_pos, self.c_effects]
 
+
 def load_csv(filepath):
     with open(filepath, newline='') as csvfile:
         file_array = list(csv.reader(csvfile))
         csvfile.close()
     return file_array
 
-def basic_stats(data, method, arg): # data=array, mode = ['1d':1d_basic_stats, 'column':c_basic_stats, 'row':r_basic_stats], arg for mode 1 or mode 2 for column or row
+
+# data=array, mode = ['1d':1d_basic_stats, 'column':c_basic_stats, 'row':r_basic_stats], arg for mode 1 or mode 2 for column or row
+def basic_stats(data, method, arg):
 
     if method == 'debug':
         return "basic_stats requires 3 args: data, mode, arg; where data is data to be analyzed, mode is an int from 0 - 2 depending on type of analysis (by column or by row) and is only applicable to 2d arrays (for 1d arrays use mode 1), and arg is row/column number for mode 1 or mode 2; function returns: [mean, median, mode, stdev, variance]"
@@ -498,11 +506,15 @@ def basic_stats(data, method, arg): # data=array, mode = ['1d':1d_basic_stats, '
     else:
         raise error("method error")
 
-def z_score(point, mean, stdev): #returns z score with inputs of point, mean and standard deviation of spread
+
+# returns z score with inputs of point, mean and standard deviation of spread
+def z_score(point, mean, stdev):
     score = (point - mean) / stdev
     return score
 
-def z_normalize(x, y, mode): #mode is either 'x' or 'y' or 'both' depending on the variable(s) to be normalized
+
+# mode is either 'x' or 'y' or 'both' depending on the variable(s) to be normalized
+def z_normalize(x, y, mode):
 
     x_norm = []
     y_norm = []
@@ -543,19 +555,23 @@ def z_normalize(x, y, mode): #mode is either 'x' or 'y' or 'both' depending on t
 
         return error('method error')
 
-def stdev_z_split(mean, stdev, delta, low_bound, high_bound): #returns n-th percentile of spread given mean, standard deviation, lower z-score, and upper z-score
+
+# returns n-th percentile of spread given mean, standard deviation, lower z-score, and upper z-score
+def stdev_z_split(mean, stdev, delta, low_bound, high_bound):
 
     z_split = []
     i = low_bound
 
     while True:
-        z_split.append(float((1 / (stdev * math.sqrt(2 * math.pi))) * math.e ** (-0.5 * (((i - mean) / stdev) ** 2))))
+        z_split.append(float((1 / (stdev * math.sqrt(2 * math.pi))) *
+                             math.e ** (-0.5 * (((i - mean) / stdev) ** 2))))
         i = i + delta
         if i > high_bound:
             break
 
     return z_split
 
+
 def histo_analysis(hist_data, delta, low_bound, high_bound):
 
     if hist_data == 'debug':
@@ -593,6 +609,7 @@ def histo_analysis(hist_data, delta, low_bound, high_bound):
 
     return predictions
 
+
 def poly_regression(x, y, power):
 
     if x == "null":  # if x is 'null', then x will be filled with integer points between 1 and the size of y
@@ -607,9 +624,11 @@ def poly_regression(x, y, power):
 
     for i in range(0, len(reg_eq), 1):
         if i < len(reg_eq) - 1:
-            eq_str = eq_str + str(reg_eq[i]) + "*(z**" + str(len(reg_eq) - i - 1) + ")+"
+            eq_str = eq_str + str(reg_eq[i]) + \
+                "*(z**" + str(len(reg_eq) - i - 1) + ")+"
         else:
-            eq_str = eq_str + str(reg_eq[i]) + "*(z**" + str(len(reg_eq) - i - 1) + ")"
+            eq_str = eq_str + str(reg_eq[i]) + \
+                "*(z**" + str(len(reg_eq) - i - 1) + ")"
 
     vals = []
 
@@ -626,18 +645,22 @@ def poly_regression(x, y, power):
 
     return [eq_str, _rms, r2_d2]
 
+
 def log_regression(x, y, base):
 
     x_fit = []
 
     for i in range(len(x)):
         try:
-			x_fit.append(np.log(x[i]) / np.log(base)) #change of base for logs
+            # change of base for logs
+            x_fit.append(np.log(x[i]) / np.log(base))
         except:
             pass
 
-	reg_eq = np.polyfit(x_fit, y, 1) # y = reg_eq[0] * log(x, base) + reg_eq[1]
-	q_str = str(reg_eq[0]) + "* (np.log(z) / np.log(" + str(base) +"))+" + str(reg_eq[1])
+    # y = reg_eq[0] * log(x, base) + reg_eq[1]
+    reg_eq = np.polyfit(x_fit, y, 1)
+    q_str = str(reg_eq[0]) + "* (np.log(z) / np.log(" + \
+        str(base) + "))+" + str(reg_eq[1])
     vals = []
 
     for i in range(len(x)):
@@ -653,18 +676,22 @@ def log_regression(x, y, base):
 
     return eq_str, _rms, r2_d2
 
+
 def exp_regression(x, y, base):
 
     y_fit = []
 
     for i in range(len(y)):
         try:
-			y_fit.append(np.log(y[i]) / np.log(base)) #change of base for logs
+            # change of base for logs
+            y_fit.append(np.log(y[i]) / np.log(base))
         except:
             pass
 
-	reg_eq = np.polyfit(x, y_fit, 1, w=np.sqrt(y_fit)) # y = base ^ (reg_eq[0] * x) * base ^ (reg_eq[1])
-	eq_str = "(" + str(base) + "**(" + str(reg_eq[0]) + "*z))*(" + str(base) + "**(" + str(reg_eq[1]) + "))"
+    # y = base ^ (reg_eq[0] * x) * base ^ (reg_eq[1])
+    reg_eq = np.polyfit(x, y_fit, 1, w=np.sqrt(y_fit))
+    eq_str = "(" + str(base) + "**(" + \
+        str(reg_eq[0]) + "*z))*(" + str(base) + "**(" + str(reg_eq[1]) + "))"
     vals = []
 
     for i in range(len(x)):
@@ -680,6 +707,7 @@ def exp_regression(x, y, base):
 
     return eq_str, _rms, r2_d2
 
+
 def tanh_regression(x, y):
 
     def tanh(x, a, b, c, d):
@@ -687,7 +715,8 @@ def tanh_regression(x, y):
         return a * np.tanh(b * (x - c)) + d
 
     reg_eq = np.float64(curve_fit(tanh, np.array(x), np.array(y))[0]).tolist()
-	eq_str = str(reg_eq[0]) + " * np.tanh(" + str(reg_eq[1]) + "*(z - " + str(reg_eq[2]) + ")) + " + str(reg_eq[3])
+    eq_str = str(reg_eq[0]) + " * np.tanh(" + str(reg_eq[1]) + \
+        "*(z - " + str(reg_eq[2]) + ")) + " + str(reg_eq[3])
     vals = []
 
     for i in range(len(x)):
@@ -702,10 +731,12 @@ def tanh_regression(x, y):
 
     return eq_str, _rms, r2_d2
 
+
 def r_squared(predictions, targets):  # assumes equal size inputs
 
     return metrics.r2_score(np.array(targets), np.array(predictions))
 
+
 def rms(predictions, targets):  # assumes equal size inputs
 
     _sum = 0
@@ -715,6 +746,7 @@ def rms(predictions, targets): # assumes equal size inputs
 
     return float(math.sqrt(_sum / len(targets)))
 
+
 def calc_overfit(equation, rms_train, r2_train, x_test, y_test):
 
     # performance overfit = performance(train) - performance(test) where performance is r^2
@@ -733,6 +765,7 @@ def calc_overfit(equation, rms_train, r2_train, x_test, y_test):
 
     return r2_train - r2_test
 
+
 def strip_data(data, mode):
 
     if mode == "adam":  # x is the row number, y are the data
@@ -744,7 +777,9 @@ def strip_data(data, mode):
     else:
         raise error("mode error")
 
-def optimize_regression(x, y, _range, resolution):#_range in poly regression is the range of powers tried, and in log/exp it is the inverse of the stepsize taken from -1000 to 1000
+
+# _range in poly regression is the range of powers tried, and in log/exp it is the inverse of the stepsize taken from -1000 to 1000
+def optimize_regression(x, y, _range, resolution):
     # usage not: for demonstration purpose only, performance is shit
     if type(resolution) != int:
         raise error("resolution must be int")
@@ -810,7 +845,8 @@ def optimize_regression(x, y, _range, resolution):#_range in poly regression is
     except:
         pass
 
-    for i in range (0, len(eqs), 1): #marks all equations where r2 = 1 as they 95% of the time overfit the data
+    # marks all equations where r2 = 1 as they 95% of the time overfit the data
+    for i in range(0, len(eqs), 1):
         if r2s[i] == 1:
             eqs[i] = ""
             rmss[i] = ""
@@ -832,6 +868,7 @@ def optimize_regression(x, y, _range, resolution):#_range in poly regression is
 
     return eqs, rmss, r2s, overfit
 
+
 def select_best_regression(eqs, rmss, r2s, overfit, selector):
 
     b_eq = ""
@@ -860,11 +897,14 @@ def select_best_regression(eqs, rmss, r2s, overfit, selector):
 
     return b_eq, b_rms, b_r2, b_overfit
 
+
 def p_value(x, y):  # takes 2 1d arrays
 
     return stats.ttest_ind(x, y)[1]
 
-def basic_analysis(data): #assumes that rows are the independent variable and columns are the dependant. also assumes that time flows from lowest column to highest column.
+
+# assumes that rows are the independent variable and columns are the dependant. also assumes that time flows from lowest column to highest column.
+def basic_analysis(data):
 
     row = len(data)
     column = []
@@ -900,6 +940,7 @@ def benchmark(x, y):
 
     return [(end_g - start_g), (end_a - start_a)]
 
+
 def generate_data(filename, x, y, low, high):
 
     file = open(filename, "w")
@@ -913,9 +954,11 @@ def generate_data(filename, x, y, low, high):
         temp = temp + str(random.uniform(low, high))
         file.write(temp + "\n")
 
+
 class StatisticsError(ValueError):
     pass
 
+
 def _sum(data, start=0):
     count = 0
     n, d = _exact_ratio(start)
@@ -936,26 +979,35 @@ def _sum(data, start=0):
         total = sum(Fraction(n, d) for d, n in sorted(partials.items()))
     return (T, total, count)
 
+
 def _isfinite(x):
     try:
         return x.is_finite()  # Likely a Decimal.
     except AttributeError:
         return math.isfinite(x)  # Coerces to float first.
 
+
 def _coerce(T, S):
 
     assert T is not bool, "initial type T is bool"
 
-    if T is S:  return T
+    if T is S:
+        return T
 
-    if S is int or S is bool:  return T
-    if T is int:  return S
+    if S is int or S is bool:
+        return T
+    if T is int:
+        return S
 
-    if issubclass(S, T):  return S
-    if issubclass(T, S):  return T
+    if issubclass(S, T):
+        return S
+    if issubclass(T, S):
+        return T
 
-    if issubclass(T, int):  return S
-    if issubclass(S, int):  return T
+    if issubclass(T, int):
+        return S
+    if issubclass(S, int):
+        return T
 
     if issubclass(T, Fraction) and issubclass(S, float):
         return S
@@ -965,6 +1017,7 @@ def _coerce(T, S):
     msg = "don't know how to coerce %s and %s"
     raise TypeError(msg % (T.__name__, S.__name__))
 
+
 def _exact_ratio(x):
 
     try:
@@ -988,6 +1041,7 @@ def _exact_ratio(x):
     msg = "can't convert type '{}' to numerator/denominator"
     raise TypeError(msg.format(type(x).__name__))
 
+
 def _convert(value, T):
 
     if type(value) is T:
@@ -1004,6 +1058,7 @@ def _convert(value, T):
         else:
             raise
 
+
 def _counts(data):
 
     table = collections.Counter(iter(data)).most_common()
@@ -1041,6 +1096,7 @@ def _fail_neg(values, errmsg='negative value'):
             raise StatisticsError(errmsg)
         yield x
 
+
 def mean(data):
 
     if iter(data) is data:
@@ -1052,6 +1108,7 @@ def mean(data):
     assert count == n
     return _convert(total / n, T)
 
+
 def median(data):
 
     data = sorted(data)
@@ -1064,6 +1121,7 @@ def median(data):
         i = n // 2
         return (data[i - 1] + data[i]) / 2
 
+
 def mode(data):
 
     table = _counts(data)
@@ -1076,6 +1134,7 @@ def mode(data):
     else:
         raise StatisticsError('no mode for empty data')
 
+
 def _ss(data, c=None):
 
     if c is None:
@@ -1088,6 +1147,7 @@ def _ss(data, c=None):
     assert not total < 0, 'negative sum of square deviations: %f' % total
     return (T, total)
 
+
 def variance(data, xbar=None):
 
     if iter(data) is data:
@@ -1098,6 +1158,7 @@ def variance(data, xbar=None):
     T, ss = _ss(data, xbar)
     return _convert(ss / (n - 1), T)
 
+
 def stdev(data, xbar=None):
 
     var = variance(data, xbar)

data analysis/analysis/compile.bat

@@ -0,0 +1,2 @@
+python setup.py build_ext --inplace
+pause

data analysis/analysis/setup.py

@@ -0,0 +1,5 @@
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(name='analysis',
+      ext_modules=cythonize("analysis.py"))

data analysis/data/scores.csv

@@ -1,36 +1,36 @@
-23,57,61,28,62,31,58,42
-35,58,61,39,51,51,42,31,33,70
-14,6,9,54,48,36,59,46,30,17,68,38
-46,71,40,52,68,57,60,57,60,60,65
-65,43,74,59,68,59,75,62,67,55,60,79,86,67,66,77,71,75
-56,44,26,50,49,41,33,40,45,44,39,53,74,63,65,70,71
-15,49,53,18,53,45,20,55,36,54,49,53,64,71,82,78,67,60,67
-27,16,41,64,48,21,65,61,46,68,46,72,67,61
-33,51,63,85,39,59,44,45,34,89,55,34,46
-42,56,53,50,37,39,52,59,38,43,56,38,42,53
-21,22,59,27,33,32,14,36,53,42,58,58
-10,54,36,44,40,63,41,31,46
-50,45,49,19,55,35,15,33
-26,62,45,37,42,29,59,44,40,47,67,42,64,63,54,60,88,76,80,86,78,78,76
-14,12,61,24,53,39,32,15,44
-47,53,34,48,42,61,34,51,34,45,50,56,46,59,54,47,53,43,62,40,80,66,94,58,64,71,82,78,67,60,67
-42,55,36,34,56,46,26,35,52,70
-50,17,66,53,32,60,32,39,53
-36,45,44,49,49,37,49,36,46
-
-32,45,61,52,34,47,59,62,48,57,71,49,79,50,48,51,55,54,42,55,47,76,43,65,55,104,57,85,55
-10,44,49,20,45,32,38,41
-58,41,38,64,61,39,42,40,54,66,69,63,34
-42,37,62,38,51,44,41,70,41,28
-
-19,55,58,42,22,34,44,45,43,33,45
-26,31,41,26,69,34,12,25
-39,45,49,44,48,49,37,39
-55,45,48,19,53,32,59,56,58,79,50
-33,68,12,18,49,36,39,35,49,35,21,38
-47,51,64,50,102,40,47,31,57,15,54,50,56,64,51,37,62,60,77,62,44,52,52,72,67
-
-9,55,26,45,33,18,52,30,53,42,58,58
-
-
-
+23,57,61,28,62,31,58,42,47,61,52,54,50,68,54,44,53,26,28,68
+35,58,61,39,51,51,42,31,33,70,47,55,44,55,62,43,69,69,65,51,46,68,60,78,96,58,63
+14,6,9,54,48,36,59,46,30,17,68,38,35,39,48,33,43,68,60,51,53,51,67,59
+46,71,40,52,68,57,60,57,60,60,65,79,55,54,47,75,80,72,45,59,64,67,57,63,77,71,72,72,77
+65,43,74,59,68,59,75,62,67,55,60,79,86,67,66,77,71,75,68,67,65,41,75,68,86,92,74,64,65,29,60,78,96,58,63
+56,44,26,50,49,41,33,40,45,44,39,53,74,63,65,70,71,52,71,54,75,52,61,46,53,53,51,48,55,67,46,58
+15,49,53,18,53,45,20,55,36,54,49,53,64,71,82,78,67,60,67,52,52,52,57,55,64,86,71,59,79,84,52,71,85,84,66,63,64
+27,16,41,64,48,21,65,61,46,68,46,72,67,61,51,52,65,55,75,54,60,56,75,55,70,55,63,77,71,72,72,77
+33,51,63,85,39,59,44,45,34,89,55,34,46,47,74,54,57,52,80,42,92,60,45,81,64,63,77,71,72,72,77
+42,56,53,50,37,39,52,59,38,43,56,38,42,53,52,67,52,47,45,57,69,51,63,64,48,30,58,48
+21,22,59,27,33,32,14,36,53,42,58,58,67,48,43,38,62,61,42,60,59,26,70,30,46,50
+10,54,36,44,40,63,41,31,46,79,40,43,55,55,65,52,74,46,48,41,81,70,70,64
+50,45,49,19,55,35,15,33,68,36,48,49,66,61,69,44,60,55,46,49,58,48
+26,62,45,37,42,29,59,44,40,47,67,42,64,63,54,60,88,76,80,86,78,78,76,79,69,66,57,57,43,60,60,63,64,70,64,63,76
+14,12,61,24,53,39,32,15,44,47,67,48,38,41,57,52,52,53,61,46,52
+47,53,34,48,42,61,34,51,34,45,50,56,46,59,54,47,53,43,62,40,80,66,94,58,64,71,82,78,67,60,67,67,58,65,75,66,74,92,65,79,55,83,60,78,96,58,63
+42,55,36,34,56,46,26,35,52,70,51,71,54,33,46,57,49,71,60,46,70,30
+50,17,66,53,32,60,32,39,53,68,39,43,61,41,64,49,69,52,45,28,64
+36,45,44,49,49,37,49,36,46,53,36,66,61,76,80,74,53,60,61,84,68,71,85,84,66,63,64
+53,48,48,41,43,54,46,49,65,46,28,57
+32,45,61,52,34,47,59,62,48,57,71,49,79,50,48,51,55,54,42,55,47,76,43,65,55,104,57,85,55,75,48,44,49,50,72,71,75,55,81,83,81,70,70,64
+10,44,49,20,45,32,38,41,35,69,65,69,46,57,65,53,65,64,70,83,46,50
+58,41,38,64,61,39,42,40,54,66,69,63,34,47,74,65,47,43,61,52,56,62,79,63,64,46,50
+42,37,62,38,51,44,41,70,41,28,32,61,54,61,66,57,60,52,75,51,65,57
+35,71,43,39,76,52,45,63,55,65,41,67,46,58
+19,55,58,42,22,34,44,45,43,33,45,75,39,48,39,57,61,86,46,62,64,55,64,58,48
+26,31,41,26,69,34,12,25,67,52,44,69,45,61,60,45,53,61,70,49
+39,45,49,44,48,49,37,39,75,40,43,41,46,64,42,44,53,64,46,55,71,85,84,66,63,64
+55,45,48,19,53,32,59,56,58,79,50,52,58,45,54,76,60,60,52,64,67,64,67,46,58
+33,68,12,18,49,36,39,35,49,35,21,38,51,55,48,38,57,54,49,52,51,81,29
+47,51,64,50,102,40,47,31,57,15,54,50,56,64,51,37,62,60,77,62,44,52,52,72,67,52,61,43,75,62,47,72,49,62,64,84,49,81,70,70,64
+53,40,45,69,50,43,49,56,60,51,55,59
+9,55,26,45,33,18,52,30,53,42,58,58,52,36,65,39,55,57,54,69,65,55,63,59,63,76
+32,69,45,49,52,54,74,46,74,26,63,52,63,76
+47,58,66,33,45,47,46,46,65,45,46,41
+32,74,54,54,41,47,65,51,60,55,46,29