added debug option,

improved memory usage slightly,
changed errorlog.txt to errorlog.log,
updated .gitignore


Former-commit-id: bd51efc6c2

.gitignore

@@ -13,6 +13,7 @@
 
 **/profile.*
 
+**/*.log
 **/errorlog.txt
 /dist/superscript.*
 /dist/superscript

src/cli/processing.py

@@ -9,11 +9,11 @@ def simplestats(data_test):
 
 	signal.signal(signal.SIGINT, signal.SIG_IGN)
 
-	data = np.array(data_test[0])
+	data = np.array(data_test[3])
 	data = data[np.isfinite(data)]
 	ranges = list(range(len(data)))
 
-	test = data_test[1]
+	test = data_test[2]
 
 	if test == "basic_stats":
 		return an.basic_stats(data)
@@ -48,13 +48,7 @@ def matchloop(client, competition, data, tests, exec_threads):
 				value = self[item] = type(self)()
 				return value
 
-	return_vector = {}
+	input_vector = []
-	
-	team_filtered = []
-	variable_filtered = []
-	variable_data = []
-	test_filtered = []
-	result_filtered = []
 	return_vector = AutoVivification()
 
 	for team in data:
@@ -65,25 +59,24 @@ def matchloop(client, competition, data, tests, exec_threads):
 
 				for test in tests[variable]:
 
-					team_filtered.append(team)
+					input_vector.append((team, variable, test, data[team][variable]))
-					variable_filtered.append(variable)
-					variable_data.append((data[team][variable], test))
-					test_filtered.append(test)
 
-	result_filtered = exec_threads.map(simplestats, variable_data)
+	result_filtered = exec_threads.map(simplestats, input_vector)
+	
 	i = 0
 
 	result_filtered = list(result_filtered)
 
 	for result in result_filtered:
 
-		filtered = test_filtered[i]
+		filtered = input_vector[i][2]
 
 		try:
 			short = short_mapping[filtered]
-			return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result[short]
+			return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result[short]
 		except KeyError: # not in mapping
-			return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result
+			return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result
+
 		i += 1
 
 	return return_vector

src/cli/superscript.py

@@ -10,10 +10,12 @@ __changelog__ = """changelog:
 	1.0.0:
 		- superscript now runs in PEP 3143 compliant well behaved daemon on Linux systems
 		- linux superscript daemon has integrated websocket output to monitor progress/status remotely
-		- linux daemon now sends stderr to errorlog.txt
+		- linux daemon now sends stderr to errorlog.log
 		- added verbose option to linux superscript to allow for interactive output
 		- moved pymongo import to superscript.py
 		- added profile option to linux superscript to profile runtime of script
+		- reduced memory usage slightly by consolidating the unwrapped input data
+		- added debug option, which performs one loop of analysis and dumps results to local files
 	0.9.3:
 		- improved data loading performance by removing redundant PyMongo client creation (120s to 14s)
 		- passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions
@@ -206,10 +208,10 @@ sample_json = """{
 	}
 }"""
 
-def main(send, verbose = False, profile = False):
+def main(send, verbose = False, profile = False, debug = False):
 
 	warnings.filterwarnings("ignore")
-	sys.stderr = open("errorlog.txt", "w")
+	sys.stderr = open("errorlog.log", "w")
 	loop_exit_code = 0
 	loop_stored_exception = None
 
@@ -348,6 +350,11 @@ def main(send, verbose = False, profile = False):
 			results = matchloop(client, competition, match_data, match_tests, exec_threads)
 			send(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds")
 
+			if debug:
+				f = open("matchloop.log", "w+")
+				json.dump(results, f)
+				f.close()
+
 			start = time.time()
 			send(stdout, INF, "uploading match results to database")
 			push_match(client, competition, results)
@@ -368,6 +375,11 @@ def main(send, verbose = False, profile = False):
 			results = pitloop(client, competition, pit_data, pit_tests)
 			send(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds")
 
+			if debug:
+				f = open("pitloop.log", "w+")
+				json.dump(results, f)
+				f.close()
+
 			start = time.time()
 			send(stdout, INF, "uploading pit results to database")
 			push_pit(client, competition, results)
@@ -420,7 +432,7 @@ def save_config(path, config_vector):
 	except:
 		return 1
 
-def start(pid_path, verbose = False, profile = False):
+def start(pid_path, verbose = False, profile = False, debug = False):
 
 	if profile:
 
@@ -440,6 +452,10 @@ def start(pid_path, verbose = False, profile = False):
 
 		main(log, verbose = verbose)
 
+	elif debug:
+
+		main(log, verbose = True, profile = True, debug = debug)
+
 	else:
 
 		f = open('errorlog.txt', 'w+')
@@ -527,6 +543,8 @@ if __name__ == "__main__":
 				start(None, verbose = True)
 			elif 'profile' == sys.argv[1]:
 				start(None, profile=True)
+			elif 'debug' == sys.argv[1]:
+				start(None, debug = True)
 			else:
 				print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0])
 				sys.exit(2)