fix: src/requirements.txt to reduce vulnerabilities

The following vulnerabilities are fixed by pinning transitive dependencies:
- https://snyk.io/vuln/SNYK-PYTHON-SETUPTOOLS-3113904

.devcontainer/Dockerfile

@@ -1,2 +1,7 @@
-FROM python
+FROM ubuntu:20.04
-WORKDIR ~/
+WORKDIR /
+RUN apt-get -y update
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN apt-get install -y python3 python3-dev git python3-pip python3-kivy python-is-python3 libgl1-mesa-dev build-essential
+RUN ln -s $(which pip3) /usr/bin/pip
+RUN pip install pymongo pandas numpy scipy scikit-learn matplotlib pylint kivy

.devcontainer/dev-dockerfile

@@ -0,0 +1,2 @@
+FROM titanscout2022/tra-analysis-base:latest
+WORKDIR /

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 {
     "name": "TRA Analysis Development Environment",
     "build": {
-		"dockerfile": "Dockerfile",
+		"dockerfile": "dev-dockerfile",
     },
     "settings": { 
 		"terminal.integrated.shell.linux": "/bin/bash",
@@ -24,5 +24,5 @@
 		"ms-python.python",
 		"waderyan.gitblame"
 	],
-	"postCreateCommand": "apt install vim -y ; pip install -r src/requirements.txt ; pip install pylint ; pip install tra-analysis; pip install pytest"
+		"postCreateCommand": "/usr/bin/pip3 install -r ${containerWorkspaceFolder}/src/requirements.txt && /usr/bin/pip3 install --no-cache-dir pylint && /usr/bin/pip3 install pytest"
 	}

.gitattributes

@@ -0,0 +1,4 @@
+# Auto detect text files and perform LF normalization
+* text=auto eol=lf
+*.{cmd,[cC][mM][dD]} text eol=crlf
+*.{bat,[bB][aA][tT]} text eol=crlf

.github/workflows/build-cli.yml

@@ -0,0 +1,17 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Superscript Unit Tests
+
+on:
+  release:
+    types: [published, edited]
+
+jobs:
+  generate:
+    name: Build Linux
+    runs-on: ubuntu-latest
+
+    steps: 
+      - name: Checkout master
+      uses: actions/checkout@master

.github/workflows/superscript-unit.yml

@@ -17,8 +17,6 @@ jobs:
       matrix:
         python-version: [3.7, 3.8]
 
-    env:
-      working-directory: ./data-analysis/
 
     steps:
     - uses: actions/checkout@v2
@@ -30,9 +28,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        if [ -f src/requirements.txt ]; then pip install -r src/requirements.txt; fi
-      working-directory: ${{ env.working-directory }}
     - name: Test with pytest
       run: |
-        pytest
+        pytest test/
-      working-directory: ${{ env.working-directory }}

.gitignore

@@ -4,6 +4,11 @@
 **/.pytest_cache/
 **/*.pyc
 
-**/build/
 **/*.egg-info/
-**/dist/
+**/config.json
+**/tra_analysis/
+**/temp/*
+
+**/errorlog.txt
+/dist/superscript.*
+/dist/superscript

LICENSE

@@ -1,6 +1,7 @@
 BSD 3-Clause License
 
-Copyright (c) 2020, Titan Scouting
+Copyright (c) 2021, Titan Scouting
+
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

README.md

@@ -1,4 +1,4 @@
-# Red Alliance Analysis · ![GitHub release (latest by date)](https://img.shields.io/github/v/release/titanscout2022/red-alliance-analysis)
+# Red Alliance Analysis · ![GitHub release (latest by date)](https://img.shields.io/github/v/release/titanscouting/tra-superscript)
 
 Titan Robotics 2022 Strategy Team Repository for Data Analysis Tools. Included with these tools are the backend data analysis engine formatted as a python package, associated binaries for the analysis package, and premade scripts that can be pulled directly from this repository and will integrate with other Red Alliance applications to quickly deploy FRC scouting tools.
 
@@ -42,4 +42,5 @@ don't worry, you may have just not configured the application correctly, but wou
 ---
 
 # Build Statuses
-![Superscript Unit Tests](https://github.com/titanscout2022/red-alliance-analysis/workflows/Superscript%20Unit%20Tests/badge.svg?branch=master)
+
+Coming soon!

build/build-CLI.bat

@@ -0,0 +1,5 @@
+set pathtospec="../src/superscript.spec"
+set pathtodist="../dist/"
+set pathtowork="temp/"
+
+pyinstaller --onefile --clean --distpath %pathtodist% --workpath %pathtowork% %pathtospec%

build/build-CLI.sh

@@ -0,0 +1,5 @@
+pathtospec="../src/superscript.spec"
+pathtodist="../dist/"
+pathtowork="temp/"
+
+pyinstaller --onefile --clean --distpath ${pathtodist} --workpath ${pathtowork} ${pathtospec}

src/config.json

@@ -1,101 +0,0 @@
-{
-	"max-threads": 0.5,
-	"team": "",
-	"competition": "",
-	"key": {
-		"database": "",
-		"tba": ""
-	},
-	"statistics": {
-		"match": {
-			"balls-blocked": [
-				"basic_stats",
-				"historical_analysis",
-				"regression_linear",
-				"regression_logarithmic",
-				"regression_exponential",
-				"regression_polynomial",
-				"regression_sigmoidal"
-			],
-			"balls-collected": [
-				"basic_stats",
-				"historical_analysis",
-				"regression_linear",
-				"regression_logarithmic",
-				"regression_exponential",
-				"regression_polynomial",
-				"regression_sigmoidal"
-			],
-			"balls-lower-teleop": [
-				"basic_stats",
-				"historical_analysis",
-				"regression_linear",
-				"regression_logarithmic",
-				"regression_exponential",
-				"regression_polynomial",
-				"regression_sigmoidal"
-			],
-			"balls-lower-auto": [
-				"basic_stats",
-				"historical_analysis",
-				"regression_linear",
-				"regression_logarithmic",
-				"regression_exponential",
-				"regression_polynomial",
-				"regression_sigmoidal"
-			],
-			"balls-started": [
-				"basic_stats",
-				"historical_analyss",
-				"regression_linear",
-				"regression_logarithmic",
-				"regression_exponential",
-				"regression_polynomial",
-				"regression_sigmoidal"
-			],
-			"balls-upper-teleop": [
-				"basic_stats",
-				"historical_analysis",
-				"regression_linear",
-				"regression_logarithmic",
-				"regression_exponential",
-				"regression_polynomial",
-				"regression_sigmoidal"
-			],
-			"balls-upper-auto": [
-				"basic_stats",
-				"historical_analysis",
-				"regression_linear",
-				"regression_logarithmic",
-				"regression_exponential",
-				"regression_polynomial",
-				"regression_sigmoidal"
-			]
-		},
-		"metric": {
-			"elo": {
-				"score": 1500,
-				"N": 400,
-				"K": 24
-			},
-			"gl2": {
-				"score": 1500,
-				"rd": 250,
-				"vol": 0.06
-			},
-			"ts": {
-				"mu": 25,
-				"sigma": 8.33
-			}
-		},
-		"pit": {
-			"wheel-mechanism": true,
-			"low-balls": true,
-			"high-balls": true,
-			"wheel-success": true,
-			"strategic-focus": true,
-			"climb-mechanism": true,
-			"attitude": true
-		}
-	}
-}

src/requirements.txt

@@ -2,4 +2,18 @@ requests
 pymongo
 pandas
 tra-analysis
+
+dnspython
+pyinstaller
+requests
+pymongo
+
+numpy
+scipy
+scikit-learn
+six
+pyparsing
+pandas
+
 kivy==2.0.0rc2
+setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability

src/superscript.py

@@ -3,10 +3,21 @@
 # Notes:
 # setup:
 
-__version__ = "0.8.2"
+__version__ = "0.8.6"
 
 # changelog should be viewed using print(analysis.__changelog__)
 __changelog__ = """changelog:
+	0.8.6:
+		- added proper main function
+	0.8.5:
+		- added more gradeful KeyboardInterrupt exiting
+		- redirected stderr to errorlog.txt
+	0.8.4:
+		- added better error message for missing config.json
+		- added automatic config.json creation
+		- added splash text with version and system info
+	0.8.3:
+		- updated matchloop with new regression format (requires tra_analysis 3.x)
 	0.8.2:
 		- readded while true to main function
 		- added more thread config options
@@ -130,25 +141,31 @@ import os
 from os import system, name
 from pathlib import Path
 from multiprocessing import Pool
-import matplotlib.pyplot as plt
+import platform
-from concurrent.futures import ThreadPoolExecutor
+import sys
 import time
 import warnings
 
-# global exec_threads
+global exec_threads
 
 def main():
 
-	# global exec_threads
+	global exec_threads
+
+	sys.stderr = open("errorlog.txt", "w")
 
 	warnings.filterwarnings("ignore")
 
-# while (True):
+	splash()
+
+	while (True):
+
+		try:
 
 			current_time = time.time()
 			print("[OK] time: " + str(current_time))
 
-	config = load_config("red-alliance-analysis\data-analysis\config.json")
+			config = load_config("config.json")
 			competition = config["competition"]
 			match_tests = config["statistics"]["match"]
 			pit_tests = config["statistics"]["pit"]
@@ -167,10 +184,10 @@ def main():
 			elif cfg_max_threads == 0:
 				alloc_processes = sys_max_threads
 			else:
-		print("[Err] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads))
+				print("[ERROR] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads))
 				exit()
-	# exec_threads = Pool(processes = alloc_processes)
+			exec_threads = Pool(processes = alloc_processes)
-	# print("[OK] " + str(alloc_processes) + " threads started")
+			print("[OK] " + str(alloc_processes) + " threads started")
 
 			apikey = config["key"]["database"]
 			tbakey = config["key"]["tba"]
@@ -203,6 +220,17 @@ def main():
 			set_current_time(apikey, current_time)
 			print("[OK] finished all tests, looping")
 
+			print_hrule()
+		
+		except KeyboardInterrupt:
+			print("\n[OK] caught KeyboardInterrupt, killing processes")
+			exec_threads.terminate()
+			print("[OK] processes killed, exiting")
+			exit()
+
+		else:
+			pass
+
 		#clear()
 
 def clear(): 
@@ -215,10 +243,38 @@ def clear():
 	else: 
 		_ = system('clear')
 
+def print_hrule():
+
+	print("#"+38*"-"+"#")
+
+def print_box(s):
+
+	temp = "|"
+	temp += s
+	temp += (40-len(s)-2)*" "
+	temp += "|"
+	print(temp)
+
+def splash():
+
+	print_hrule()
+	print_box(" superscript version: " + __version__)
+	print_box(" os: " + platform.system())
+	print_box(" python: " + platform.python_version())
+	print_hrule()
+
 def load_config(file):
 
 	config_vector = {}
-	with open(file) as f:
+
+	try:
+		f = open(file)
+	except:
+		print("[ERROR] could not locate config.json, generating blank config.json and exiting")
+		f = open(file, "w")
+		f.write(sample_json)
+		exit()
+	
 	config_vector = json.load(f)
 
 	return config_vector
@@ -284,6 +340,8 @@ def matchloop(apikey, competition, data, tests): # expects 3D array with [Team][
 
 	global exec_threads
 
+	short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"}
+
 	class AutoVivification(dict):
 		def __getitem__(self, item):
 			try:
@@ -314,13 +372,19 @@ def matchloop(apikey, competition, data, tests): # expects 3D array with [Team][
 					variable_data.append((data[team][variable], test))
 					test_filtered.append(test)
 
-	result_filtered = map(simplestats, variable_data)
+	result_filtered = exec_threads.map(simplestats, variable_data)
 	i = 0
 
 	result_filtered = list(result_filtered)
 
 	for result in result_filtered:
 
+		filtered = test_filtered[i]
+
+		try:
+			short = short_mapping[filtered]
+			return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result[short]
+		except KeyError: # not in mapping
 			return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result
 		i += 1
 
@@ -510,27 +574,54 @@ def get_team_metrics(apikey, tbakey, competition):
 
 	return {"elo-ranks": elo_ranked, "glicko2-ranks": gl2_ranked}
 
-def graph_pit_histogram(apikey, competition, figsize=(80,15)):
+sample_json = """{
+	"max-threads": 0.5,
+	"team": "",
+	"competition": "2020ilch",
+	"key":{
+		"database":"",
+		"tba":""
+	},
+	"statistics":{
+		"match":{
+			"balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"],
+			"balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"],
+			"balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"],
+			"balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"],
+			"balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"],
+			"balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"],
+			"balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"]
 
-	pit = d.get_pit_variable_formatted(apikey, competition)
+		},
-
+		"metric":{
-	fig, ax = plt.subplots(1, len(pit), sharey=True, figsize=figsize)
+			"elo":{
-
+				"score":1500,
-	i = 0
+				"N":400,
-
+				"K":24
-	for variable in pit:
+			},
-
+			"gl2":{
-		ax[i].hist(pit[variable])
+				"score":1500,
-		ax[i].invert_xaxis()
+				"rd":250,
-
+				"vol":0.06
-		ax[i].set_xlabel('')
+			},
-		ax[i].set_ylabel('Frequency')
+			"ts":{
-		ax[i].set_title(variable)
+				"mu":25,
-
+				"sigma":8.33
-		plt.yticks(np.arange(len(pit[variable])))
+			}
-
+		},
-		i+=1
+		"pit":{
-
+			"wheel-mechanism":true,
-	plt.show()
+			"low-balls":true,
+			"high-balls":true,
+			"wheel-success":true,
+			"strategic-focus":true,
+			"climb-mechanism":true,
+			"attitude":true
+		}
+	}
+}"""
 
+if __name__ == "__main__":
+	if sys.platform.startswith('win'):
+		multiprocessing.freeze_support()
 	main()

src/superscript.spec

@@ -0,0 +1,37 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+
+a = Analysis(['superscript.py'],
+             pathex=['/workspaces/tra-data-analysis/src'],
+             binaries=[],
+             datas=[],
+             hiddenimports=[
+                 "dnspython", 
+                 "sklearn.utils._weight_vector",
+                 "requests",
+             ],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          [('W ignore', None, 'OPTION')],
+          name='superscript',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          upx_exclude=[],
+          runtime_tmpdir=None,
+          console=True )