From 3273bdef5ddb30e32977e124f61dc0db6ca982ee Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Thu, 1 Apr 2021 13:09:18 -0500 Subject: [PATCH 001/143] Create README.md --- README.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..8589af6 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ + +# tra-data-analysis + +To facilitate data analysis of collected scouting data in a user firendly tool, we created the data-analysis application. At its core it uses the tra-analysis package to conduct any number of user selected tests on data collected from the TRA scouting app. It uploads these tests back to MongoDB where it can be viewed from the app at any time. + +The data-analysis application also uses the TRA API to interface with MongoDB and uses the TBA API to collect additional data (match win/loss). + +The application can be configured with a configuration tool or by editing the config.json directly. + +## Prerequisites + +--- + +Before installing and using data-analysis, make sure that you have installed the folowing prerequisites: +- A common operating system like **Windows** or (*most*) distributions of **Linux**. BSD may work but has not been tested nor is it reccomended. +- [Python](https://www.python.org/) version **3.6** or higher +- [Pip](https://pip.pypa.io/en/stable/) (installation instructions [here](https://pip.pypa.io/en/stable/installing/)) + +## Installing Requirements + +--- + +Once navigated to the data-analysis folder run `pip install -r requirements.txt` to install all of the required python libraries. + +## Scripts + +--- + +The data-analysis application is a collection of various scripts and one config file. For users, only the main application `superscript.py` and the config file `config.json` are important. + +To run the data-analysis application, navigate to the data-analysis folder once all requirements have been installed and run `python superscript.py`. If you encounter the error: + +`pymongo.errors.ConfigurationError: Empty host (or extra comma in host list).` + +Don't worry, you may have just not configured the application correctly, but would otherwise work. Refer to [the documentation](https://titanscouting.github.io/analysis/data_analysis/Config) to learn how to configure data-analysis. + +# Contributing + +Read our included contributing guidelines (`CONTRIBUTING.md`) for more information and feel free to reach out to any current maintainer for more information. + +# Build Statuses + +Coming soon! From bc0405665c07afdf100fc3a9ca6cf3396b53c11d Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Thu, 1 Apr 2021 13:10:14 -0500 Subject: [PATCH 002/143] Create CONTRIBUTING.md --- CONTRIBUTING.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4e220db --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,66 @@ +# Contributing Guidelines + +This project accept contributions via GitHub pull requests. +This document outlines some of the +conventions on development workflow, commit message formatting, contact points, +and other resources to make it easier to get your contribution accepted. + +## Certificate of Origin + +By contributing to this project, you agree to the [Developer Certificate of +Origin (DCO)](https://developercertificate.org/). This document was created by the Linux Kernel community and is a +simple statement that you, as a contributor, have the legal right to make the +contribution. + +In order to show your agreement with the DCO you should include at the end of the commit message, +the following line: `Signed-off-by: John Doe `, using your real name. + +This can be done easily using the [`-s`](https://github.com/git/git/blob/b2c150d3aa82f6583b9aadfecc5f8fa1c74aca09/Documentation/git-commit.txt#L154-L161) flag on the `git commit`. + +Visual Studio code also has a flag to enable signoff on commits + +If you find yourself pushed a few commits without `Signed-off-by`, you can still add it afterwards. Read this for help: [fix-DCO.md](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md). + +## Support Channels + +The official support channel, for both users and contributors, is: + +- GitHub issues: each repository has its own list of issues. + +*Before opening a new issue or submitting a new pull request, it's helpful to +search the project - it's likely that another user has already reported the +issue you're facing, or it's a known issue that we're already aware of. + + +## How to Contribute +In general, please use conventional approaches to development and contribution such as: +* Create branches for additions or deletions, and or side projects +* Do not commit to master! +* Use Pull Requests (PRs) to indicate that an addition is ready to merge. +PRs are the main and exclusive way to contribute code to source{d} projects. +In order for a PR to be accepted it needs to pass this list of requirements: + +- The contribution must be correctly explained with natural language and providing a minimum working example that reproduces it. +- All PRs must be written idiomaticly: + - for Node: formatted according to [AirBnB standards](https://github.com/airbnb/javascript), and no warnings from `eslint` using the AirBnB style guide + - for other languages, similar constraints apply. +- They should in general include tests, and those shall pass. + - In any case, all the PRs have to pass the personal evaluation of at least one of the [maintainers](MAINTAINERS) of the project. + + +### Format of the commit message + +Every commit message should describe what was changed, under which context and, if applicable, the issue it relates to (mentioning a GitHub issue number when applicable): + +For small changes, or changes to a testing or personal branch, the commit message should be a short changelog entry + +For larger changes or for changes on branches that are more widely used, the commit message should simply reference an entry to some other changelog system. It is encouraged to use some sort of versioning system to log changes. Example commit messages: +``` +superscript.py v 2.0.5.006 +``` + +The format can be described more formally as follows: + +``` + v +``` From 4b26e4c531bbefb8ac368ac2a85bb1d17af9c05f Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Thu, 1 Apr 2021 13:10:50 -0500 Subject: [PATCH 003/143] Create LICENSE --- LICENSE | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b348082 --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, Titan Scouting +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 4872af581aa06170d7c67f6c76567deb7a2da672 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Thu, 1 Apr 2021 13:11:22 -0500 Subject: [PATCH 004/143] Create MAINTAINERS --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 MAINTAINERS diff --git a/MAINTAINERS b/MAINTAINERS new file mode 100644 index 0000000..60e7196 --- /dev/null +++ b/MAINTAINERS @@ -0,0 +1,3 @@ +Arthur Lu +Jacob Levine +Dev Singh From f5f0c0321814f80ef9659c57064a70733e9d06cc Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Thu, 1 Apr 2021 13:11:38 -0500 Subject: [PATCH 005/143] Create SECURITY.md --- SECURITY.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..7754343 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,6 @@ +# Security Policy + + +## Reporting a Vulnerability + +Please email `titanscout2022@gmail.com` to report a vulnerability. From 51b4943307dc4b4efb5d776022f56c55e4b1c182 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Thu, 1 Apr 2021 13:38:53 -0500 Subject: [PATCH 006/143] fix ut and file structure --- .../{ut-superscript.yml => superscript-unit.yml} | 8 ++------ {src => test}/test_superscript.py | 0 2 files changed, 2 insertions(+), 6 deletions(-) rename .github/workflows/{ut-superscript.yml => superscript-unit.yml} (76%) rename {src => test}/test_superscript.py (100%) diff --git a/.github/workflows/ut-superscript.yml b/.github/workflows/superscript-unit.yml similarity index 76% rename from .github/workflows/ut-superscript.yml rename to .github/workflows/superscript-unit.yml index fcd2b69..3a4e4c5 100644 --- a/.github/workflows/ut-superscript.yml +++ b/.github/workflows/superscript-unit.yml @@ -17,8 +17,6 @@ jobs: matrix: python-version: [3.7, 3.8] - env: - working-directory: ./data-analysis/ steps: - uses: actions/checkout@v2 @@ -30,9 +28,7 @@ jobs: run: | python -m pip install --upgrade pip pip install pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - working-directory: ${{ env.working-directory }} + if [ -f src/requirements.txt ]; then pip install -r src/requirements.txt; fi - name: Test with pytest run: | - pytest - working-directory: ${{ env.working-directory }} + pytest test/ diff --git a/src/test_superscript.py b/test/test_superscript.py similarity index 100% rename from src/test_superscript.py rename to test/test_superscript.py From 8ede63ed045527a876d2569e7df3c30303c6e29e Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 2 Apr 2021 01:28:25 +0000 Subject: [PATCH 007/143] fixed spelling in default config, added config to git ignore --- .gitignore | 3 ++- src/config.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6c766f6..dbe53f9 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ **/build/ **/*.egg-info/ -**/dist/ \ No newline at end of file +**/dist/ +**/config.json \ No newline at end of file diff --git a/src/config.json b/src/config.json index 0c58fe4..b94d399 100644 --- a/src/config.json +++ b/src/config.json @@ -46,7 +46,7 @@ ], "balls-started": [ "basic_stats", - "historical_analyss", + "historical_analysis", "regression_linear", "regression_logarithmic", "regression_exponential", From 70afd23f2c0819ddc0ef8ea1cd6a9994e8b66c6b Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 2 Apr 2021 21:35:05 +0000 Subject: [PATCH 008/143] deleted config.json changed superscript config lookup to relative path added additional requirements to requirements.txt added build spec file for superscript --- .devcontainer/Dockerfile | 9 ++- .devcontainer/dev-dockerfile | 2 + .devcontainer/devcontainer.json | 6 +- src/config.json | 101 ------------------------------- src/requirements.txt | 14 +++++ src/superscript.py | 102 ++++++++++++++++---------------- src/superscript.spec | 37 ++++++++++++ 7 files changed, 114 insertions(+), 157 deletions(-) create mode 100644 .devcontainer/dev-dockerfile delete mode 100644 src/config.json create mode 100644 src/superscript.spec diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 364066e..160a9b9 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,2 +1,7 @@ -FROM python -WORKDIR ~/ +FROM ubuntu:20.04 +WORKDIR / +RUN apt-get -y update +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata +RUN apt-get install -y python3 python3-dev git python3-pip python3-kivy python-is-python3 libgl1-mesa-dev build-essential +RUN ln -s $(which pip3) /usr/bin/pip +RUN pip install pymongo pandas numpy scipy scikit-learn matplotlib pylint kivy \ No newline at end of file diff --git a/.devcontainer/dev-dockerfile b/.devcontainer/dev-dockerfile new file mode 100644 index 0000000..74659d4 --- /dev/null +++ b/.devcontainer/dev-dockerfile @@ -0,0 +1,2 @@ +FROM titanscout2022/tra-analysis-base:latest +WORKDIR / \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a14a6fb..824072f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "name": "TRA Analysis Development Environment", "build": { - "dockerfile": "Dockerfile", + "dockerfile": "dev-dockerfile", }, "settings": { "terminal.integrated.shell.linux": "/bin/bash", @@ -24,5 +24,5 @@ "ms-python.python", "waderyan.gitblame" ], - "postCreateCommand": "apt install vim -y ; pip install -r src/requirements.txt ; pip install pylint ; pip install tra-analysis; pip install pytest" -} \ No newline at end of file + "postCreateCommand": "/usr/bin/pip3 install -r /workspaces/tra-data-analysis/src/requirements.txt && /usr/bin/pip3 install --no-cache-dir pylint && /usr/bin/pip3 install pytest" + } \ No newline at end of file diff --git a/src/config.json b/src/config.json deleted file mode 100644 index b94d399..0000000 --- a/src/config.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "max-threads": 0.5, - "team": "", - "competition": "", - "key": { - "database": "", - "tba": "" - }, - "statistics": { - "match": { - "balls-blocked": [ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-collected": [ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-lower-teleop": [ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-lower-auto": [ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-started": [ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-upper-teleop": [ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-upper-auto": [ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ] - }, - "metric": { - "elo": { - "score": 1500, - "N": 400, - "K": 24 - }, - "gl2": { - "score": 1500, - "rd": 250, - "vol": 0.06 - }, - "ts": { - "mu": 25, - "sigma": 8.33 - } - }, - "pit": { - "wheel-mechanism": true, - "low-balls": true, - "high-balls": true, - "wheel-success": true, - "strategic-focus": true, - "climb-mechanism": true, - "attitude": true - } - } -} \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt index f95ca14..a8f7063 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -2,4 +2,18 @@ requests pymongo pandas tra-analysis + +dnspython +pyinstaller +requests +pymongo + +numpy +scipy +scikit-learn +six +matplotlib +pyparsing +pandas + kivy==2.0.0rc2 \ No newline at end of file diff --git a/src/superscript.py b/src/superscript.py index 4c147de..1f4b511 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -143,67 +143,67 @@ def main(): warnings.filterwarnings("ignore") -# while (True): + while (True): - current_time = time.time() - print("[OK] time: " + str(current_time)) + current_time = time.time() + print("[OK] time: " + str(current_time)) - config = load_config("red-alliance-analysis\data-analysis\config.json") - competition = config["competition"] - match_tests = config["statistics"]["match"] - pit_tests = config["statistics"]["pit"] - metrics_tests = config["statistics"]["metric"] - print("[OK] configs loaded") + config = load_config("config.json") + competition = config["competition"] + match_tests = config["statistics"]["match"] + pit_tests = config["statistics"]["pit"] + metrics_tests = config["statistics"]["metric"] + print("[OK] configs loaded") - print("[OK] starting threads") - cfg_max_threads = config["max-threads"] - sys_max_threads = os.cpu_count() - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - print("[Err] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads)) - exit() - # exec_threads = Pool(processes = alloc_processes) - # print("[OK] " + str(alloc_processes) + " threads started") + print("[OK] starting threads") + cfg_max_threads = config["max-threads"] + sys_max_threads = os.cpu_count() + if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : + alloc_processes = sys_max_threads + cfg_max_threads + elif cfg_max_threads > 0 and cfg_max_threads < 1: + alloc_processes = math.floor(cfg_max_threads * sys_max_threads) + elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: + alloc_processes = cfg_max_threads + elif cfg_max_threads == 0: + alloc_processes = sys_max_threads + else: + print("[Err] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads)) + exit() + exec_threads = Pool(processes = alloc_processes) + print("[OK] " + str(alloc_processes) + " threads started") - apikey = config["key"]["database"] - tbakey = config["key"]["tba"] - print("[OK] loaded keys") + apikey = config["key"]["database"] + tbakey = config["key"]["tba"] + print("[OK] loaded keys") - previous_time = get_previous_time(apikey) - print("[OK] analysis backtimed to: " + str(previous_time)) + previous_time = get_previous_time(apikey) + print("[OK] analysis backtimed to: " + str(previous_time)) - print("[OK] loading data") - start = time.time() - match_data = load_match(apikey, competition) - pit_data = load_pit(apikey, competition) - print("[OK] loaded data in " + str(time.time() - start) + " seconds") + print("[OK] loading data") + start = time.time() + match_data = load_match(apikey, competition) + pit_data = load_pit(apikey, competition) + print("[OK] loaded data in " + str(time.time() - start) + " seconds") - print("[OK] running match stats") - start = time.time() - matchloop(apikey, competition, match_data, match_tests) - print("[OK] finished match stats in " + str(time.time() - start) + " seconds") + print("[OK] running match stats") + start = time.time() + matchloop(apikey, competition, match_data, match_tests) + print("[OK] finished match stats in " + str(time.time() - start) + " seconds") - print("[OK] running team metrics") - start = time.time() - metricloop(tbakey, apikey, competition, previous_time, metrics_tests) - print("[OK] finished team metrics in " + str(time.time() - start) + " seconds") + print("[OK] running team metrics") + start = time.time() + metricloop(tbakey, apikey, competition, previous_time, metrics_tests) + print("[OK] finished team metrics in " + str(time.time() - start) + " seconds") - print("[OK] running pit analysis") - start = time.time() - pitloop(apikey, competition, pit_data, pit_tests) - print("[OK] finished pit analysis in " + str(time.time() - start) + " seconds") - - set_current_time(apikey, current_time) - print("[OK] finished all tests, looping") + print("[OK] running pit analysis") + start = time.time() + pitloop(apikey, competition, pit_data, pit_tests) + print("[OK] finished pit analysis in " + str(time.time() - start) + " seconds") + + set_current_time(apikey, current_time) + print("[OK] finished all tests, looping") - # clear() + # clear() def clear(): diff --git a/src/superscript.spec b/src/superscript.spec new file mode 100644 index 0000000..b9f9ca7 --- /dev/null +++ b/src/superscript.spec @@ -0,0 +1,37 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + + +a = Analysis(['superscript.py'], + pathex=['/workspaces/tra-data-analysis/src'], + binaries=[], + datas=[], + hiddenimports=[ + "dnspython", + "sklearn.utils._weight_vector", + "requests", + ], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='superscript', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True ) From e6e0351288a5cae41c118c914bc675e3412d33cc Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 2 Apr 2021 21:58:35 +0000 Subject: [PATCH 009/143] fixed .gitignore added build-CLI script fixed threading in superscript Former-commit-id: 1fa36e8694acfc485f727573d40382dd147b6470 --- .gitignore | 2 -- build/build-CLI.sh | 5 +++++ dist/superscript.REMOVED.git-id | 1 + src/superscript.py | 8 ++++---- 4 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 build/build-CLI.sh create mode 100644 dist/superscript.REMOVED.git-id diff --git a/.gitignore b/.gitignore index dbe53f9..325f585 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,5 @@ **/.pytest_cache/ **/*.pyc -**/build/ **/*.egg-info/ -**/dist/ **/config.json \ No newline at end of file diff --git a/build/build-CLI.sh b/build/build-CLI.sh new file mode 100644 index 0000000..8d8a0b8 --- /dev/null +++ b/build/build-CLI.sh @@ -0,0 +1,5 @@ +pathtospec="../src/superscript.spec" +pathtodist="../dist/" +pathtowork="/temp/" + +pyinstaller --onefile --clean --distpath ${pathtodist} --workpath ${pathtowork} ${pathtospec} \ No newline at end of file diff --git a/dist/superscript.REMOVED.git-id b/dist/superscript.REMOVED.git-id new file mode 100644 index 0000000..a7adaca --- /dev/null +++ b/dist/superscript.REMOVED.git-id @@ -0,0 +1 @@ +7fc1ccdee362692f0cfaf8a6ffe282ab8e47adca \ No newline at end of file diff --git a/src/superscript.py b/src/superscript.py index 1f4b511..4c80206 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -135,11 +135,11 @@ from concurrent.futures import ThreadPoolExecutor import time import warnings -# global exec_threads +global exec_threads def main(): - # global exec_threads + global exec_threads warnings.filterwarnings("ignore") @@ -203,7 +203,7 @@ def main(): set_current_time(apikey, current_time) print("[OK] finished all tests, looping") - # clear() + #clear() def clear(): @@ -314,7 +314,7 @@ def matchloop(apikey, competition, data, tests): # expects 3D array with [Team][ variable_data.append((data[team][variable], test)) test_filtered.append(test) - result_filtered = map(simplestats, variable_data) + result_filtered = exec_threads.map(simplestats, variable_data) i = 0 result_filtered = list(result_filtered) From 5a454e0e3981d4fecd87c826a6526bc5ed842722 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 2 Apr 2021 22:04:06 +0000 Subject: [PATCH 010/143] built and verified threading fixes Former-commit-id: 06d0df00ee7eedbd6fcef2bb2bbd29307f0a05fc --- dist/superscript.REMOVED.git-id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/superscript.REMOVED.git-id b/dist/superscript.REMOVED.git-id index a7adaca..3612e9c 100644 --- a/dist/superscript.REMOVED.git-id +++ b/dist/superscript.REMOVED.git-id @@ -1 +1 @@ -7fc1ccdee362692f0cfaf8a6ffe282ab8e47adca \ No newline at end of file +444a669a78cfcfa4224d73c7b84aa0a2b379024f \ No newline at end of file From 8f79be68d462d0070636c2e67c9c061b1244f6c9 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 3 Apr 2021 20:47:45 +0000 Subject: [PATCH 011/143] superscript v 0.8.3 Former-commit-id: 7d85a18b4b4dd07b1627340cd0a8cc19860f71cc --- .gitignore | 3 ++- src/superscript.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 325f585..c1d3443 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ **/*.pyc **/*.egg-info/ -**/config.json \ No newline at end of file +**/config.json +**/tra_analysis/ \ No newline at end of file diff --git a/src/superscript.py b/src/superscript.py index 4c80206..fbc8b99 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -3,10 +3,12 @@ # Notes: # setup: -__version__ = "0.8.2" +__version__ = "0.8.3" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 0.8.3: + - updated matchloop with new regression format (requires tra_analysis 3.x) 0.8.2: - readded while true to main function - added more thread config options @@ -284,6 +286,8 @@ def matchloop(apikey, competition, data, tests): # expects 3D array with [Team][ global exec_threads + short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} + class AutoVivification(dict): def __getitem__(self, item): try: @@ -321,7 +325,13 @@ def matchloop(apikey, competition, data, tests): # expects 3D array with [Team][ for result in result_filtered: - return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result + filtered = test_filtered[i] + + try: + short = short_mapping[filtered] + return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result[short] + except KeyError: # not in mapping + return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result i += 1 push_match(apikey, competition, return_vector) From 7abfb2d90aa61862410536cd24a3f3ca79f39a4b Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 9 Apr 2021 23:45:16 +0000 Subject: [PATCH 012/143] superscript v 0.8.4 Former-commit-id: 5ea22970179ba6fee0222871ad73d63ef46025f7 --- dist/superscript.REMOVED.git-id | 2 +- src/superscript.py | 92 +++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/dist/superscript.REMOVED.git-id b/dist/superscript.REMOVED.git-id index 3612e9c..819a8ba 100644 --- a/dist/superscript.REMOVED.git-id +++ b/dist/superscript.REMOVED.git-id @@ -1 +1 @@ -444a669a78cfcfa4224d73c7b84aa0a2b379024f \ No newline at end of file +e5f402045a28f8602e17dbd7e4ca6641c33ccd65 \ No newline at end of file diff --git a/src/superscript.py b/src/superscript.py index fbc8b99..6adaf6f 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -3,10 +3,14 @@ # Notes: # setup: -__version__ = "0.8.3" +__version__ = "0.8.4" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 0.84: + - added better error message for missing config.json + - added automatic config.json creation + - added splash text with version and system info 0.8.3: - updated matchloop with new regression format (requires tra_analysis 3.x) 0.8.2: @@ -132,6 +136,7 @@ import os from os import system, name from pathlib import Path from multiprocessing import Pool +import platform import matplotlib.pyplot as plt from concurrent.futures import ThreadPoolExecutor import time @@ -145,6 +150,8 @@ def main(): warnings.filterwarnings("ignore") + splash() + while (True): current_time = time.time() @@ -169,7 +176,7 @@ def main(): elif cfg_max_threads == 0: alloc_processes = sys_max_threads else: - print("[Err] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads)) + print("[ERROR] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads)) exit() exec_threads = Pool(processes = alloc_processes) print("[OK] " + str(alloc_processes) + " threads started") @@ -205,6 +212,8 @@ def main(): set_current_time(apikey, current_time) print("[OK] finished all tests, looping") + print_hrule() + #clear() def clear(): @@ -217,11 +226,39 @@ def clear(): else: _ = system('clear') +def print_hrule(): + + print("#"+38*"-"+"#") + +def print_box(s): + + temp = "|" + temp += s + temp += (40-len(s)-2)*" " + temp += "|" + print(temp) + +def splash(): + + print_hrule() + print_box(" superscript version: " + __version__) + print_box(" os: " + platform.system()) + print_box(" python: " + platform.python_version()) + print_hrule() + def load_config(file): config_vector = {} - with open(file) as f: - config_vector = json.load(f) + + try: + f = open(file) + except: + print("[ERROR] could not locate config.json, generating blank config.json and exiting") + f = open(file, "w") + f.write(sample_json) + exit() + + config_vector = json.load(f) return config_vector @@ -543,4 +580,51 @@ def graph_pit_histogram(apikey, competition, figsize=(80,15)): plt.show() +sample_json = """{ + "max-threads": 0.5, + "team": "", + "competition": "2020ilch", + "key":{ + "database":"", + "tba":"" + }, + "statistics":{ + "match":{ + "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] + + }, + "metric":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + }, + "pit":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true + } + } +}""" + main() \ No newline at end of file From 014570930a00220fab406f71d449083eb5acfd37 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 10 Apr 2021 06:08:18 +0000 Subject: [PATCH 013/143] superscript v 0.8.5 Former-commit-id: 4de011ef45c6c0785784582d31ec3718116a3493 --- .gitignore | 4 +- dist/superscript.REMOVED.git-id | 2 +- src/superscript.py | 119 ++++++++++++++++++-------------- 3 files changed, 72 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index c1d3443..edba599 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ **/*.egg-info/ **/config.json -**/tra_analysis/ \ No newline at end of file +**/tra_analysis/ + +**/errorlog.txt \ No newline at end of file diff --git a/dist/superscript.REMOVED.git-id b/dist/superscript.REMOVED.git-id index 819a8ba..f30f495 100644 --- a/dist/superscript.REMOVED.git-id +++ b/dist/superscript.REMOVED.git-id @@ -1 +1 @@ -e5f402045a28f8602e17dbd7e4ca6641c33ccd65 \ No newline at end of file +d7d6e7a11e9acaf8966e4a25edcacad84793282e \ No newline at end of file diff --git a/src/superscript.py b/src/superscript.py index 6adaf6f..d93f7e0 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -3,11 +3,14 @@ # Notes: # setup: -__version__ = "0.8.4" +__version__ = "0.8.5" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: - 0.84: + 0.8.5: + - added more gradeful KeyboardInterrupt exiting + - redirected stderr to errorlog.txt + 0.8.4: - added better error message for missing config.json - added automatic config.json creation - added splash text with version and system info @@ -138,6 +141,7 @@ from pathlib import Path from multiprocessing import Pool import platform import matplotlib.pyplot as plt +import sys from concurrent.futures import ThreadPoolExecutor import time import warnings @@ -148,71 +152,84 @@ def main(): global exec_threads + sys.stderr = open("errorlog.txt", "w") + warnings.filterwarnings("ignore") splash() while (True): - current_time = time.time() - print("[OK] time: " + str(current_time)) + try: - config = load_config("config.json") - competition = config["competition"] - match_tests = config["statistics"]["match"] - pit_tests = config["statistics"]["pit"] - metrics_tests = config["statistics"]["metric"] - print("[OK] configs loaded") + current_time = time.time() + print("[OK] time: " + str(current_time)) - print("[OK] starting threads") - cfg_max_threads = config["max-threads"] - sys_max_threads = os.cpu_count() - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - print("[ERROR] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads)) - exit() - exec_threads = Pool(processes = alloc_processes) - print("[OK] " + str(alloc_processes) + " threads started") + config = load_config("config.json") + competition = config["competition"] + match_tests = config["statistics"]["match"] + pit_tests = config["statistics"]["pit"] + metrics_tests = config["statistics"]["metric"] + print("[OK] configs loaded") - apikey = config["key"]["database"] - tbakey = config["key"]["tba"] - print("[OK] loaded keys") + print("[OK] starting threads") + cfg_max_threads = config["max-threads"] + sys_max_threads = os.cpu_count() + if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : + alloc_processes = sys_max_threads + cfg_max_threads + elif cfg_max_threads > 0 and cfg_max_threads < 1: + alloc_processes = math.floor(cfg_max_threads * sys_max_threads) + elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: + alloc_processes = cfg_max_threads + elif cfg_max_threads == 0: + alloc_processes = sys_max_threads + else: + print("[ERROR] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads)) + exit() + exec_threads = Pool(processes = alloc_processes) + print("[OK] " + str(alloc_processes) + " threads started") - previous_time = get_previous_time(apikey) - print("[OK] analysis backtimed to: " + str(previous_time)) + apikey = config["key"]["database"] + tbakey = config["key"]["tba"] + print("[OK] loaded keys") - print("[OK] loading data") - start = time.time() - match_data = load_match(apikey, competition) - pit_data = load_pit(apikey, competition) - print("[OK] loaded data in " + str(time.time() - start) + " seconds") + previous_time = get_previous_time(apikey) + print("[OK] analysis backtimed to: " + str(previous_time)) - print("[OK] running match stats") - start = time.time() - matchloop(apikey, competition, match_data, match_tests) - print("[OK] finished match stats in " + str(time.time() - start) + " seconds") + print("[OK] loading data") + start = time.time() + match_data = load_match(apikey, competition) + pit_data = load_pit(apikey, competition) + print("[OK] loaded data in " + str(time.time() - start) + " seconds") - print("[OK] running team metrics") - start = time.time() - metricloop(tbakey, apikey, competition, previous_time, metrics_tests) - print("[OK] finished team metrics in " + str(time.time() - start) + " seconds") + print("[OK] running match stats") + start = time.time() + matchloop(apikey, competition, match_data, match_tests) + print("[OK] finished match stats in " + str(time.time() - start) + " seconds") - print("[OK] running pit analysis") - start = time.time() - pitloop(apikey, competition, pit_data, pit_tests) - print("[OK] finished pit analysis in " + str(time.time() - start) + " seconds") + print("[OK] running team metrics") + start = time.time() + metricloop(tbakey, apikey, competition, previous_time, metrics_tests) + print("[OK] finished team metrics in " + str(time.time() - start) + " seconds") + + print("[OK] running pit analysis") + start = time.time() + pitloop(apikey, competition, pit_data, pit_tests) + print("[OK] finished pit analysis in " + str(time.time() - start) + " seconds") + + set_current_time(apikey, current_time) + print("[OK] finished all tests, looping") + + print_hrule() - set_current_time(apikey, current_time) - print("[OK] finished all tests, looping") + except KeyboardInterrupt: + print("\n[OK] caught KeyboardInterrupt, killing processes") + exec_threads.terminate() + print("[OK] processes killed, exiting") + exit() - print_hrule() + else: + pass #clear() From 75dde2171c2b9513731c80694c05201e675f3378 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 12 Apr 2021 06:27:50 +0000 Subject: [PATCH 014/143] quick patch for devcontainer.json Former-commit-id: 1252865ced2d95627fef5574efcde1f70d220360 --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 824072f..31f7e98 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,5 +24,5 @@ "ms-python.python", "waderyan.gitblame" ], - "postCreateCommand": "/usr/bin/pip3 install -r /workspaces/tra-data-analysis/src/requirements.txt && /usr/bin/pip3 install --no-cache-dir pylint && /usr/bin/pip3 install pytest" + "postCreateCommand": "/usr/bin/pip3 install -r /workspaces/tra-superscript/src/requirements.txt && /usr/bin/pip3 install --no-cache-dir pylint && /usr/bin/pip3 install pytest" } \ No newline at end of file From 08292d5dc8af197fe810695907f847f39ecdea03 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 12 Apr 2021 06:30:21 +0000 Subject: [PATCH 015/143] better fix for devcontainer.json Former-commit-id: eb5d89ea354f0f87289296d798a5b77f1b477182 --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 31f7e98..c380475 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,5 +24,5 @@ "ms-python.python", "waderyan.gitblame" ], - "postCreateCommand": "/usr/bin/pip3 install -r /workspaces/tra-superscript/src/requirements.txt && /usr/bin/pip3 install --no-cache-dir pylint && /usr/bin/pip3 install pytest" + "postCreateCommand": "/usr/bin/pip3 install -r ${containerWorkspaceFolder}/src/requirements.txt && /usr/bin/pip3 install --no-cache-dir pylint && /usr/bin/pip3 install pytest" } \ No newline at end of file From 2619f9a729d8623b99cb6cc874d45ece208c199d Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 12 Apr 2021 14:39:00 -0700 Subject: [PATCH 016/143] created batch script for windows compilation Former-commit-id: 241f0a62edb44bb91075809b9dab6355776f050c --- build/build-CLI.bat | 5 +++++ dist/superscript.exe.REMOVED.git-id | 1 + 2 files changed, 6 insertions(+) create mode 100644 build/build-CLI.bat create mode 100644 dist/superscript.exe.REMOVED.git-id diff --git a/build/build-CLI.bat b/build/build-CLI.bat new file mode 100644 index 0000000..77790bb --- /dev/null +++ b/build/build-CLI.bat @@ -0,0 +1,5 @@ +set pathtospec="../src/superscript.spec" +set pathtodist="../dist/" +set pathtowork="/temp/" + +pyinstaller --onefile --clean --distpath %pathtodist% --workpath %pathtowork% %pathtospec% \ No newline at end of file diff --git a/dist/superscript.exe.REMOVED.git-id b/dist/superscript.exe.REMOVED.git-id new file mode 100644 index 0000000..bc93620 --- /dev/null +++ b/dist/superscript.exe.REMOVED.git-id @@ -0,0 +1 @@ +e3e745f422bf373316e2aa3f8e2d0722317a57bf \ No newline at end of file From 651ae0c2db21cd8c9ab5d462a51bff6b76e6c86d Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 12 Apr 2021 15:13:54 -0700 Subject: [PATCH 017/143] removed matplotlib import removed plotting pit analysis fixed warning supression for win exe superscript v 0.8.6 Former-commit-id: 1b01ede16161e48348219d7d664dcb5737df8bb5 --- dist/superscript.exe.REMOVED.git-id | 2 +- src/requirements.txt | 1 - src/superscript.py | 33 ++++++----------------------- src/superscript.spec | 2 +- 4 files changed, 9 insertions(+), 29 deletions(-) diff --git a/dist/superscript.exe.REMOVED.git-id b/dist/superscript.exe.REMOVED.git-id index bc93620..6d975ce 100644 --- a/dist/superscript.exe.REMOVED.git-id +++ b/dist/superscript.exe.REMOVED.git-id @@ -1 +1 @@ -e3e745f422bf373316e2aa3f8e2d0722317a57bf \ No newline at end of file +ab62effa6bb1776cc136a7c60afc666d75acefd5 \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt index a8f7063..a3d30e7 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -12,7 +12,6 @@ numpy scipy scikit-learn six -matplotlib pyparsing pandas diff --git a/src/superscript.py b/src/superscript.py index d93f7e0..b408369 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -3,10 +3,12 @@ # Notes: # setup: -__version__ = "0.8.5" +__version__ = "0.8.6" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 0.8.6: + - added proper main function 0.8.5: - added more gradeful KeyboardInterrupt exiting - redirected stderr to errorlog.txt @@ -140,7 +142,6 @@ from os import system, name from pathlib import Path from multiprocessing import Pool import platform -import matplotlib.pyplot as plt import sys from concurrent.futures import ThreadPoolExecutor import time @@ -574,29 +575,6 @@ def get_team_metrics(apikey, tbakey, competition): return {"elo-ranks": elo_ranked, "glicko2-ranks": gl2_ranked} -def graph_pit_histogram(apikey, competition, figsize=(80,15)): - - pit = d.get_pit_variable_formatted(apikey, competition) - - fig, ax = plt.subplots(1, len(pit), sharey=True, figsize=figsize) - - i = 0 - - for variable in pit: - - ax[i].hist(pit[variable]) - ax[i].invert_xaxis() - - ax[i].set_xlabel('') - ax[i].set_ylabel('Frequency') - ax[i].set_title(variable) - - plt.yticks(np.arange(len(pit[variable]))) - - i+=1 - - plt.show() - sample_json = """{ "max-threads": 0.5, "team": "", @@ -644,4 +622,7 @@ sample_json = """{ } }""" -main() \ No newline at end of file +if __name__ == "__main__": + if sys.platform.startswith('win'): + multiprocessing.freeze_support() + main() \ No newline at end of file diff --git a/src/superscript.spec b/src/superscript.spec index b9f9ca7..5ffc4bf 100644 --- a/src/superscript.spec +++ b/src/superscript.spec @@ -26,7 +26,7 @@ exe = EXE(pyz, a.binaries, a.zipfiles, a.datas, - [], + [('W ignore', None, 'OPTION')], name='superscript', debug=False, bootloader_ignore_signals=False, From 78f737d45f601127a422df583b830bb9ed160520 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 13 Apr 2021 04:03:07 +0000 Subject: [PATCH 018/143] removed compiled binaries added compiled binaries in /dist/ to gitignore Former-commit-id: 11d3b49d8277747795108951bd3f17683fef1bbe --- .gitignore | 3 ++- dist/superscript.REMOVED.git-id | 1 - dist/superscript.exe.REMOVED.git-id | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 dist/superscript.REMOVED.git-id delete mode 100644 dist/superscript.exe.REMOVED.git-id diff --git a/.gitignore b/.gitignore index edba599..616629e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ **/config.json **/tra_analysis/ -**/errorlog.txt \ No newline at end of file +**/errorlog.txt +/dist/superscript.* \ No newline at end of file diff --git a/dist/superscript.REMOVED.git-id b/dist/superscript.REMOVED.git-id deleted file mode 100644 index f30f495..0000000 --- a/dist/superscript.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -d7d6e7a11e9acaf8966e4a25edcacad84793282e \ No newline at end of file diff --git a/dist/superscript.exe.REMOVED.git-id b/dist/superscript.exe.REMOVED.git-id deleted file mode 100644 index 6d975ce..0000000 --- a/dist/superscript.exe.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -ab62effa6bb1776cc136a7c60afc666d75acefd5 \ No newline at end of file From a367e7254b9e8cfaeea53a86475610512dfb58d5 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 13 Apr 2021 04:05:46 +0000 Subject: [PATCH 019/143] added compiled binaries with no file endings to gitignore Former-commit-id: 9fb5ba7cc494156b361b7eabe8ea51501527a1c4 --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 616629e..6b1a37d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ **/tra_analysis/ **/errorlog.txt -/dist/superscript.* \ No newline at end of file +/dist/superscript.* +/dist/superscript \ No newline at end of file From d16ef53457d14a44b3937e4cd496c9fa82f4e66e Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 15 Apr 2021 19:41:10 +0000 Subject: [PATCH 020/143] added .gitattributes Former-commit-id: 64cb232565fc0cfd290a77c902619960b9fc08f1 --- .gitattributes | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..262f911 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Auto detect text files and perform LF normalization +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf \ No newline at end of file From 727398d32ffdd134ad7c2d7dfc4f30f22c598bcd Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sun, 25 Apr 2021 03:51:01 +0000 Subject: [PATCH 021/143] added sample build-cli workflow Former-commit-id: 36dad7b22d95a2b66ae00da609d3a9a406cceb6e --- .github/workflows/build-cli.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/build-cli.yml diff --git a/.github/workflows/build-cli.yml b/.github/workflows/build-cli.yml new file mode 100644 index 0000000..681ab31 --- /dev/null +++ b/.github/workflows/build-cli.yml @@ -0,0 +1,17 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Superscript Unit Tests + +on: + release: + types: [published, edited] + +jobs: + generate: + name: Build Linux + runs-on: ubuntu-latest + + steps: + - name: Checkout master + uses: actions/checkout@master From 6c385b5bd33b756d37b1dfe228597a7bf82e1b9e Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sun, 25 Apr 2021 06:05:33 +0000 Subject: [PATCH 022/143] removed ThreadPoolExecutor import Former-commit-id: 139410f7f0886025e68e09b8810470b22d41dd7b --- src/superscript.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/superscript.py b/src/superscript.py index b408369..e912951 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -143,7 +143,6 @@ from pathlib import Path from multiprocessing import Pool import platform import sys -from concurrent.futures import ThreadPoolExecutor import time import warnings From 40191aa6b57df168ee02794134b736f922a88742 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 27 Apr 2021 07:26:14 +0000 Subject: [PATCH 023/143] fixed pathing for build-CLI.* added temp directory to gitignore Former-commit-id: 73888cbc9ef9451dfa077812cdeace86393ec7db --- .gitignore | 1 + build/build-CLI.bat | 2 +- build/build-CLI.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6b1a37d..0e3db09 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ **/*.egg-info/ **/config.json **/tra_analysis/ +**/temp/* **/errorlog.txt /dist/superscript.* diff --git a/build/build-CLI.bat b/build/build-CLI.bat index 77790bb..522e5fc 100644 --- a/build/build-CLI.bat +++ b/build/build-CLI.bat @@ -1,5 +1,5 @@ set pathtospec="../src/superscript.spec" set pathtodist="../dist/" -set pathtowork="/temp/" +set pathtowork="temp/" pyinstaller --onefile --clean --distpath %pathtodist% --workpath %pathtowork% %pathtospec% \ No newline at end of file diff --git a/build/build-CLI.sh b/build/build-CLI.sh index 8d8a0b8..9837f2a 100644 --- a/build/build-CLI.sh +++ b/build/build-CLI.sh @@ -1,5 +1,5 @@ pathtospec="../src/superscript.spec" pathtodist="../dist/" -pathtowork="/temp/" +pathtowork="temp/" pyinstaller --onefile --clean --distpath ${pathtodist} --workpath ${pathtowork} ${pathtospec} \ No newline at end of file From 089ff7ec0171a52c5fb4bc244af4c4eb8c442f60 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 12 Jun 2021 07:09:26 +0000 Subject: [PATCH 024/143] separated gui and cli dev files began refactoring superscript in cli application Former-commit-id: ee309b9f8b6c5317422f1b2593bde027efb7337b --- build/build-CLI.bat | 2 +- build/build-CLI.sh | 2 +- src/cli/cli_interface.py | 44 +++++++ src/{ => cli}/data.py | 0 src/cli/superscript.py | 214 +++++++++++++++++++++++++++++++++ src/{ => cli}/superscript.spec | 0 src/gui/data.py | 129 ++++++++++++++++++++ src/{ => gui}/design.kv | 0 src/{ => gui}/main.py | 0 src/{ => gui}/superscript.py | 0 10 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 src/cli/cli_interface.py rename src/{ => cli}/data.py (100%) create mode 100644 src/cli/superscript.py rename src/{ => cli}/superscript.spec (100%) create mode 100644 src/gui/data.py rename src/{ => gui}/design.kv (100%) rename src/{ => gui}/main.py (100%) rename src/{ => gui}/superscript.py (100%) diff --git a/build/build-CLI.bat b/build/build-CLI.bat index 522e5fc..22dd93c 100644 --- a/build/build-CLI.bat +++ b/build/build-CLI.bat @@ -1,4 +1,4 @@ -set pathtospec="../src/superscript.spec" +set pathtospec="../src/cli/superscript.spec" set pathtodist="../dist/" set pathtowork="temp/" diff --git a/build/build-CLI.sh b/build/build-CLI.sh index 9837f2a..302265b 100644 --- a/build/build-CLI.sh +++ b/build/build-CLI.sh @@ -1,4 +1,4 @@ -pathtospec="../src/superscript.spec" +pathtospec="../src/cli/superscript.spec" pathtodist="../dist/" pathtowork="temp/" diff --git a/src/cli/cli_interface.py b/src/cli/cli_interface.py new file mode 100644 index 0000000..9fd5765 --- /dev/null +++ b/src/cli/cli_interface.py @@ -0,0 +1,44 @@ +import sys +import time +from os import system, name +import platform + +empty_delim = " " +hard_divided_delim = "|" +soft_divided_delim = ":" +l_brack = "[" +r_brack = "]" + +ERR = "[ERR]" +INF = "[INF]" + +stdout = sys.stdout +stderr = sys.stderr + +def log(target, level, message, code = 0): + + message = time.ctime() + empty_delim + str(level) + l_brack + str(code) + r_brack + empty_delim + soft_divided_delim + empty_delim + message + print(message, file = target) + +def clear(): + if name == "nt": + system("cls") + else: + system("clear") + +def splash(version): + + def hrule(): + print("#"+38*"-"+"#") + def box(s): + temp = "|" + temp += s + temp += (40-len(s)-2)*" " + temp += "|" + print(temp) + + hrule() + box(" superscript version: " + version) + box(" os: " + platform.system()) + box(" python: " + platform.python_version()) + hrule() \ No newline at end of file diff --git a/src/data.py b/src/cli/data.py similarity index 100% rename from src/data.py rename to src/cli/data.py diff --git a/src/cli/superscript.py b/src/cli/superscript.py new file mode 100644 index 0000000..dfd0151 --- /dev/null +++ b/src/cli/superscript.py @@ -0,0 +1,214 @@ +# Titan Robotics Team 2022: Superscript Script +# Written by Arthur Lu, Jacob Levine, and Dev Singh +# Notes: +# setup: + +__version__ = "0.8.6" + +# changelog should be viewed using print(analysis.__changelog__) +__changelog__ = """changelog: + 0.8.6: + - added proper main function + 0.8.5: + - added more gradeful KeyboardInterrupt exiting + - redirected stderr to errorlog.txt + 0.8.4: + - added better error message for missing config.json + - added automatic config.json creation + - added splash text with version and system info + 0.8.3: + - updated matchloop with new regression format (requires tra_analysis 3.x) + 0.8.2: + - readded while true to main function + - added more thread config options + 0.8.1: + - optimized matchloop further by bypassing GIL + 0.8.0: + - added multithreading to matchloop + - tweaked user log + 0.7.0: + - finished implementing main function + 0.6.2: + - integrated get_team_rankings.py as get_team_metrics() function + - integrated visualize_pit.py as graph_pit_histogram() function + 0.6.1: + - bug fixes with analysis.Metric() calls + - modified metric functions to use config.json defined default values + 0.6.0: + - removed main function + - changed load_config function + - added save_config function + - added load_match function + - renamed simpleloop to matchloop + - moved simplestats function inside matchloop + - renamed load_metrics to load_metric + - renamed metricsloop to metricloop + - split push to database functions amon push_match, push_metric, push_pit + - moved + 0.5.2: + - made changes due to refactoring of analysis + 0.5.1: + - text fixes + - removed matplotlib requirement + 0.5.0: + - improved user interface + 0.4.2: + - removed unessasary code + 0.4.1: + - fixed bug where X range for regression was determined before sanitization + - better sanitized data + 0.4.0: + - fixed spelling issue in __changelog__ + - addressed nan bug in regression + - fixed errors on line 335 with metrics calling incorrect key "glicko2" + - fixed errors in metrics computing + 0.3.0: + - added analysis to pit data + 0.2.1: + - minor stability patches + - implemented db syncing for timestamps + - fixed bugs + 0.2.0: + - finalized testing and small fixes + 0.1.4: + - finished metrics implement, trueskill is bugged + 0.1.3: + - working + 0.1.2: + - started implement of metrics + 0.1.1: + - cleaned up imports + 0.1.0: + - tested working, can push to database + 0.0.9: + - tested working + - prints out stats for the time being, will push to database later + 0.0.8: + - added data import + - removed tba import + - finished main method + 0.0.7: + - added load_config + - optimized simpleloop for readibility + - added __all__ entries + - added simplestats engine + - pending testing + 0.0.6: + - fixes + 0.0.5: + - imported pickle + - created custom database object + 0.0.4: + - fixed simpleloop to actually return a vector + 0.0.3: + - added metricsloop which is unfinished + 0.0.2: + - added simpleloop which is untested until data is provided + 0.0.1: + - created script + - added analysis, numba, numpy imports +""" + +__author__ = ( + "Arthur Lu ", + "Jacob Levine ", +) + +__all__ = [ + "load_config", + "save_config", + "get_previous_time", + "load_match", + "matchloop", + "load_metric", + "metricloop", + "load_pit", + "pitloop", + "push_match", + "push_metric", + "push_pit", +] + +# imports: + +import json + +from cli_interface import splash, log, ERR, INF, stdout, stderr + +config_path = "config.json" +sample_json = """{ + "max-threads": 0.5, + "team": "", + "competition": "2020ilch", + "key":{ + "database":"", + "tba":"" + }, + "statistics":{ + "match":{ + "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] + + }, + "metric":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + }, + "pit":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true + } + } +}""" + +def main(): + + splash(__version__) + +def load_config(config_vector): + try: + f = open(path, "r") + except: + log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting") + f = open(path, "w") + f.write(sample_json) + f.close() + return 1 + + config_vector = json.load(f) + f.close() + return 0 + +def save_config(path, config_vector): + try: + f = open(path) + json.dump(config_vector) + f.close() + return 0 + except: + return 1 + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/superscript.spec b/src/cli/superscript.spec similarity index 100% rename from src/superscript.spec rename to src/cli/superscript.spec diff --git a/src/gui/data.py b/src/gui/data.py new file mode 100644 index 0000000..641aba7 --- /dev/null +++ b/src/gui/data.py @@ -0,0 +1,129 @@ +import requests +import pymongo +import pandas as pd +import time + +def pull_new_tba_matches(apikey, competition, cutoff): + api_key= apikey + x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth_Key":api_key}) + out = [] + for i in x.json(): + if i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm": + out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) + return out + +def get_team_match_data(apikey, competition, team_num): + client = pymongo.MongoClient(apikey) + db = client.data_scouting + mdata = db.matchdata + out = {} + for i in mdata.find({"competition" : competition, "team_scouted": team_num}): + out[i['match']] = i['data'] + return pd.DataFrame(out) + +def get_team_pit_data(apikey, competition, team_num): + client = pymongo.MongoClient(apikey) + db = client.data_scouting + mdata = db.pitdata + out = {} + return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"] + +def get_team_metrics_data(apikey, competition, team_num): + client = pymongo.MongoClient(apikey) + db = client.data_processing + mdata = db.team_metrics + return mdata.find_one({"competition" : competition, "team": team_num}) + +def get_match_data_formatted(apikey, competition): + client = pymongo.MongoClient(apikey) + db = client.data_scouting + mdata = db.teamlist + x=mdata.find_one({"competition":competition}) + out = {} + for i in x: + try: + out[int(i)] = unkeyify_2l(get_team_match_data(apikey, competition, int(i)).transpose().to_dict()) + except: + pass + return out + +def get_metrics_data_formatted(apikey, competition): + client = pymongo.MongoClient(apikey) + db = client.data_scouting + mdata = db.teamlist + x=mdata.find_one({"competition":competition}) + out = {} + for i in x: + try: + out[int(i)] = d.get_team_metrics_data(apikey, competition, int(i)) + except: + pass + return out + +def get_pit_data_formatted(apikey, competition): + client = pymongo.MongoClient(apikey) + db = client.data_scouting + mdata = db.teamlist + x=mdata.find_one({"competition":competition}) + out = {} + for i in x: + try: + out[int(i)] = get_team_pit_data(apikey, competition, int(i)) + except: + pass + return out + +def get_pit_variable_data(apikey, competition): + client = pymongo.MongoClient(apikey) + db = client.data_processing + mdata = db.team_pit + out = {} + return mdata.find() + +def get_pit_variable_formatted(apikey, competition): + temp = get_pit_variable_data(apikey, competition) + out = {} + for i in temp: + out[i["variable"]] = i["data"] + return out + +def push_team_tests_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_tests"): + client = pymongo.MongoClient(apikey) + db = client[dbname] + mdata = db[colname] + mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "data" : data}, True) + +def push_team_metrics_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"): + client = pymongo.MongoClient(apikey) + db = client[dbname] + mdata = db[colname] + mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True) + +def push_team_pit_data(apikey, competition, variable, data, dbname = "data_processing", colname = "team_pit"): + client = pymongo.MongoClient(apikey) + db = client[dbname] + mdata = db[colname] + mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True) + +def get_analysis_flags(apikey, flag): + client = pymongo.MongoClient(apikey) + db = client.data_processing + mdata = db.flags + return mdata.find_one({flag:{"$exists":True}}) + +def set_analysis_flags(apikey, flag, data): + client = pymongo.MongoClient(apikey) + db = client.data_processing + mdata = db.flags + return mdata.replace_one({flag:{"$exists":True}}, data, True) + +def unkeyify_2l(layered_dict): + out = {} + for i in layered_dict.keys(): + add = [] + sortkey = [] + for j in layered_dict[i].keys(): + add.append([j,layered_dict[i][j]]) + add.sort(key = lambda x: x[0]) + out[i] = list(map(lambda x: x[1], add)) + return out \ No newline at end of file diff --git a/src/design.kv b/src/gui/design.kv similarity index 100% rename from src/design.kv rename to src/gui/design.kv diff --git a/src/main.py b/src/gui/main.py similarity index 100% rename from src/main.py rename to src/gui/main.py diff --git a/src/superscript.py b/src/gui/superscript.py similarity index 100% rename from src/superscript.py rename to src/gui/superscript.py From b3c26ce2cfc4cb04e776b4a58f06d94883d9dea0 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 12 Jun 2021 07:16:11 +0000 Subject: [PATCH 025/143] renamed cli_interface.py to interface.py Former-commit-id: b94c9125b37bb348a523951f6f6977522da1ab7c --- src/cli/{cli_interface.py => interface.py} | 0 src/cli/superscript.py | 8 ++++++-- 2 files changed, 6 insertions(+), 2 deletions(-) rename src/cli/{cli_interface.py => interface.py} (100%) diff --git a/src/cli/cli_interface.py b/src/cli/interface.py similarity index 100% rename from src/cli/cli_interface.py rename to src/cli/interface.py diff --git a/src/cli/superscript.py b/src/cli/superscript.py index dfd0151..09ef513 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -3,10 +3,14 @@ # Notes: # setup: -__version__ = "0.8.6" +__version__ = "0.9.0" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 0.9.0: + - moved printing and logging related functions to interface.py (changelog will stay in this file) + - changed function return files for load_config and save_config to standard C values (0 for success, 1 for error) + - added local variables for config location 0.8.6: - added proper main function 0.8.5: @@ -187,7 +191,7 @@ def main(): splash(__version__) -def load_config(config_vector): +def load_config(path, config_vector): try: f = open(path, "r") except: From 4c65e88903323afe2ef75e2eede861d7b57ad098 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 12 Jun 2021 23:57:16 +0000 Subject: [PATCH 026/143] finished refactor Former-commit-id: b2f2dfe2a4989da5c62c2895eae73e9000760901 --- src/cli/dataset.py | 74 +++++++++++++++ src/cli/interface.py | 4 +- src/cli/processing.py | 191 ++++++++++++++++++++++++++++++++++++++ src/cli/superscript.py | 204 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 455 insertions(+), 18 deletions(-) create mode 100644 src/cli/dataset.py create mode 100644 src/cli/processing.py diff --git a/src/cli/dataset.py b/src/cli/dataset.py new file mode 100644 index 0000000..5a69cc9 --- /dev/null +++ b/src/cli/dataset.py @@ -0,0 +1,74 @@ +import data as d + +def get_previous_time(apikey): + + previous_time = d.get_analysis_flags(apikey, "latest_update") + + if previous_time == None: + + d.set_analysis_flags(apikey, "latest_update", 0) + previous_time = 0 + + else: + + previous_time = previous_time["latest_update"] + + return previous_time + +def set_current_time(apikey, current_time): + + d.set_analysis_flags(apikey, "latest_update", {"latest_update":current_time}) + +def load_match(apikey, competition): + + return d.get_match_data_formatted(apikey, competition) + +def load_metric(apikey, competition, match, group_name, metrics): + + group = {} + + for team in match[group_name]: + + db_data = d.get_team_metrics_data(apikey, competition, team) + + if d.get_team_metrics_data(apikey, competition, team) == None: + + elo = {"score": metrics["elo"]["score"]} + gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} + ts = {"mu": metrics["ts"]["mu"], "sigm+a": metrics["ts"]["sigma"]} + + group[team] = {"elo": elo, "gl2": gl2, "ts": ts} + + else: + + metrics = db_data["metrics"] + + elo = metrics["elo"] + gl2 = metrics["gl2"] + ts = metrics["ts"] + + group[team] = {"elo": elo, "gl2": gl2, "ts": ts} + + return group + +def load_pit(apikey, competition): + + return d.get_pit_data_formatted(apikey, competition) + +def push_match(apikey, competition, results): + + for team in results: + + d.push_team_tests_data(apikey, competition, team, results[team]) + +def push_metric(apikey, competition, metric): + + for team in metric: + + d.push_team_metrics_data(apikey, competition, team, metric[team]) + +def push_pit(apikey, competition, pit): + + for variable in pit: + + d.push_team_pit_data(apikey, competition, variable, pit[variable]) \ No newline at end of file diff --git a/src/cli/interface.py b/src/cli/interface.py index 9fd5765..9ee821a 100644 --- a/src/cli/interface.py +++ b/src/cli/interface.py @@ -5,7 +5,7 @@ import platform empty_delim = " " hard_divided_delim = "|" -soft_divided_delim = ":" +soft_divided_delim = "|" l_brack = "[" r_brack = "]" @@ -17,7 +17,7 @@ stderr = sys.stderr def log(target, level, message, code = 0): - message = time.ctime() + empty_delim + str(level) + l_brack + str(code) + r_brack + empty_delim + soft_divided_delim + empty_delim + message + message = time.ctime() + empty_delim + str(level) + l_brack + f"{code:04}" + r_brack + empty_delim + soft_divided_delim + empty_delim + message print(message, file = target) def clear(): diff --git a/src/cli/processing.py b/src/cli/processing.py new file mode 100644 index 0000000..1f7e322 --- /dev/null +++ b/src/cli/processing.py @@ -0,0 +1,191 @@ +import numpy as np + +from tra_analysis import Analysis as an +from dataset import push_metric +from data import pull_new_tba_matches + +def simplestats(data_test): + + data = np.array(data_test[0]) + data = data[np.isfinite(data)] + ranges = list(range(len(data))) + + test = data_test[1] + + if test == "basic_stats": + return an.basic_stats(data) + + if test == "historical_analysis": + return an.histo_analysis([ranges, data]) + + if test == "regression_linear": + return an.regression(ranges, data, ['lin']) + + if test == "regression_logarithmic": + return an.regression(ranges, data, ['log']) + + if test == "regression_exponential": + return an.regression(ranges, data, ['exp']) + + if test == "regression_polynomial": + return an.regression(ranges, data, ['ply']) + + if test == "regression_sigmoidal": + return an.regression(ranges, data, ['sig']) + +def matchloop(apikey, competition, data, tests, exec_threads): + + short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} + + class AutoVivification(dict): + def __getitem__(self, item): + try: + return dict.__getitem__(self, item) + except KeyError: + value = self[item] = type(self)() + return value + + return_vector = {} + + team_filtered = [] + variable_filtered = [] + variable_data = [] + test_filtered = [] + result_filtered = [] + return_vector = AutoVivification() + + for team in data: + + for variable in data[team]: + + if variable in tests: + + for test in tests[variable]: + + team_filtered.append(team) + variable_filtered.append(variable) + variable_data.append((data[team][variable], test)) + test_filtered.append(test) + + result_filtered = exec_threads.map(simplestats, variable_data) + i = 0 + + result_filtered = list(result_filtered) + + for result in result_filtered: + + filtered = test_filtered[i] + + try: + short = short_mapping[filtered] + return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result[short] + except KeyError: # not in mapping + return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result + i += 1 + + return return_vector + +def metricloop(tbakey, apikey, competition, timestamp, metrics): # listener based metrics update + + elo_N = metrics["elo"]["N"] + elo_K = metrics["elo"]["K"] + + matches = pull_new_tba_matches(tbakey, competition, timestamp) + + red = {} + blu = {} + + for match in matches: + + red = load_metric(apikey, competition, match, "red", metrics) + blu = load_metric(apikey, competition, match, "blue", metrics) + + elo_red_total = 0 + elo_blu_total = 0 + + gl2_red_score_total = 0 + gl2_blu_score_total = 0 + + gl2_red_rd_total = 0 + gl2_blu_rd_total = 0 + + gl2_red_vol_total = 0 + gl2_blu_vol_total = 0 + + for team in red: + + elo_red_total += red[team]["elo"]["score"] + + gl2_red_score_total += red[team]["gl2"]["score"] + gl2_red_rd_total += red[team]["gl2"]["rd"] + gl2_red_vol_total += red[team]["gl2"]["vol"] + + for team in blu: + + elo_blu_total += blu[team]["elo"]["score"] + + gl2_blu_score_total += blu[team]["gl2"]["score"] + gl2_blu_rd_total += blu[team]["gl2"]["rd"] + gl2_blu_vol_total += blu[team]["gl2"]["vol"] + + red_elo = {"score": elo_red_total / len(red)} + blu_elo = {"score": elo_blu_total / len(blu)} + + red_gl2 = {"score": gl2_red_score_total / len(red), "rd": gl2_red_rd_total / len(red), "vol": gl2_red_vol_total / len(red)} + blu_gl2 = {"score": gl2_blu_score_total / len(blu), "rd": gl2_blu_rd_total / len(blu), "vol": gl2_blu_vol_total / len(blu)} + + + if match["winner"] == "red": + + observations = {"red": 1, "blu": 0} + + elif match["winner"] == "blue": + + observations = {"red": 0, "blu": 1} + + else: + + observations = {"red": 0.5, "blu": 0.5} + + red_elo_delta = an.Metric().elo(red_elo["score"], blu_elo["score"], observations["red"], elo_N, elo_K) - red_elo["score"] + blu_elo_delta = an.Metric().elo(blu_elo["score"], red_elo["score"], observations["blu"], elo_N, elo_K) - blu_elo["score"] + + new_red_gl2_score, new_red_gl2_rd, new_red_gl2_vol = an.Metric().glicko2(red_gl2["score"], red_gl2["rd"], red_gl2["vol"], [blu_gl2["score"]], [blu_gl2["rd"]], [observations["red"], observations["blu"]]) + new_blu_gl2_score, new_blu_gl2_rd, new_blu_gl2_vol = an.Metric().glicko2(blu_gl2["score"], blu_gl2["rd"], blu_gl2["vol"], [red_gl2["score"]], [red_gl2["rd"]], [observations["blu"], observations["red"]]) + + red_gl2_delta = {"score": new_red_gl2_score - red_gl2["score"], "rd": new_red_gl2_rd - red_gl2["rd"], "vol": new_red_gl2_vol - red_gl2["vol"]} + blu_gl2_delta = {"score": new_blu_gl2_score - blu_gl2["score"], "rd": new_blu_gl2_rd - blu_gl2["rd"], "vol": new_blu_gl2_vol - blu_gl2["vol"]} + + for team in red: + + red[team]["elo"]["score"] = red[team]["elo"]["score"] + red_elo_delta + + red[team]["gl2"]["score"] = red[team]["gl2"]["score"] + red_gl2_delta["score"] + red[team]["gl2"]["rd"] = red[team]["gl2"]["rd"] + red_gl2_delta["rd"] + red[team]["gl2"]["vol"] = red[team]["gl2"]["vol"] + red_gl2_delta["vol"] + + for team in blu: + + blu[team]["elo"]["score"] = blu[team]["elo"]["score"] + blu_elo_delta + + blu[team]["gl2"]["score"] = blu[team]["gl2"]["score"] + blu_gl2_delta["score"] + blu[team]["gl2"]["rd"] = blu[team]["gl2"]["rd"] + blu_gl2_delta["rd"] + blu[team]["gl2"]["vol"] = blu[team]["gl2"]["vol"] + blu_gl2_delta["vol"] + + temp_vector = {} + temp_vector.update(red) + temp_vector.update(blu) + + push_metric(apikey, competition, temp_vector) + +def pitloop(apikey, competition, pit, tests): + + return_vector = {} + for team in pit: + for variable in pit[team]: + if variable in tests: + if not variable in return_vector: + return_vector[variable] = [] + return_vector[variable].append(pit[team][variable]) + + return return_vector \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 09ef513..0cbe4a0 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -11,6 +11,8 @@ __changelog__ = """changelog: - moved printing and logging related functions to interface.py (changelog will stay in this file) - changed function return files for load_config and save_config to standard C values (0 for success, 1 for error) - added local variables for config location + - moved dataset getting and setting functions to dataset.py (changelog will stay in this file) + - moved matchloop, metricloop, pitloop and helper functions (simplestats) to processing.py 0.8.6: - added proper main function 0.8.5: @@ -121,23 +123,22 @@ __author__ = ( __all__ = [ "load_config", "save_config", - "get_previous_time", - "load_match", - "matchloop", - "load_metric", - "metricloop", - "load_pit", - "pitloop", - "push_match", - "push_metric", - "push_pit", ] # imports: import json +import multiprocessing +import os +import math +from multiprocessing import Pool +import time +import warnings +import sys -from cli_interface import splash, log, ERR, INF, stdout, stderr +from interface import splash, log, ERR, INF, stdout, stderr +from dataset import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit +from processing import matchloop, metricloop, pitloop config_path = "config.json" sample_json = """{ @@ -189,21 +190,190 @@ sample_json = """{ def main(): + warnings.filterwarnings("ignore") + sys.stderr = open("errorlog.txt", "w") + splash(__version__) + loop_exit_code = 0 + loop_stored_exception = None + + while True: + + try: + + loop_start = time.time() + + current_time = time.time() + log(stdout, INF, "current time: " + str(current_time)) + + config = {} + if load_config(config_path, config) == 1: + exit(1) + + error_flag = False + + try: + competition = config["competition"] + except: + log(stderr, ERR, "could not find competition field in config", code = 101) + error_flag = True + try: + match_tests = config["statistics"]["match"] + except: + log(stderr, ERR, "could not find match_tests field in config", code = 102) + error_flag = True + try: + metrics_tests = config["statistics"]["metric"] + except: + log(stderr, ERR, "could not find metrics_tests field in config", code = 103) + error_flag = True + try: + pit_tests = config["statistics"]["pit"] + except: + log(stderr, ERR, "could not find pit_tests field in config", code = 104) + error_flag = True + + if error_flag: + exit(1) + error_flag = False + + if competition == None or competition == "": + log(stderr, ERR, "competition field in config must not be empty", code = 105) + error_flag = True + if match_tests == None: + log(stderr, ERR, "match_tests field in config must not be empty", code = 106) + error_flag = True + if metrics_tests == None: + log(stderr, ERR, "metrics_tests field in config must not be empty", code = 107) + error_flag = True + if pit_tests == None: + log(stderr, ERR, "pit_tests field in config must not be empty", code = 108) + error_flag = True + + if error_flag: + exit(1) + + log(stdout, INF, "found and loaded competition, match_tests, metrics_tests, pit_tests from config") + + sys_max_threads = os.cpu_count() + try: + cfg_max_threads = config["max-threads"] + except: + log(stderr, ERR, "max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) + exit(1) + + if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : + alloc_processes = sys_max_threads + cfg_max_threads + elif cfg_max_threads > 0 and cfg_max_threads < 1: + alloc_processes = math.floor(cfg_max_threads * sys_max_threads) + elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: + alloc_processes = cfg_max_threads + elif cfg_max_threads == 0: + alloc_processes = sys_max_threads + else: + log(stderr, ERR, "max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, code = 110) + exit(1) + + log(stdout, INF, "found and loaded max-threads from config") + log(stdout, INF, "attempting to start " + str(alloc_processes) + " threads") + try: + exec_threads = Pool(processes = alloc_processes) + except Exception as e: + log(stderr, ERR, "unable to start threads", code = 200) + log(stderr, INF, e) + exit(1) + log(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") + + exit_flag = False + + try: + apikey = config["key"]["database"] + except: + log(stderr, ERR, "database key field in config must not be empty, please populate the database key", code = 111) + exit_flag = True + try: + tbakey = config["key"]["tba"] + except: + log(stderr, ERR, "tba key field in config must not be empty, please populate the tba key", code = 112) + exit_flag = True + + if exit_flag: + exit(1) + + log(stdout, INF, "found and loaded database and tba keys") + + previous_time = get_previous_time(apikey) + log(stdout, INF, "analysis backtimed to: " + str(previous_time)) + + start = time.time() + log(stdout, INF, "loading match data") + match_data = load_match(apikey, competition) + log(stdout, INF, "finished loading match data in " + str(time.time() - start) + " seconds") + + start = time.time() + log(stdout, INF, "performing analysis on match data") + results = matchloop(apikey, competition, match_data, match_tests, exec_threads) + log(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds") + + start = time.time() + log(stdout, INF, "uploading match results to database") + push_match(apikey, competition, results) + log(stdout, INF, "finished uploading match results in " + str(time.time() - start) + " seconds") + + start = time.time() + log(stdout, INF, "performing analysis on team metrics") + results = metricloop(tbakey, apikey, competition, current_time, metrics_tests) + log(stdout, INF, "finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") + + start = time.time() + log(stdout, INF, "loading pit data") + pit_data = load_pit(apikey, competition) + log(stdout, INF, "finished loading pit data in " + str(time.time() - start) + " seconds") + + start = time.time() + log(stdout, INF, "performing analysis on pit data") + results = pitloop(apikey, competition, pit_data, pit_tests) + log(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds") + + start = time.time() + log(stdout, INF, "uploading pit results to database") + push_pit(apikey, competition, results) + log(stdout, INF, "finished uploading pit results in " + str(time.time() - start) + " seconds") + + set_current_time(apikey, current_time) + log(stdout, INF, "finished all tests in " + str(time.time() - loop_start) + " seconds, looping") + + except KeyboardInterrupt: + log(stdout, INF, "detected KeyboardInterrupt, killing threads") + if "exec_threads" in locals(): + exec_threads.terminate() + exec_threads.close() + log(stdout, INF, "terminated threads, exiting") + loop_stored_exception = sys.exc_info() + loop_exit_code = 0 + break + except Exception as e: + log(stderr, ERR, "encountered an exception while running") + print(e, file = stderr) + loop_exit_code = 1 + break + + sys.exit(loop_exit_code) + def load_config(path, config_vector): try: f = open(path, "r") + config_vector.update(json.load(f)) + f.close() + log(stdout, INF, "found and opened config at <" + path + ">") + return 0 except: - log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting") + log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting", code = 100) f = open(path, "w") f.write(sample_json) f.close() return 1 - - config_vector = json.load(f) - f.close() - return 0 def save_config(path, config_vector): try: @@ -215,4 +385,6 @@ def save_config(path, config_vector): return 1 if __name__ == "__main__": + if sys.platform.startswith("win"): + multiprocessing.freeze_support() main() \ No newline at end of file From 3c7262498c049cc6666add57ced1d7f534059f46 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sun, 13 Jun 2021 00:49:04 +0000 Subject: [PATCH 027/143] superscript v 0.9.1 Former-commit-id: 246efd524bd0536150858703180ac8a77f1b0587 --- src/cli/superscript.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 0cbe4a0..1ad9058 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -3,10 +3,12 @@ # Notes: # setup: -__version__ = "0.9.0" +__version__ = "0.9.1" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 0.9.1: + - fixed bugs in configuration item loading exception handling 0.9.0: - moved printing and logging related functions to interface.py (changelog will stay in this file) - changed function return files for load_config and save_config to standard C values (0 for success, 1 for error) @@ -209,7 +211,7 @@ def main(): config = {} if load_config(config_path, config) == 1: - exit(1) + sys.exit(1) error_flag = False @@ -235,7 +237,7 @@ def main(): error_flag = True if error_flag: - exit(1) + sys.exit(1) error_flag = False if competition == None or competition == "": @@ -252,7 +254,7 @@ def main(): error_flag = True if error_flag: - exit(1) + sys.exit(1) log(stdout, INF, "found and loaded competition, match_tests, metrics_tests, pit_tests from config") @@ -261,7 +263,7 @@ def main(): cfg_max_threads = config["max-threads"] except: log(stderr, ERR, "max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) - exit(1) + sys.exit(1) if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : alloc_processes = sys_max_threads + cfg_max_threads @@ -273,7 +275,7 @@ def main(): alloc_processes = sys_max_threads else: log(stderr, ERR, "max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, code = 110) - exit(1) + sys.exit(1) log(stdout, INF, "found and loaded max-threads from config") log(stdout, INF, "attempting to start " + str(alloc_processes) + " threads") @@ -282,7 +284,7 @@ def main(): except Exception as e: log(stderr, ERR, "unable to start threads", code = 200) log(stderr, INF, e) - exit(1) + sys.exit(1) log(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") exit_flag = False @@ -290,16 +292,23 @@ def main(): try: apikey = config["key"]["database"] except: - log(stderr, ERR, "database key field in config must not be empty, please populate the database key", code = 111) + log(stderr, ERR, "database key field in config must be present", code = 111) exit_flag = True try: tbakey = config["key"]["tba"] except: - log(stderr, ERR, "tba key field in config must not be empty, please populate the tba key", code = 112) + log(stderr, ERR, "tba key field in config must be present", code = 112) + exit_flag = True + + if apikey == None or apikey == "": + log(stderr, ERR, "database key field in config must not be empty, please populate the database key") + exit_flag = True + if tbakey == None or tbakey == "": + log(stderr, ERR, "tba key field in config must not be empty, please populate the tba key") exit_flag = True if exit_flag: - exit(1) + sys.exit(1) log(stdout, INF, "found and loaded database and tba keys") From 7e800c9004ddb94748cace28c100ea1603963494 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 16 Jul 2021 06:12:26 +0000 Subject: [PATCH 028/143] added Pool.join to allow threads to exit safely added keyboard interrupt signal ignore in threads Former-commit-id: 836e9ca6be56adffffce9be0ac4cceaed08e6918 --- src/cli/processing.py | 4 ++++ src/cli/superscript.py | 1 + 2 files changed, 5 insertions(+) diff --git a/src/cli/processing.py b/src/cli/processing.py index 1f7e322..3778926 100644 --- a/src/cli/processing.py +++ b/src/cli/processing.py @@ -4,8 +4,12 @@ from tra_analysis import Analysis as an from dataset import push_metric from data import pull_new_tba_matches +import signal + def simplestats(data_test): + signal.signal(signal.SIGINT, signal.SIG_IGN) + data = np.array(data_test[0]) data = data[np.isfinite(data)] ranges = list(range(len(data))) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 1ad9058..2fe295a 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -357,6 +357,7 @@ def main(): log(stdout, INF, "detected KeyboardInterrupt, killing threads") if "exec_threads" in locals(): exec_threads.terminate() + exec_threads.join() exec_threads.close() log(stdout, INF, "terminated threads, exiting") loop_stored_exception = sys.exc_info() From 5d95913467e0a79c232f6c1fd5617971148eee49 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 11 Aug 2021 00:33:06 +0000 Subject: [PATCH 029/143] removed duplicate requirements Former-commit-id: f258a768bee1fb81a6baccfd7b2576a66514d946 --- src/requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/requirements.txt b/src/requirements.txt index a3d30e7..f99f4cd 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -5,8 +5,6 @@ tra-analysis dnspython pyinstaller -requests -pymongo numpy scipy From fb2ea60fea879df0ae185567b042f5c24273ad31 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 11 Aug 2021 18:10:04 +0000 Subject: [PATCH 030/143] started socketing superscript statuses Former-commit-id: a703fe486757989822960471a7181c8b72afe9fc --- src/cli/socket-test.html | 20 +++ src/cli/superscript-socket.py | 275 ++++++++++++++++++++++++++++++++++ src/cli/time-test.py | 20 +++ 3 files changed, 315 insertions(+) create mode 100644 src/cli/socket-test.html create mode 100644 src/cli/superscript-socket.py create mode 100644 src/cli/time-test.py diff --git a/src/cli/socket-test.html b/src/cli/socket-test.html new file mode 100644 index 0000000..98f28a1 --- /dev/null +++ b/src/cli/socket-test.html @@ -0,0 +1,20 @@ + + + + WebSocket demo + + + + + \ No newline at end of file diff --git a/src/cli/superscript-socket.py b/src/cli/superscript-socket.py new file mode 100644 index 0000000..b87f636 --- /dev/null +++ b/src/cli/superscript-socket.py @@ -0,0 +1,275 @@ +# testing purposes only, not to be used or run + +import json +import multiprocessing +import os +import math +from multiprocessing import Pool +import time +import warnings +import sys +import asyncio +import websockets + +from interface import splash, log, ERR, INF, stdout, stderr +from dataset import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit +from processing import matchloop, metricloop, pitloop + +config_path = "config.json" +sample_json = """{ + "max-threads": 0.5, + "team": "", + "competition": "2020ilch", + "key":{ + "database":"", + "tba":"" + }, + "statistics":{ + "match":{ + "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] + + }, + "metric":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + }, + "pit":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true + } + } +}""" + +async def main(socket, path): + + #warnings.filterwarnings("ignore") + #sys.stderr = open("errorlog.txt", "w") + + #splash(__version__) + + #loop_exit_code = 0 + #loop_stored_exception = None + + while True: + + try: + + loop_start = time.time() + + current_time = time.time() + await socket.send("current time: " + str(current_time)) + + config = {} + if load_config(config_path, config) == 1: + sys.exit(1) + + error_flag = False + + try: + competition = config["competition"] + except: + await socket.send("could not find competition field in config") + error_flag = True + try: + match_tests = config["statistics"]["match"] + except: + await socket.send("could not find match_tests field in config") + error_flag = True + try: + metrics_tests = config["statistics"]["metric"] + except: + await socket.send("could not find metrics_tests field in config") + error_flag = True + try: + pit_tests = config["statistics"]["pit"] + except: + await socket.send("could not find pit_tests field in config") + error_flag = True + + if error_flag: + sys.exit(1) + error_flag = False + + if competition == None or competition == "": + await socket.send("competition field in config must not be empty") + error_flag = True + if match_tests == None: + await socket.send("match_tests field in config must not be empty") + error_flag = True + if metrics_tests == None: + await socket.send("metrics_tests field in config must not be empty") + error_flag = True + if pit_tests == None: + await socket.send("pit_tests field in config must not be empty") + error_flag = True + + if error_flag: + sys.exit(1) + + await socket.send("found and loaded competition, match_tests, metrics_tests, pit_tests from config") + + sys_max_threads = os.cpu_count() + try: + cfg_max_threads = config["max-threads"] + except: + await socket.send("max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) + sys.exit(1) + + if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : + alloc_processes = sys_max_threads + cfg_max_threads + elif cfg_max_threads > 0 and cfg_max_threads < 1: + alloc_processes = math.floor(cfg_max_threads * sys_max_threads) + elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: + alloc_processes = cfg_max_threads + elif cfg_max_threads == 0: + alloc_processes = sys_max_threads + else: + await socket.send("max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads) + sys.exit(1) + + await socket.send("found and loaded max-threads from config") + await socket.send("attempting to start " + str(alloc_processes) + " threads") + try: + exec_threads = Pool(processes = alloc_processes) + except Exception as e: + await socket.send("unable to start threads") + #log(stderr, INF, e) + sys.exit(1) + await socket.send("successfully initialized " + str(alloc_processes) + " threads") + + exit_flag = False + + try: + apikey = config["key"]["database"] + except: + await socket.send("database key field in config must be present") + exit_flag = True + try: + tbakey = config["key"]["tba"] + except: + await socket.send("tba key field in config must be present") + exit_flag = True + + if apikey == None or apikey == "": + await socket.send("database key field in config must not be empty, please populate the database key") + exit_flag = True + if tbakey == None or tbakey == "": + await socket.send("tba key field in config must not be empty, please populate the tba key") + exit_flag = True + + if exit_flag: + sys.exit(1) + + await socket.send("found and loaded database and tba keys") + + previous_time = get_previous_time(apikey) + await socket.send("analysis backtimed to: " + str(previous_time)) + + start = time.time() + await socket.send("loading match data") + match_data = load_match(apikey, competition) + await socket.send("finished loading match data in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("performing analysis on match data") + results = matchloop(apikey, competition, match_data, match_tests, exec_threads) + await socket.send("finished match analysis in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("uploading match results to database") + push_match(apikey, competition, results) + await socket.send("finished uploading match results in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("performing analysis on team metrics") + results = metricloop(tbakey, apikey, competition, current_time, metrics_tests) + await socket.send("finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("loading pit data") + pit_data = load_pit(apikey, competition) + await socket.send("finished loading pit data in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("performing analysis on pit data") + results = pitloop(apikey, competition, pit_data, pit_tests) + await socket.send("finished pit analysis in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("uploading pit results to database") + push_pit(apikey, competition, results) + await socket.send("finished uploading pit results in " + str(time.time() - start) + " seconds") + + set_current_time(apikey, current_time) + await socket.send("finished all tests in " + str(time.time() - loop_start) + " seconds, looping") + + except KeyboardInterrupt: + await socket.send("detected KeyboardInterrupt, killing threads") + if "exec_threads" in locals(): + exec_threads.terminate() + exec_threads.join() + exec_threads.close() + await socket.send("terminated threads, exiting") + loop_stored_exception = sys.exc_info() + loop_exit_code = 0 + break + except Exception as e: + await socket.send("encountered an exception while running") + print(e, file = stderr) + loop_exit_code = 1 + break + + sys.exit(loop_exit_code) + +def load_config(path, config_vector): + try: + f = open(path, "r") + config_vector.update(json.load(f)) + f.close() + #socket.send("found and opened config at <" + path + ">") + return 0 + except: + #log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting", code = 100) + f = open(path, "w") + f.write(sample_json) + f.close() + return 1 + +def save_config(path, config_vector): + try: + f = open(path) + json.dump(config_vector) + f.close() + return 0 + except: + return 1 + +if __name__ == "__main__": + if sys.platform.startswith("win"): + multiprocessing.freeze_support() + start_server = websockets.serve(main, "127.0.0.1", 5678) + asyncio.get_event_loop().run_until_complete(start_server) + asyncio.get_event_loop().run_forever() \ No newline at end of file diff --git a/src/cli/time-test.py b/src/cli/time-test.py new file mode 100644 index 0000000..9d2c587 --- /dev/null +++ b/src/cli/time-test.py @@ -0,0 +1,20 @@ +import asyncio +import datetime +import random +import websockets + +async def time(websocket, path): + print(path) + i = 0 + while True: + #now = datetime.datetime.utcnow().isoformat() + "Z" + #await websocket.send(now) + #await asyncio.sleep(random.random() * 3) + i += 1 + await websocket.send(str(i)) + await asyncio.sleep(1) + +start_server = websockets.serve(time, "127.0.0.1", 5678) + +asyncio.get_event_loop().run_until_complete(start_server) +asyncio.get_event_loop().run_forever() \ No newline at end of file From 19bca6967cc5ff118ae1543beb6ec2516e04a779 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 11 Aug 2021 18:21:34 +0000 Subject: [PATCH 031/143] added websockets to requirements.txt Former-commit-id: d9ba7bcb2485c5342ac626d5b5259204568a12d0 --- src/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index f99f4cd..9833941 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -13,4 +13,6 @@ six pyparsing pandas -kivy==2.0.0rc2 \ No newline at end of file +kivy==2.0.0rc2 + +websockets \ No newline at end of file From 91f34a8d740e754101a924246e997be9f8f11d8a Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 11 Aug 2021 22:28:24 +0000 Subject: [PATCH 032/143] daemonized superscript socket example added python-daemon to requirements.txt Former-commit-id: 922095ebe0187477e5468d2e9bfa96bd98016d4a --- src/cli/superscript-socket.py | 18 +++++++++++++----- src/requirements.txt | 3 ++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/cli/superscript-socket.py b/src/cli/superscript-socket.py index b87f636..24395a2 100644 --- a/src/cli/superscript-socket.py +++ b/src/cli/superscript-socket.py @@ -10,6 +10,7 @@ import warnings import sys import asyncio import websockets +import lockfile from interface import splash, log, ERR, INF, stdout, stderr from dataset import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit @@ -267,9 +268,16 @@ def save_config(path, config_vector): except: return 1 +import daemon +from daemon import pidfile + if __name__ == "__main__": - if sys.platform.startswith("win"): - multiprocessing.freeze_support() - start_server = websockets.serve(main, "127.0.0.1", 5678) - asyncio.get_event_loop().run_until_complete(start_server) - asyncio.get_event_loop().run_forever() \ No newline at end of file + with daemon.DaemonContext( + working_directory=os.getcwd(), + pidfile=pidfile.TimeoutPIDLockFile("/var/run/tra-daemon.pid"), + ): + if sys.platform.startswith("win"): + multiprocessing.freeze_support() + start_server = websockets.serve(main, "127.0.0.1", 5678) + asyncio.get_event_loop().run_until_complete(start_server) + asyncio.get_event_loop().run_forever() \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt index 9833941..644ff65 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -15,4 +15,5 @@ pandas kivy==2.0.0rc2 -websockets \ No newline at end of file +websockets +python-daemon \ No newline at end of file From 962061007b49274950335a16dc3bf41ae9b447be Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 11 Aug 2021 22:54:24 +0000 Subject: [PATCH 033/143] removed time-test Former-commit-id: c09f4d0897d110bb7207d5aaeafeb6c3cd2d3f9f --- src/cli/time-test.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 src/cli/time-test.py diff --git a/src/cli/time-test.py b/src/cli/time-test.py deleted file mode 100644 index 9d2c587..0000000 --- a/src/cli/time-test.py +++ /dev/null @@ -1,20 +0,0 @@ -import asyncio -import datetime -import random -import websockets - -async def time(websocket, path): - print(path) - i = 0 - while True: - #now = datetime.datetime.utcnow().isoformat() + "Z" - #await websocket.send(now) - #await asyncio.sleep(random.random() * 3) - i += 1 - await websocket.send(str(i)) - await asyncio.sleep(1) - -start_server = websockets.serve(time, "127.0.0.1", 5678) - -asyncio.get_event_loop().run_until_complete(start_server) -asyncio.get_event_loop().run_forever() \ No newline at end of file From 3a068654eda9fc116cd3fd9d8b77246f62028a21 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 11 Aug 2021 23:26:52 +0000 Subject: [PATCH 034/143] added start/stop/restart argument functionality slightly buggy Former-commit-id: ba793140af2116a485bd4dfa5df45f407baec753 --- src/cli/superscript-socket.py | 53 +++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/src/cli/superscript-socket.py b/src/cli/superscript-socket.py index 24395a2..7a56d98 100644 --- a/src/cli/superscript-socket.py +++ b/src/cli/superscript-socket.py @@ -270,14 +270,57 @@ def save_config(path, config_vector): import daemon from daemon import pidfile +from signal import SIGTERM -if __name__ == "__main__": +def start(pid_path): + #print("starting") with daemon.DaemonContext( working_directory=os.getcwd(), - pidfile=pidfile.TimeoutPIDLockFile("/var/run/tra-daemon.pid"), + pidfile=pidfile.TimeoutPIDLockFile(pid_path), ): - if sys.platform.startswith("win"): - multiprocessing.freeze_support() start_server = websockets.serve(main, "127.0.0.1", 5678) asyncio.get_event_loop().run_until_complete(start_server) - asyncio.get_event_loop().run_forever() \ No newline at end of file + asyncio.get_event_loop().run_forever() + +def stop(pid_path): + #print("stopping") + try: + pf = open(pid_path, 'r') + pid = int(pf.read().strip()) + pf.close() + except IOError: + sys.stderr.write("pidfile at <" + pid_path + "> does not exist. Daemon not running?\n") + return + + try: + os.kill(pid, SIGTERM) + return + except OSError as err: + if err.find("No such process") > 0: + if os.path.exists(pid_path): + os.remove(pid_path) + else: + print(str(err)) + sys.exit(1) + +def restart(pid_path): + #print("restarting") + stop(pid_path) + start(pid_path) + +if __name__ == "__main__": + pid_path = "/var/run/tra-daemon.pid" + if len(sys.argv) == 2: + if 'start' == sys.argv[1]: + start(pid_path) + elif 'stop' == sys.argv[1]: + stop(pid_path) + elif 'restart' == sys.argv[1]: + restart(pid_path) + else: + print("usage: %s start|stop|restart" % sys.argv[0]) + sys.exit(2) + sys.exit(0) + else: + print("usage: %s start|stop|restart" % sys.argv[0]) + sys.exit(2) \ No newline at end of file From 30641e43d8bedf1ef77087e7df4c27fcd8e75d1a Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 11 Aug 2021 23:37:57 +0000 Subject: [PATCH 035/143] fixed delay in daemon stopping Former-commit-id: 1d9fa990582c214f3f40d1a10e97accfc38e3468 --- src/cli/superscript-socket.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cli/superscript-socket.py b/src/cli/superscript-socket.py index 7a56d98..14a96a3 100644 --- a/src/cli/superscript-socket.py +++ b/src/cli/superscript-socket.py @@ -291,11 +291,13 @@ def stop(pid_path): except IOError: sys.stderr.write("pidfile at <" + pid_path + "> does not exist. Daemon not running?\n") return - + try: - os.kill(pid, SIGTERM) - return + while True: + os.kill(pid, SIGTERM) + time.sleep(0.01) except OSError as err: + err = str(err) if err.find("No such process") > 0: if os.path.exists(pid_path): os.remove(pid_path) From b3ab9156dbecd8283597aecd98191effc1c10a2f Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 12 Aug 2021 21:53:03 +0000 Subject: [PATCH 036/143] consolidated datraset.py into datas.py removed unessasary time import in data.py added missing import to processing.py Former-commit-id: f74cfb3ae32172ffff7f472301a46a8cc3578918 --- src/cli/data.py | 78 ++++++++++++++++++++++++++++++++++++++-- src/cli/dataset.py | 74 -------------------------------------- src/cli/processing.py | 2 +- src/cli/superscript.spec | 1 - 4 files changed, 76 insertions(+), 79 deletions(-) delete mode 100644 src/cli/dataset.py diff --git a/src/cli/data.py b/src/cli/data.py index 641aba7..fb5dc27 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -1,7 +1,6 @@ import requests import pymongo import pandas as pd -import time def pull_new_tba_matches(apikey, competition, cutoff): api_key= apikey @@ -55,7 +54,7 @@ def get_metrics_data_formatted(apikey, competition): out = {} for i in x: try: - out[int(i)] = d.get_team_metrics_data(apikey, competition, int(i)) + out[int(i)] = get_team_metrics_data(apikey, competition, int(i)) except: pass return out @@ -126,4 +125,77 @@ def unkeyify_2l(layered_dict): add.append([j,layered_dict[i][j]]) add.sort(key = lambda x: x[0]) out[i] = list(map(lambda x: x[1], add)) - return out \ No newline at end of file + return out + +def get_previous_time(apikey): + + previous_time = get_analysis_flags(apikey, "latest_update") + + if previous_time == None: + + set_analysis_flags(apikey, "latest_update", 0) + previous_time = 0 + + else: + + previous_time = previous_time["latest_update"] + + return previous_time + +def set_current_time(apikey, current_time): + + set_analysis_flags(apikey, "latest_update", {"latest_update":current_time}) + +def load_match(apikey, competition): + + return get_match_data_formatted(apikey, competition) + +def load_metric(apikey, competition, match, group_name, metrics): + + group = {} + + for team in match[group_name]: + + db_data = get_team_metrics_data(apikey, competition, team) + + if get_team_metrics_data(apikey, competition, team) == None: + + elo = {"score": metrics["elo"]["score"]} + gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} + ts = {"mu": metrics["ts"]["mu"], "sigm+a": metrics["ts"]["sigma"]} + + group[team] = {"elo": elo, "gl2": gl2, "ts": ts} + + else: + + metrics = db_data["metrics"] + + elo = metrics["elo"] + gl2 = metrics["gl2"] + ts = metrics["ts"] + + group[team] = {"elo": elo, "gl2": gl2, "ts": ts} + + return group + +def load_pit(apikey, competition): + + return get_pit_data_formatted(apikey, competition) + +def push_match(apikey, competition, results): + + for team in results: + + push_team_tests_data(apikey, competition, team, results[team]) + +def push_metric(apikey, competition, metric): + + for team in metric: + + push_team_metrics_data(apikey, competition, team, metric[team]) + +def push_pit(apikey, competition, pit): + + for variable in pit: + + push_team_pit_data(apikey, competition, variable, pit[variable]) \ No newline at end of file diff --git a/src/cli/dataset.py b/src/cli/dataset.py deleted file mode 100644 index 5a69cc9..0000000 --- a/src/cli/dataset.py +++ /dev/null @@ -1,74 +0,0 @@ -import data as d - -def get_previous_time(apikey): - - previous_time = d.get_analysis_flags(apikey, "latest_update") - - if previous_time == None: - - d.set_analysis_flags(apikey, "latest_update", 0) - previous_time = 0 - - else: - - previous_time = previous_time["latest_update"] - - return previous_time - -def set_current_time(apikey, current_time): - - d.set_analysis_flags(apikey, "latest_update", {"latest_update":current_time}) - -def load_match(apikey, competition): - - return d.get_match_data_formatted(apikey, competition) - -def load_metric(apikey, competition, match, group_name, metrics): - - group = {} - - for team in match[group_name]: - - db_data = d.get_team_metrics_data(apikey, competition, team) - - if d.get_team_metrics_data(apikey, competition, team) == None: - - elo = {"score": metrics["elo"]["score"]} - gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} - ts = {"mu": metrics["ts"]["mu"], "sigm+a": metrics["ts"]["sigma"]} - - group[team] = {"elo": elo, "gl2": gl2, "ts": ts} - - else: - - metrics = db_data["metrics"] - - elo = metrics["elo"] - gl2 = metrics["gl2"] - ts = metrics["ts"] - - group[team] = {"elo": elo, "gl2": gl2, "ts": ts} - - return group - -def load_pit(apikey, competition): - - return d.get_pit_data_formatted(apikey, competition) - -def push_match(apikey, competition, results): - - for team in results: - - d.push_team_tests_data(apikey, competition, team, results[team]) - -def push_metric(apikey, competition, metric): - - for team in metric: - - d.push_team_metrics_data(apikey, competition, team, metric[team]) - -def push_pit(apikey, competition, pit): - - for variable in pit: - - d.push_team_pit_data(apikey, competition, variable, pit[variable]) \ No newline at end of file diff --git a/src/cli/processing.py b/src/cli/processing.py index 3778926..dd5ee2f 100644 --- a/src/cli/processing.py +++ b/src/cli/processing.py @@ -1,7 +1,7 @@ import numpy as np from tra_analysis import Analysis as an -from dataset import push_metric +from dataset import push_metric, load_metric from data import pull_new_tba_matches import signal diff --git a/src/cli/superscript.spec b/src/cli/superscript.spec index 5ffc4bf..04cafa7 100644 --- a/src/cli/superscript.spec +++ b/src/cli/superscript.spec @@ -2,7 +2,6 @@ block_cipher = None - a = Analysis(['superscript.py'], pathex=['/workspaces/tra-data-analysis/src'], binaries=[], From 6819aaf1431c44f5624fefe2040f28da1f31fc72 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 12 Aug 2021 21:58:41 +0000 Subject: [PATCH 037/143] consolidated dataset/data imports Former-commit-id: 6364746d7ab445c2a1164850a5fe3795d3e9688a --- src/cli/processing.py | 3 +-- src/cli/superscript-socket.py | 4 +--- src/cli/superscript.py | 7 +++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/cli/processing.py b/src/cli/processing.py index dd5ee2f..901522e 100644 --- a/src/cli/processing.py +++ b/src/cli/processing.py @@ -1,8 +1,7 @@ import numpy as np from tra_analysis import Analysis as an -from dataset import push_metric, load_metric -from data import pull_new_tba_matches +from data import pull_new_tba_matches, push_metric, load_metric import signal diff --git a/src/cli/superscript-socket.py b/src/cli/superscript-socket.py index 14a96a3..0df6922 100644 --- a/src/cli/superscript-socket.py +++ b/src/cli/superscript-socket.py @@ -1,7 +1,6 @@ # testing purposes only, not to be used or run import json -import multiprocessing import os import math from multiprocessing import Pool @@ -10,10 +9,9 @@ import warnings import sys import asyncio import websockets -import lockfile from interface import splash, log, ERR, INF, stdout, stderr -from dataset import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit +from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit from processing import matchloop, metricloop, pitloop config_path = "config.json" diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 2fe295a..01d2397 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -130,16 +130,15 @@ __all__ = [ # imports: import json -import multiprocessing import os import math -from multiprocessing import Pool +from multiprocessing import Pool, freeze_support import time import warnings import sys from interface import splash, log, ERR, INF, stdout, stderr -from dataset import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit +from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit from processing import matchloop, metricloop, pitloop config_path = "config.json" @@ -396,5 +395,5 @@ def save_config(path, config_vector): if __name__ == "__main__": if sys.platform.startswith("win"): - multiprocessing.freeze_support() + freeze_support() main() \ No newline at end of file From 2ebd2cba8a4f6fe0a8a5a312c5985660fa960ac4 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 12 Aug 2021 22:26:37 +0000 Subject: [PATCH 038/143] removed socket/daemon test scripts Former-commit-id: d1a9e567038fe705ced9ca8890cc79976326d4f8 --- src/cli/socket-test.html | 20 --- src/cli/superscript-socket.py | 326 ---------------------------------- 2 files changed, 346 deletions(-) delete mode 100644 src/cli/socket-test.html delete mode 100644 src/cli/superscript-socket.py diff --git a/src/cli/socket-test.html b/src/cli/socket-test.html deleted file mode 100644 index 98f28a1..0000000 --- a/src/cli/socket-test.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - WebSocket demo - - - - - \ No newline at end of file diff --git a/src/cli/superscript-socket.py b/src/cli/superscript-socket.py deleted file mode 100644 index 0df6922..0000000 --- a/src/cli/superscript-socket.py +++ /dev/null @@ -1,326 +0,0 @@ -# testing purposes only, not to be used or run - -import json -import os -import math -from multiprocessing import Pool -import time -import warnings -import sys -import asyncio -import websockets - -from interface import splash, log, ERR, INF, stdout, stderr -from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit -from processing import matchloop, metricloop, pitloop - -config_path = "config.json" -sample_json = """{ - "max-threads": 0.5, - "team": "", - "competition": "2020ilch", - "key":{ - "database":"", - "tba":"" - }, - "statistics":{ - "match":{ - "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] - - }, - "metric":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 - }, - "ts":{ - "mu":25, - "sigma":8.33 - } - }, - "pit":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true - } - } -}""" - -async def main(socket, path): - - #warnings.filterwarnings("ignore") - #sys.stderr = open("errorlog.txt", "w") - - #splash(__version__) - - #loop_exit_code = 0 - #loop_stored_exception = None - - while True: - - try: - - loop_start = time.time() - - current_time = time.time() - await socket.send("current time: " + str(current_time)) - - config = {} - if load_config(config_path, config) == 1: - sys.exit(1) - - error_flag = False - - try: - competition = config["competition"] - except: - await socket.send("could not find competition field in config") - error_flag = True - try: - match_tests = config["statistics"]["match"] - except: - await socket.send("could not find match_tests field in config") - error_flag = True - try: - metrics_tests = config["statistics"]["metric"] - except: - await socket.send("could not find metrics_tests field in config") - error_flag = True - try: - pit_tests = config["statistics"]["pit"] - except: - await socket.send("could not find pit_tests field in config") - error_flag = True - - if error_flag: - sys.exit(1) - error_flag = False - - if competition == None or competition == "": - await socket.send("competition field in config must not be empty") - error_flag = True - if match_tests == None: - await socket.send("match_tests field in config must not be empty") - error_flag = True - if metrics_tests == None: - await socket.send("metrics_tests field in config must not be empty") - error_flag = True - if pit_tests == None: - await socket.send("pit_tests field in config must not be empty") - error_flag = True - - if error_flag: - sys.exit(1) - - await socket.send("found and loaded competition, match_tests, metrics_tests, pit_tests from config") - - sys_max_threads = os.cpu_count() - try: - cfg_max_threads = config["max-threads"] - except: - await socket.send("max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) - sys.exit(1) - - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - await socket.send("max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads) - sys.exit(1) - - await socket.send("found and loaded max-threads from config") - await socket.send("attempting to start " + str(alloc_processes) + " threads") - try: - exec_threads = Pool(processes = alloc_processes) - except Exception as e: - await socket.send("unable to start threads") - #log(stderr, INF, e) - sys.exit(1) - await socket.send("successfully initialized " + str(alloc_processes) + " threads") - - exit_flag = False - - try: - apikey = config["key"]["database"] - except: - await socket.send("database key field in config must be present") - exit_flag = True - try: - tbakey = config["key"]["tba"] - except: - await socket.send("tba key field in config must be present") - exit_flag = True - - if apikey == None or apikey == "": - await socket.send("database key field in config must not be empty, please populate the database key") - exit_flag = True - if tbakey == None or tbakey == "": - await socket.send("tba key field in config must not be empty, please populate the tba key") - exit_flag = True - - if exit_flag: - sys.exit(1) - - await socket.send("found and loaded database and tba keys") - - previous_time = get_previous_time(apikey) - await socket.send("analysis backtimed to: " + str(previous_time)) - - start = time.time() - await socket.send("loading match data") - match_data = load_match(apikey, competition) - await socket.send("finished loading match data in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("performing analysis on match data") - results = matchloop(apikey, competition, match_data, match_tests, exec_threads) - await socket.send("finished match analysis in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("uploading match results to database") - push_match(apikey, competition, results) - await socket.send("finished uploading match results in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("performing analysis on team metrics") - results = metricloop(tbakey, apikey, competition, current_time, metrics_tests) - await socket.send("finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("loading pit data") - pit_data = load_pit(apikey, competition) - await socket.send("finished loading pit data in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("performing analysis on pit data") - results = pitloop(apikey, competition, pit_data, pit_tests) - await socket.send("finished pit analysis in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("uploading pit results to database") - push_pit(apikey, competition, results) - await socket.send("finished uploading pit results in " + str(time.time() - start) + " seconds") - - set_current_time(apikey, current_time) - await socket.send("finished all tests in " + str(time.time() - loop_start) + " seconds, looping") - - except KeyboardInterrupt: - await socket.send("detected KeyboardInterrupt, killing threads") - if "exec_threads" in locals(): - exec_threads.terminate() - exec_threads.join() - exec_threads.close() - await socket.send("terminated threads, exiting") - loop_stored_exception = sys.exc_info() - loop_exit_code = 0 - break - except Exception as e: - await socket.send("encountered an exception while running") - print(e, file = stderr) - loop_exit_code = 1 - break - - sys.exit(loop_exit_code) - -def load_config(path, config_vector): - try: - f = open(path, "r") - config_vector.update(json.load(f)) - f.close() - #socket.send("found and opened config at <" + path + ">") - return 0 - except: - #log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting", code = 100) - f = open(path, "w") - f.write(sample_json) - f.close() - return 1 - -def save_config(path, config_vector): - try: - f = open(path) - json.dump(config_vector) - f.close() - return 0 - except: - return 1 - -import daemon -from daemon import pidfile -from signal import SIGTERM - -def start(pid_path): - #print("starting") - with daemon.DaemonContext( - working_directory=os.getcwd(), - pidfile=pidfile.TimeoutPIDLockFile(pid_path), - ): - start_server = websockets.serve(main, "127.0.0.1", 5678) - asyncio.get_event_loop().run_until_complete(start_server) - asyncio.get_event_loop().run_forever() - -def stop(pid_path): - #print("stopping") - try: - pf = open(pid_path, 'r') - pid = int(pf.read().strip()) - pf.close() - except IOError: - sys.stderr.write("pidfile at <" + pid_path + "> does not exist. Daemon not running?\n") - return - - try: - while True: - os.kill(pid, SIGTERM) - time.sleep(0.01) - except OSError as err: - err = str(err) - if err.find("No such process") > 0: - if os.path.exists(pid_path): - os.remove(pid_path) - else: - print(str(err)) - sys.exit(1) - -def restart(pid_path): - #print("restarting") - stop(pid_path) - start(pid_path) - -if __name__ == "__main__": - pid_path = "/var/run/tra-daemon.pid" - if len(sys.argv) == 2: - if 'start' == sys.argv[1]: - start(pid_path) - elif 'stop' == sys.argv[1]: - stop(pid_path) - elif 'restart' == sys.argv[1]: - restart(pid_path) - else: - print("usage: %s start|stop|restart" % sys.argv[0]) - sys.exit(2) - sys.exit(0) - else: - print("usage: %s start|stop|restart" % sys.argv[0]) - sys.exit(2) \ No newline at end of file From c2f35f4cb2033c126c8277222f9428c1d5dd4433 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 13 Aug 2021 21:57:03 +0000 Subject: [PATCH 039/143] superscript v 0.9.2 Former-commit-id: e559ced751ae22c8c91c024940f50150e7da28ea --- src/cli/interface.py | 2 +- src/cli/superscript-socket.py | 2 +- src/cli/superscript.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/cli/interface.py b/src/cli/interface.py index 9ee821a..3754e77 100644 --- a/src/cli/interface.py +++ b/src/cli/interface.py @@ -17,7 +17,7 @@ stderr = sys.stderr def log(target, level, message, code = 0): - message = time.ctime() + empty_delim + str(level) + l_brack + f"{code:04}" + r_brack + empty_delim + soft_divided_delim + empty_delim + message + message = time.ctime() + empty_delim + str(level) + l_brack + f"{code:+05}" + r_brack + empty_delim + soft_divided_delim + empty_delim + message print(message, file = target) def clear(): diff --git a/src/cli/superscript-socket.py b/src/cli/superscript-socket.py index 0df6922..f6a46ba 100644 --- a/src/cli/superscript-socket.py +++ b/src/cli/superscript-socket.py @@ -309,7 +309,7 @@ def restart(pid_path): start(pid_path) if __name__ == "__main__": - pid_path = "/var/run/tra-daemon.pid" + pid_path = "tra-daemon.pid" if len(sys.argv) == 2: if 'start' == sys.argv[1]: start(pid_path) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 01d2397..5c146aa 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -3,10 +3,13 @@ # Notes: # setup: -__version__ = "0.9.1" +__version__ = "0.9.2" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 0.9.2: + - removed unessasary imports from data + - minor changes to interface 0.9.1: - fixed bugs in configuration item loading exception handling 0.9.0: @@ -138,7 +141,7 @@ import warnings import sys from interface import splash, log, ERR, INF, stdout, stderr -from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit +from data import get_previous_time, set_current_time, load_match, push_match, load_pit, push_pit from processing import matchloop, metricloop, pitloop config_path = "config.json" From b4c7365bf04e450343e453b213427b223f3cae02 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 13 Aug 2021 23:32:59 +0000 Subject: [PATCH 040/143] added *.pid files to git ignore Former-commit-id: f8f8d6bf2776ef26b79c23aac300756ad10b2781 --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 0e3db09..1158381 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ **/tra_analysis/ **/temp/* +**/*.pid + **/errorlog.txt /dist/superscript.* /dist/superscript \ No newline at end of file From bcbb653696dadcd803e46f0d94090b6a20917479 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 17 Aug 2021 21:02:08 +0000 Subject: [PATCH 041/143] daemonized, socketed superscript improved runtime removed superscript-socket.py Former-commit-id: 6fa29f767dcc649c7d049baddaaa1d865c5c3517 --- .gitignore | 2 + src/cli/data.py | 48 ++--- src/cli/processing.py | 12 +- src/cli/superscript-socket.py | 326 ---------------------------------- src/cli/superscript.py | 275 ++++++++++++++++++++++++++-- 5 files changed, 286 insertions(+), 377 deletions(-) delete mode 100644 src/cli/superscript-socket.py diff --git a/.gitignore b/.gitignore index 1158381..97705c5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ **/*.pid +**/profile + **/errorlog.txt /dist/superscript.* /dist/superscript \ No newline at end of file diff --git a/src/cli/data.py b/src/cli/data.py index fb5dc27..5c35d43 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -11,8 +11,7 @@ def pull_new_tba_matches(apikey, competition, cutoff): out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) return out -def get_team_match_data(apikey, competition, team_num): - client = pymongo.MongoClient(apikey) +def get_team_match_data(client, competition, team_num): db = client.data_scouting mdata = db.matchdata out = {} @@ -20,98 +19,87 @@ def get_team_match_data(apikey, competition, team_num): out[i['match']] = i['data'] return pd.DataFrame(out) -def get_team_pit_data(apikey, competition, team_num): - client = pymongo.MongoClient(apikey) +def get_team_pit_data(client, competition, team_num): db = client.data_scouting mdata = db.pitdata out = {} return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"] -def get_team_metrics_data(apikey, competition, team_num): - client = pymongo.MongoClient(apikey) +def get_team_metrics_data(client, competition, team_num): db = client.data_processing mdata = db.team_metrics return mdata.find_one({"competition" : competition, "team": team_num}) -def get_match_data_formatted(apikey, competition): - client = pymongo.MongoClient(apikey) +def get_match_data_formatted(client, competition): db = client.data_scouting mdata = db.teamlist x=mdata.find_one({"competition":competition}) out = {} for i in x: try: - out[int(i)] = unkeyify_2l(get_team_match_data(apikey, competition, int(i)).transpose().to_dict()) + out[int(i)] = unkeyify_2l(get_team_match_data(client, competition, int(i)).transpose().to_dict()) except: pass return out -def get_metrics_data_formatted(apikey, competition): - client = pymongo.MongoClient(apikey) +def get_metrics_data_formatted(client, competition): db = client.data_scouting mdata = db.teamlist x=mdata.find_one({"competition":competition}) out = {} for i in x: try: - out[int(i)] = get_team_metrics_data(apikey, competition, int(i)) + out[int(i)] = get_team_metrics_data(client, competition, int(i)) except: pass return out -def get_pit_data_formatted(apikey, competition): - client = pymongo.MongoClient(apikey) +def get_pit_data_formatted(client, competition): db = client.data_scouting mdata = db.teamlist x=mdata.find_one({"competition":competition}) out = {} for i in x: try: - out[int(i)] = get_team_pit_data(apikey, competition, int(i)) + out[int(i)] = get_team_pit_data(client, competition, int(i)) except: pass return out -def get_pit_variable_data(apikey, competition): - client = pymongo.MongoClient(apikey) +def get_pit_variable_data(client, competition): db = client.data_processing mdata = db.team_pit out = {} return mdata.find() -def get_pit_variable_formatted(apikey, competition): - temp = get_pit_variable_data(apikey, competition) +def get_pit_variable_formatted(client, competition): + temp = get_pit_variable_data(client, competition) out = {} for i in temp: out[i["variable"]] = i["data"] return out -def push_team_tests_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_tests"): - client = pymongo.MongoClient(apikey) +def push_team_tests_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_tests"): db = client[dbname] mdata = db[colname] mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "data" : data}, True) -def push_team_metrics_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"): - client = pymongo.MongoClient(apikey) +def push_team_metrics_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"): db = client[dbname] mdata = db[colname] mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True) -def push_team_pit_data(apikey, competition, variable, data, dbname = "data_processing", colname = "team_pit"): - client = pymongo.MongoClient(apikey) +def push_team_pit_data(client, competition, variable, data, dbname = "data_processing", colname = "team_pit"): db = client[dbname] mdata = db[colname] mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True) -def get_analysis_flags(apikey, flag): - client = pymongo.MongoClient(apikey) +def get_analysis_flags(client, flag): db = client.data_processing mdata = db.flags return mdata.find_one({flag:{"$exists":True}}) -def set_analysis_flags(apikey, flag, data): - client = pymongo.MongoClient(apikey) +def set_analysis_flags(client, flag, data): db = client.data_processing mdata = db.flags return mdata.replace_one({flag:{"$exists":True}}, data, True) @@ -158,7 +146,7 @@ def load_metric(apikey, competition, match, group_name, metrics): db_data = get_team_metrics_data(apikey, competition, team) - if get_team_metrics_data(apikey, competition, team) == None: + if db_data == None: elo = {"score": metrics["elo"]["score"]} gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} diff --git a/src/cli/processing.py b/src/cli/processing.py index 901522e..fe028e5 100644 --- a/src/cli/processing.py +++ b/src/cli/processing.py @@ -36,7 +36,7 @@ def simplestats(data_test): if test == "regression_sigmoidal": return an.regression(ranges, data, ['sig']) -def matchloop(apikey, competition, data, tests, exec_threads): +def matchloop(client, competition, data, tests, exec_threads): short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} @@ -88,7 +88,7 @@ def matchloop(apikey, competition, data, tests, exec_threads): return return_vector -def metricloop(tbakey, apikey, competition, timestamp, metrics): # listener based metrics update +def metricloop(tbakey, client, competition, timestamp, metrics): # listener based metrics update elo_N = metrics["elo"]["N"] elo_K = metrics["elo"]["K"] @@ -100,8 +100,8 @@ def metricloop(tbakey, apikey, competition, timestamp, metrics): # listener base for match in matches: - red = load_metric(apikey, competition, match, "red", metrics) - blu = load_metric(apikey, competition, match, "blue", metrics) + red = load_metric(client, competition, match, "red", metrics) + blu = load_metric(client, competition, match, "blue", metrics) elo_red_total = 0 elo_blu_total = 0 @@ -179,9 +179,9 @@ def metricloop(tbakey, apikey, competition, timestamp, metrics): # listener base temp_vector.update(red) temp_vector.update(blu) - push_metric(apikey, competition, temp_vector) + push_metric(client, competition, temp_vector) -def pitloop(apikey, competition, pit, tests): +def pitloop(client, competition, pit, tests): return_vector = {} for team in pit: diff --git a/src/cli/superscript-socket.py b/src/cli/superscript-socket.py deleted file mode 100644 index f6a46ba..0000000 --- a/src/cli/superscript-socket.py +++ /dev/null @@ -1,326 +0,0 @@ -# testing purposes only, not to be used or run - -import json -import os -import math -from multiprocessing import Pool -import time -import warnings -import sys -import asyncio -import websockets - -from interface import splash, log, ERR, INF, stdout, stderr -from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit -from processing import matchloop, metricloop, pitloop - -config_path = "config.json" -sample_json = """{ - "max-threads": 0.5, - "team": "", - "competition": "2020ilch", - "key":{ - "database":"", - "tba":"" - }, - "statistics":{ - "match":{ - "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] - - }, - "metric":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 - }, - "ts":{ - "mu":25, - "sigma":8.33 - } - }, - "pit":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true - } - } -}""" - -async def main(socket, path): - - #warnings.filterwarnings("ignore") - #sys.stderr = open("errorlog.txt", "w") - - #splash(__version__) - - #loop_exit_code = 0 - #loop_stored_exception = None - - while True: - - try: - - loop_start = time.time() - - current_time = time.time() - await socket.send("current time: " + str(current_time)) - - config = {} - if load_config(config_path, config) == 1: - sys.exit(1) - - error_flag = False - - try: - competition = config["competition"] - except: - await socket.send("could not find competition field in config") - error_flag = True - try: - match_tests = config["statistics"]["match"] - except: - await socket.send("could not find match_tests field in config") - error_flag = True - try: - metrics_tests = config["statistics"]["metric"] - except: - await socket.send("could not find metrics_tests field in config") - error_flag = True - try: - pit_tests = config["statistics"]["pit"] - except: - await socket.send("could not find pit_tests field in config") - error_flag = True - - if error_flag: - sys.exit(1) - error_flag = False - - if competition == None or competition == "": - await socket.send("competition field in config must not be empty") - error_flag = True - if match_tests == None: - await socket.send("match_tests field in config must not be empty") - error_flag = True - if metrics_tests == None: - await socket.send("metrics_tests field in config must not be empty") - error_flag = True - if pit_tests == None: - await socket.send("pit_tests field in config must not be empty") - error_flag = True - - if error_flag: - sys.exit(1) - - await socket.send("found and loaded competition, match_tests, metrics_tests, pit_tests from config") - - sys_max_threads = os.cpu_count() - try: - cfg_max_threads = config["max-threads"] - except: - await socket.send("max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) - sys.exit(1) - - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - await socket.send("max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads) - sys.exit(1) - - await socket.send("found and loaded max-threads from config") - await socket.send("attempting to start " + str(alloc_processes) + " threads") - try: - exec_threads = Pool(processes = alloc_processes) - except Exception as e: - await socket.send("unable to start threads") - #log(stderr, INF, e) - sys.exit(1) - await socket.send("successfully initialized " + str(alloc_processes) + " threads") - - exit_flag = False - - try: - apikey = config["key"]["database"] - except: - await socket.send("database key field in config must be present") - exit_flag = True - try: - tbakey = config["key"]["tba"] - except: - await socket.send("tba key field in config must be present") - exit_flag = True - - if apikey == None or apikey == "": - await socket.send("database key field in config must not be empty, please populate the database key") - exit_flag = True - if tbakey == None or tbakey == "": - await socket.send("tba key field in config must not be empty, please populate the tba key") - exit_flag = True - - if exit_flag: - sys.exit(1) - - await socket.send("found and loaded database and tba keys") - - previous_time = get_previous_time(apikey) - await socket.send("analysis backtimed to: " + str(previous_time)) - - start = time.time() - await socket.send("loading match data") - match_data = load_match(apikey, competition) - await socket.send("finished loading match data in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("performing analysis on match data") - results = matchloop(apikey, competition, match_data, match_tests, exec_threads) - await socket.send("finished match analysis in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("uploading match results to database") - push_match(apikey, competition, results) - await socket.send("finished uploading match results in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("performing analysis on team metrics") - results = metricloop(tbakey, apikey, competition, current_time, metrics_tests) - await socket.send("finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("loading pit data") - pit_data = load_pit(apikey, competition) - await socket.send("finished loading pit data in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("performing analysis on pit data") - results = pitloop(apikey, competition, pit_data, pit_tests) - await socket.send("finished pit analysis in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("uploading pit results to database") - push_pit(apikey, competition, results) - await socket.send("finished uploading pit results in " + str(time.time() - start) + " seconds") - - set_current_time(apikey, current_time) - await socket.send("finished all tests in " + str(time.time() - loop_start) + " seconds, looping") - - except KeyboardInterrupt: - await socket.send("detected KeyboardInterrupt, killing threads") - if "exec_threads" in locals(): - exec_threads.terminate() - exec_threads.join() - exec_threads.close() - await socket.send("terminated threads, exiting") - loop_stored_exception = sys.exc_info() - loop_exit_code = 0 - break - except Exception as e: - await socket.send("encountered an exception while running") - print(e, file = stderr) - loop_exit_code = 1 - break - - sys.exit(loop_exit_code) - -def load_config(path, config_vector): - try: - f = open(path, "r") - config_vector.update(json.load(f)) - f.close() - #socket.send("found and opened config at <" + path + ">") - return 0 - except: - #log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting", code = 100) - f = open(path, "w") - f.write(sample_json) - f.close() - return 1 - -def save_config(path, config_vector): - try: - f = open(path) - json.dump(config_vector) - f.close() - return 0 - except: - return 1 - -import daemon -from daemon import pidfile -from signal import SIGTERM - -def start(pid_path): - #print("starting") - with daemon.DaemonContext( - working_directory=os.getcwd(), - pidfile=pidfile.TimeoutPIDLockFile(pid_path), - ): - start_server = websockets.serve(main, "127.0.0.1", 5678) - asyncio.get_event_loop().run_until_complete(start_server) - asyncio.get_event_loop().run_forever() - -def stop(pid_path): - #print("stopping") - try: - pf = open(pid_path, 'r') - pid = int(pf.read().strip()) - pf.close() - except IOError: - sys.stderr.write("pidfile at <" + pid_path + "> does not exist. Daemon not running?\n") - return - - try: - while True: - os.kill(pid, SIGTERM) - time.sleep(0.01) - except OSError as err: - err = str(err) - if err.find("No such process") > 0: - if os.path.exists(pid_path): - os.remove(pid_path) - else: - print(str(err)) - sys.exit(1) - -def restart(pid_path): - #print("restarting") - stop(pid_path) - start(pid_path) - -if __name__ == "__main__": - pid_path = "tra-daemon.pid" - if len(sys.argv) == 2: - if 'start' == sys.argv[1]: - start(pid_path) - elif 'stop' == sys.argv[1]: - stop(pid_path) - elif 'restart' == sys.argv[1]: - restart(pid_path) - else: - print("usage: %s start|stop|restart" % sys.argv[0]) - sys.exit(2) - sys.exit(0) - else: - print("usage: %s start|stop|restart" % sys.argv[0]) - sys.exit(2) \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 5c146aa..009c579 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -3,10 +3,17 @@ # Notes: # setup: -__version__ = "0.9.2" +__version__ = "1.0.0" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 1.0.0: + - superscript now runs in PEP 3143 compliant well behaved daemon on Linux systems + - linux superscript daemon has integrated websocket output to monitor progress/status remotely + - linux daemon now sends stderr to errorlog.txt + 0.9.3: + - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) + - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions 0.9.2: - removed unessasary imports from data - minor changes to interface @@ -139,9 +146,12 @@ from multiprocessing import Pool, freeze_support import time import warnings import sys +import asyncio +import websockets +import pymongo from interface import splash, log, ERR, INF, stdout, stderr -from data import get_previous_time, set_current_time, load_match, push_match, load_pit, push_pit +from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit from processing import matchloop, metricloop, pitloop config_path = "config.json" @@ -192,7 +202,181 @@ sample_json = """{ } }""" -def main(): +async def main_lin(socket, path): + + while True: + + try: + + loop_start = time.time() + + current_time = time.time() + await socket.send("current time: " + str(current_time)) + + config = {} + if load_config(config_path, config) == 1: + sys.exit(1) + + error_flag = False + + try: + competition = config["competition"] + except: + await socket.send("could not find competition field in config") + error_flag = True + try: + match_tests = config["statistics"]["match"] + except: + await socket.send("could not find match_tests field in config") + error_flag = True + try: + metrics_tests = config["statistics"]["metric"] + except: + await socket.send("could not find metrics_tests field in config") + error_flag = True + try: + pit_tests = config["statistics"]["pit"] + except: + await socket.send("could not find pit_tests field in config") + error_flag = True + + if error_flag: + sys.exit(1) + error_flag = False + + if competition == None or competition == "": + await socket.send("competition field in config must not be empty") + error_flag = True + if match_tests == None: + await socket.send("match_tests field in config must not be empty") + error_flag = True + if metrics_tests == None: + await socket.send("metrics_tests field in config must not be empty") + error_flag = True + if pit_tests == None: + await socket.send("pit_tests field in config must not be empty") + error_flag = True + + if error_flag: + sys.exit(1) + + await socket.send("found and loaded competition, match_tests, metrics_tests, pit_tests from config") + + sys_max_threads = os.cpu_count() + try: + cfg_max_threads = config["max-threads"] + except: + await socket.send("max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) + sys.exit(1) + + if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : + alloc_processes = sys_max_threads + cfg_max_threads + elif cfg_max_threads > 0 and cfg_max_threads < 1: + alloc_processes = math.floor(cfg_max_threads * sys_max_threads) + elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: + alloc_processes = cfg_max_threads + elif cfg_max_threads == 0: + alloc_processes = sys_max_threads + else: + await socket.send("max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads) + sys.exit(1) + + await socket.send("found and loaded max-threads from config") + await socket.send("attempting to start " + str(alloc_processes) + " threads") + try: + exec_threads = Pool(processes = alloc_processes) + except Exception as e: + await socket.send("unable to start threads") + sys.exit(1) + await socket.send("successfully initialized " + str(alloc_processes) + " threads") + + exit_flag = False + + try: + apikey = config["key"]["database"] + except: + await socket.send("database key field in config must be present") + exit_flag = True + try: + tbakey = config["key"]["tba"] + except: + await socket.send("tba key field in config must be present") + exit_flag = True + + if apikey == None or apikey == "": + await socket.send("database key field in config must not be empty, please populate the database key") + exit_flag = True + if tbakey == None or tbakey == "": + await socket.send("tba key field in config must not be empty, please populate the tba key") + exit_flag = True + + if exit_flag: + sys.exit(1) + + await socket.send("found and loaded database and tba keys") + + client = pymongo.MongoClient(apikey) + + previous_time = get_previous_time(client) + await socket.send("analysis backtimed to: " + str(previous_time)) + + start = time.time() + await socket.send("loading match data") + match_data = load_match(client, competition) + await socket.send("finished loading match data in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("performing analysis on match data") + results = matchloop(client, competition, match_data, match_tests, exec_threads) + await socket.send("finished match analysis in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("uploading match results to database") + push_match(client, competition, results) + await socket.send("finished uploading match results in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("performing analysis on team metrics") + results = metricloop(tbakey, client, competition, current_time, metrics_tests) + await socket.send("finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("loading pit data") + pit_data = load_pit(client, competition) + await socket.send("finished loading pit data in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("performing analysis on pit data") + results = pitloop(client, competition, pit_data, pit_tests) + await socket.send("finished pit analysis in " + str(time.time() - start) + " seconds") + + start = time.time() + await socket.send("uploading pit results to database") + push_pit(client, competition, results) + await socket.send("finished uploading pit results in " + str(time.time() - start) + " seconds") + + set_current_time(client, current_time) + await socket.send("finished all tests in " + str(time.time() - loop_start) + " seconds, looping") + + except KeyboardInterrupt: + await socket.send("detected KeyboardInterrupt, killing threads") + if "exec_threads" in locals(): + exec_threads.terminate() + exec_threads.join() + exec_threads.close() + await socket.send("terminated threads, exiting") + loop_stored_exception = sys.exc_info() + loop_exit_code = 0 + break + except Exception as e: + await socket.send("encountered an exception while running") + print(e) + loop_exit_code = 1 + break + + sys.exit(loop_exit_code) + +def main_win(): # windows main function warnings.filterwarnings("ignore") sys.stderr = open("errorlog.txt", "w") @@ -314,45 +498,47 @@ def main(): log(stdout, INF, "found and loaded database and tba keys") - previous_time = get_previous_time(apikey) + client = pymongo.MongoClient(apikey) + + previous_time = get_previous_time(client) log(stdout, INF, "analysis backtimed to: " + str(previous_time)) start = time.time() log(stdout, INF, "loading match data") - match_data = load_match(apikey, competition) + match_data = load_match(client, competition) log(stdout, INF, "finished loading match data in " + str(time.time() - start) + " seconds") start = time.time() log(stdout, INF, "performing analysis on match data") - results = matchloop(apikey, competition, match_data, match_tests, exec_threads) + results = matchloop(client, competition, match_data, match_tests, exec_threads) log(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds") start = time.time() log(stdout, INF, "uploading match results to database") - push_match(apikey, competition, results) + push_match(client, competition, results) log(stdout, INF, "finished uploading match results in " + str(time.time() - start) + " seconds") start = time.time() log(stdout, INF, "performing analysis on team metrics") - results = metricloop(tbakey, apikey, competition, current_time, metrics_tests) + results = metricloop(tbakey, client, competition, current_time, metrics_tests) log(stdout, INF, "finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") start = time.time() log(stdout, INF, "loading pit data") - pit_data = load_pit(apikey, competition) + pit_data = load_pit(client, competition) log(stdout, INF, "finished loading pit data in " + str(time.time() - start) + " seconds") start = time.time() log(stdout, INF, "performing analysis on pit data") - results = pitloop(apikey, competition, pit_data, pit_tests) + results = pitloop(client, competition, pit_data, pit_tests) log(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds") start = time.time() log(stdout, INF, "uploading pit results to database") - push_pit(apikey, competition, results) + push_pit(client, competition, results) log(stdout, INF, "finished uploading pit results in " + str(time.time() - start) + " seconds") - set_current_time(apikey, current_time) + set_current_time(client, current_time) log(stdout, INF, "finished all tests in " + str(time.time() - loop_start) + " seconds, looping") except KeyboardInterrupt: @@ -378,10 +564,10 @@ def load_config(path, config_vector): f = open(path, "r") config_vector.update(json.load(f)) f.close() - log(stdout, INF, "found and opened config at <" + path + ">") + #socket.send("found and opened config at <" + path + ">") return 0 except: - log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting", code = 100) + #log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting", code = 100) f = open(path, "w") f.write(sample_json) f.close() @@ -396,7 +582,66 @@ def save_config(path, config_vector): except: return 1 +import daemon +from daemon import pidfile +from signal import SIGTERM + +def start(pid_path, profile = False): + f = open('errorlog.txt', 'w+') + with daemon.DaemonContext( + working_directory=os.getcwd(), + pidfile=pidfile.TimeoutPIDLockFile(pid_path), + stderr=f + ): + start_server = websockets.serve(main_lin, "127.0.0.1", 5678) + asyncio.get_event_loop().run_until_complete(start_server) + asyncio.get_event_loop().run_forever() + +def stop(pid_path): + try: + pf = open(pid_path, 'r') + pid = int(pf.read().strip()) + pf.close() + except IOError: + sys.stderr.write("pidfile at <" + pid_path + "> does not exist. Daemon not running?\n") + return + + try: + while True: + os.kill(pid, SIGTERM) + time.sleep(0.01) + except OSError as err: + err = str(err) + if err.find("No such process") > 0: + if os.path.exists(pid_path): + os.remove(pid_path) + else: + print(str(err)) + sys.exit(1) + +def restart(pid_path): + stop(pid_path) + start(pid_path) + if __name__ == "__main__": + if sys.platform.startswith("win"): freeze_support() - main() \ No newline at end of file + main_win() + + else: + pid_path = "tra-daemon.pid" + if len(sys.argv) == 2: + if 'start' == sys.argv[1]: + start(pid_path) + elif 'stop' == sys.argv[1]: + stop(pid_path) + elif 'restart' == sys.argv[1]: + restart(pid_path) + else: + print("usage: %s start|stop|restart|profile" % sys.argv[0]) + sys.exit(2) + sys.exit(0) + else: + print("usage: %s start|stop|restart|profile" % sys.argv[0]) + sys.exit(2) \ No newline at end of file From 871b313d95cc0e49a62beffddf344d5c744d4f22 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 17 Aug 2021 14:17:40 -0700 Subject: [PATCH 042/143] fixed linux only import in windows exec Former-commit-id: 61ae377e97a63385ffaaaa13413b207f54524902 --- src/cli/superscript.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 009c579..c16b7a7 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -376,7 +376,7 @@ async def main_lin(socket, path): sys.exit(loop_exit_code) -def main_win(): # windows main function +def main_win(): warnings.filterwarnings("ignore") sys.stderr = open("errorlog.txt", "w") @@ -582,10 +582,6 @@ def save_config(path, config_vector): except: return 1 -import daemon -from daemon import pidfile -from signal import SIGTERM - def start(pid_path, profile = False): f = open('errorlog.txt', 'w+') with daemon.DaemonContext( @@ -630,6 +626,9 @@ if __name__ == "__main__": main_win() else: + import daemon + from daemon import pidfile + from signal import SIGTERM pid_path = "tra-daemon.pid" if len(sys.argv) == 2: if 'start' == sys.argv[1]: From cafb773d8b25e0ec44ff80eedd3c5704e60501a8 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 18 Aug 2021 00:43:00 +0000 Subject: [PATCH 043/143] replaced profile to verbose option Former-commit-id: 055c296c00aa23bbc43e3c5e264a4e09c435b1cb --- src/cli/superscript.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index c16b7a7..de7f637 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -11,6 +11,7 @@ __changelog__ = """changelog: - superscript now runs in PEP 3143 compliant well behaved daemon on Linux systems - linux superscript daemon has integrated websocket output to monitor progress/status remotely - linux daemon now sends stderr to errorlog.txt + - added verbose option to linux superscript to allow for interactive output 0.9.3: - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions @@ -637,10 +638,12 @@ if __name__ == "__main__": stop(pid_path) elif 'restart' == sys.argv[1]: restart(pid_path) + elif 'verbose' == sys.argv[1]: + main_win() else: - print("usage: %s start|stop|restart|profile" % sys.argv[0]) + print("usage: %s start|stop|restart|verbose" % sys.argv[0]) sys.exit(2) sys.exit(0) else: - print("usage: %s start|stop|restart|profile" % sys.argv[0]) + print("usage: %s start|stop|restart|verbose" % sys.argv[0]) sys.exit(2) \ No newline at end of file From b6a1dfedb9c05ddf74cb9f906b017d79b6b2ac45 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 18 Aug 2021 03:04:01 +0000 Subject: [PATCH 044/143] not working Former-commit-id: e82177d17be396d02c30f55dc70b1a014e3388dd --- src/cli/superscript.py | 2 +- src/cli/superscript.spec | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index de7f637..800ddfc 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -371,7 +371,7 @@ async def main_lin(socket, path): break except Exception as e: await socket.send("encountered an exception while running") - print(e) + await socket.send(str(e)) loop_exit_code = 1 break diff --git a/src/cli/superscript.spec b/src/cli/superscript.spec index 04cafa7..7664ae0 100644 --- a/src/cli/superscript.spec +++ b/src/cli/superscript.spec @@ -10,6 +10,8 @@ a = Analysis(['superscript.py'], "dnspython", "sklearn.utils._weight_vector", "requests", + "websockets.legacy", + "websockets.legacy.server", ], hookspath=[], runtime_hooks=[], From 0287b5c0e209ea4389713c65d96553e5bcaf6a2b Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 18 Aug 2021 03:26:47 +0000 Subject: [PATCH 045/143] temporary workaround for missing ssl cert Former-commit-id: 739a2f36f8bfd0cce2b272c82f9a961554386f60 --- src/cli/data.py | 3 +-- src/cli/superscript.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/data.py b/src/cli/data.py index 5c35d43..e5bfb5b 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -1,10 +1,9 @@ import requests -import pymongo import pandas as pd def pull_new_tba_matches(apikey, competition, cutoff): api_key= apikey - x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth_Key":api_key}) + x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth_Key":api_key}, verify=False) out = [] for i in x.json(): if i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm": diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 800ddfc..34d82ee 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -12,6 +12,7 @@ __changelog__ = """changelog: - linux superscript daemon has integrated websocket output to monitor progress/status remotely - linux daemon now sends stderr to errorlog.txt - added verbose option to linux superscript to allow for interactive output + - moved pymongo import to superscript.py 0.9.3: - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions From ffead9e240c96b680663593725a4d3eb7c11bd2c Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 18 Aug 2021 05:13:25 +0000 Subject: [PATCH 046/143] fixed daemon no start issue Former-commit-id: efc353dafb9026d2d1fab1fa1be000e8997326d2 --- src/cli/superscript.py | 369 ++++++++++++++++++++++------------------- 1 file changed, 198 insertions(+), 171 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 34d82ee..d06dbea 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -151,6 +151,7 @@ import sys import asyncio import websockets import pymongo +import threading from interface import splash, log, ERR, INF, stdout, stderr from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit @@ -204,179 +205,213 @@ sample_json = """{ } }""" -async def main_lin(socket, path): +def main_lin(pid_path): + f = open('errorlog.txt', 'w+') + with daemon.DaemonContext( + working_directory=os.getcwd(), + pidfile=pidfile.TimeoutPIDLockFile(pid_path), + stderr=f + ): - while True: + async def handler(client, path): + clients.append(client) + while True: + try: + pong_waiter = await client.ping() + await pong_waiter + time.sleep(3) + except Exception as e: + clients.remove(client) + break - try: + clients = [] + start_server = websockets.serve(handler, "127.0.0.1", 5678) - loop_start = time.time() + asyncio.get_event_loop().run_until_complete(start_server) + threading.Thread(target = asyncio.get_event_loop().run_forever).start() - current_time = time.time() - await socket.send("current time: " + str(current_time)) + while True: - config = {} - if load_config(config_path, config) == 1: - sys.exit(1) - - error_flag = False - - try: - competition = config["competition"] - except: - await socket.send("could not find competition field in config") - error_flag = True - try: - match_tests = config["statistics"]["match"] - except: - await socket.send("could not find match_tests field in config") - error_flag = True - try: - metrics_tests = config["statistics"]["metric"] - except: - await socket.send("could not find metrics_tests field in config") - error_flag = True - try: - pit_tests = config["statistics"]["pit"] - except: - await socket.send("could not find pit_tests field in config") - error_flag = True + async def send_one(client, data): + await client.send(data) - if error_flag: - sys.exit(1) - error_flag = False + def send(data): + message_clients = clients.copy() + for client in message_clients: + try: + asyncio.run(send_one(client, data)) + except: + pass - if competition == None or competition == "": - await socket.send("competition field in config must not be empty") - error_flag = True - if match_tests == None: - await socket.send("match_tests field in config must not be empty") - error_flag = True - if metrics_tests == None: - await socket.send("metrics_tests field in config must not be empty") - error_flag = True - if pit_tests == None: - await socket.send("pit_tests field in config must not be empty") - error_flag = True - - if error_flag: - sys.exit(1) - - await socket.send("found and loaded competition, match_tests, metrics_tests, pit_tests from config") - - sys_max_threads = os.cpu_count() try: - cfg_max_threads = config["max-threads"] - except: - await socket.send("max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) - sys.exit(1) - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - await socket.send("max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads) - sys.exit(1) + loop_start = time.time() - await socket.send("found and loaded max-threads from config") - await socket.send("attempting to start " + str(alloc_processes) + " threads") - try: - exec_threads = Pool(processes = alloc_processes) + current_time = time.time() + send("current time: " + str(current_time)) + + config = {} + if load_config(config_path, config) == 1: + sys.exit(1) + + error_flag = False + + try: + competition = config["competition"] + except: + send("could not find competition field in config") + error_flag = True + try: + match_tests = config["statistics"]["match"] + except: + send("could not find match_tests field in config") + error_flag = True + try: + metrics_tests = config["statistics"]["metric"] + except: + send("could not find metrics_tests field in config") + error_flag = True + try: + pit_tests = config["statistics"]["pit"] + except: + send("could not find pit_tests field in config") + error_flag = True + + if error_flag: + sys.exit(1) + error_flag = False + + if competition == None or competition == "": + send("competition field in config must not be empty") + error_flag = True + if match_tests == None: + send("match_tests field in config must not be empty") + error_flag = True + if metrics_tests == None: + send("metrics_tests field in config must not be empty") + error_flag = True + if pit_tests == None: + send("pit_tests field in config must not be empty") + error_flag = True + + if error_flag: + sys.exit(1) + + send("found and loaded competition, match_tests, metrics_tests, pit_tests from config") + + sys_max_threads = os.cpu_count() + try: + cfg_max_threads = config["max-threads"] + except: + send("max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) + sys.exit(1) + + if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : + alloc_processes = sys_max_threads + cfg_max_threads + elif cfg_max_threads > 0 and cfg_max_threads < 1: + alloc_processes = math.floor(cfg_max_threads * sys_max_threads) + elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: + alloc_processes = cfg_max_threads + elif cfg_max_threads == 0: + alloc_processes = sys_max_threads + else: + send("max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads) + sys.exit(1) + + send("found and loaded max-threads from config") + send("attempting to start " + str(alloc_processes) + " threads") + try: + exec_threads = Pool(processes = alloc_processes) + except Exception as e: + send("unable to start threads") + sys.exit(1) + send("successfully initialized " + str(alloc_processes) + " threads") + + exit_flag = False + + try: + apikey = config["key"]["database"] + except: + send("database key field in config must be present") + exit_flag = True + try: + tbakey = config["key"]["tba"] + except: + send("tba key field in config must be present") + exit_flag = True + + if apikey == None or apikey == "": + send("database key field in config must not be empty, please populate the database key") + exit_flag = True + if tbakey == None or tbakey == "": + send("tba key field in config must not be empty, please populate the tba key") + exit_flag = True + + if exit_flag: + sys.exit(1) + + send("found and loaded database and tba keys") + + client = pymongo.MongoClient(apikey) + + previous_time = get_previous_time(client) + send("analysis backtimed to: " + str(previous_time)) + + start = time.time() + send("loading match data") + match_data = load_match(client, competition) + send("finished loading match data in " + str(time.time() - start) + " seconds") + + start = time.time() + send("performing analysis on match data") + results = matchloop(client, competition, match_data, match_tests, exec_threads) + send("finished match analysis in " + str(time.time() - start) + " seconds") + + start = time.time() + send("uploading match results to database") + push_match(client, competition, results) + send("finished uploading match results in " + str(time.time() - start) + " seconds") + + start = time.time() + send("performing analysis on team metrics") + results = metricloop(tbakey, client, competition, current_time, metrics_tests) + send("finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") + + start = time.time() + send("loading pit data") + pit_data = load_pit(client, competition) + send("finished loading pit data in " + str(time.time() - start) + " seconds") + + start = time.time() + send("performing analysis on pit data") + results = pitloop(client, competition, pit_data, pit_tests) + send("finished pit analysis in " + str(time.time() - start) + " seconds") + + start = time.time() + send("uploading pit results to database") + push_pit(client, competition, results) + send("finished uploading pit results in " + str(time.time() - start) + " seconds") + + set_current_time(client, current_time) + send("finished all tests in " + str(time.time() - loop_start) + " seconds, looping") + + except KeyboardInterrupt: + send("detected KeyboardInterrupt, killing threads") + if "exec_threads" in locals(): + exec_threads.terminate() + exec_threads.join() + exec_threads.close() + send("terminated threads, exiting") + loop_stored_exception = sys.exc_info() + loop_exit_code = 0 + break except Exception as e: - await socket.send("unable to start threads") - sys.exit(1) - await socket.send("successfully initialized " + str(alloc_processes) + " threads") + send("encountered an exception while running") + send(str(e)) + loop_exit_code = 1 + break - exit_flag = False - - try: - apikey = config["key"]["database"] - except: - await socket.send("database key field in config must be present") - exit_flag = True - try: - tbakey = config["key"]["tba"] - except: - await socket.send("tba key field in config must be present") - exit_flag = True - - if apikey == None or apikey == "": - await socket.send("database key field in config must not be empty, please populate the database key") - exit_flag = True - if tbakey == None or tbakey == "": - await socket.send("tba key field in config must not be empty, please populate the tba key") - exit_flag = True - - if exit_flag: - sys.exit(1) - - await socket.send("found and loaded database and tba keys") - - client = pymongo.MongoClient(apikey) - - previous_time = get_previous_time(client) - await socket.send("analysis backtimed to: " + str(previous_time)) - - start = time.time() - await socket.send("loading match data") - match_data = load_match(client, competition) - await socket.send("finished loading match data in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("performing analysis on match data") - results = matchloop(client, competition, match_data, match_tests, exec_threads) - await socket.send("finished match analysis in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("uploading match results to database") - push_match(client, competition, results) - await socket.send("finished uploading match results in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("performing analysis on team metrics") - results = metricloop(tbakey, client, competition, current_time, metrics_tests) - await socket.send("finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("loading pit data") - pit_data = load_pit(client, competition) - await socket.send("finished loading pit data in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("performing analysis on pit data") - results = pitloop(client, competition, pit_data, pit_tests) - await socket.send("finished pit analysis in " + str(time.time() - start) + " seconds") - - start = time.time() - await socket.send("uploading pit results to database") - push_pit(client, competition, results) - await socket.send("finished uploading pit results in " + str(time.time() - start) + " seconds") - - set_current_time(client, current_time) - await socket.send("finished all tests in " + str(time.time() - loop_start) + " seconds, looping") - - except KeyboardInterrupt: - await socket.send("detected KeyboardInterrupt, killing threads") - if "exec_threads" in locals(): - exec_threads.terminate() - exec_threads.join() - exec_threads.close() - await socket.send("terminated threads, exiting") - loop_stored_exception = sys.exc_info() - loop_exit_code = 0 - break - except Exception as e: - await socket.send("encountered an exception while running") - await socket.send(str(e)) - loop_exit_code = 1 - break - - sys.exit(loop_exit_code) + sys.exit(loop_exit_code) def main_win(): @@ -584,16 +619,8 @@ def save_config(path, config_vector): except: return 1 -def start(pid_path, profile = False): - f = open('errorlog.txt', 'w+') - with daemon.DaemonContext( - working_directory=os.getcwd(), - pidfile=pidfile.TimeoutPIDLockFile(pid_path), - stderr=f - ): - start_server = websockets.serve(main_lin, "127.0.0.1", 5678) - asyncio.get_event_loop().run_until_complete(start_server) - asyncio.get_event_loop().run_forever() +def start(pid_path): + main_lin(pid_path) def stop(pid_path): try: From 20f2040a1a0a5053b8faada17d9c87d0f590c687 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 18 Aug 2021 05:58:21 +0000 Subject: [PATCH 047/143] changed websocket bind address to 0.0.0.0 Former-commit-id: e9258c831d535e44096a0a7e36ea869de7030f81 --- src/cli/superscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index d06dbea..244b625 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -225,7 +225,7 @@ def main_lin(pid_path): break clients = [] - start_server = websockets.serve(handler, "127.0.0.1", 5678) + start_server = websockets.serve(handler, "0.0.0.0", 5678) asyncio.get_event_loop().run_until_complete(start_server) threading.Thread(target = asyncio.get_event_loop().run_forever).start() From 76f78047b311402f9437d3f5350d9e26e2d23456 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 18 Aug 2021 22:35:42 +0000 Subject: [PATCH 048/143] unified main_lin, main_win functions Former-commit-id: 342dae302291871603328224ae0a23ebeada47ce --- src/cli/superscript.py | 361 ++++++++++++----------------------------- 1 file changed, 101 insertions(+), 260 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 244b625..b375d2c 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -205,223 +205,17 @@ sample_json = """{ } }""" -def main_lin(pid_path): - f = open('errorlog.txt', 'w+') - with daemon.DaemonContext( - working_directory=os.getcwd(), - pidfile=pidfile.TimeoutPIDLockFile(pid_path), - stderr=f - ): +def main(send, verbose = False): - async def handler(client, path): - clients.append(client) - while True: - try: - pong_waiter = await client.ping() - await pong_waiter - time.sleep(3) - except Exception as e: - clients.remove(client) - break + if verbose : - clients = [] - start_server = websockets.serve(handler, "0.0.0.0", 5678) + warnings.filterwarnings("ignore") + sys.stderr = open("errorlog.txt", "w") - asyncio.get_event_loop().run_until_complete(start_server) - threading.Thread(target = asyncio.get_event_loop().run_forever).start() + splash(__version__) - while True: - - async def send_one(client, data): - await client.send(data) - - def send(data): - message_clients = clients.copy() - for client in message_clients: - try: - asyncio.run(send_one(client, data)) - except: - pass - - try: - - loop_start = time.time() - - current_time = time.time() - send("current time: " + str(current_time)) - - config = {} - if load_config(config_path, config) == 1: - sys.exit(1) - - error_flag = False - - try: - competition = config["competition"] - except: - send("could not find competition field in config") - error_flag = True - try: - match_tests = config["statistics"]["match"] - except: - send("could not find match_tests field in config") - error_flag = True - try: - metrics_tests = config["statistics"]["metric"] - except: - send("could not find metrics_tests field in config") - error_flag = True - try: - pit_tests = config["statistics"]["pit"] - except: - send("could not find pit_tests field in config") - error_flag = True - - if error_flag: - sys.exit(1) - error_flag = False - - if competition == None or competition == "": - send("competition field in config must not be empty") - error_flag = True - if match_tests == None: - send("match_tests field in config must not be empty") - error_flag = True - if metrics_tests == None: - send("metrics_tests field in config must not be empty") - error_flag = True - if pit_tests == None: - send("pit_tests field in config must not be empty") - error_flag = True - - if error_flag: - sys.exit(1) - - send("found and loaded competition, match_tests, metrics_tests, pit_tests from config") - - sys_max_threads = os.cpu_count() - try: - cfg_max_threads = config["max-threads"] - except: - send("max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) - sys.exit(1) - - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - send("max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads) - sys.exit(1) - - send("found and loaded max-threads from config") - send("attempting to start " + str(alloc_processes) + " threads") - try: - exec_threads = Pool(processes = alloc_processes) - except Exception as e: - send("unable to start threads") - sys.exit(1) - send("successfully initialized " + str(alloc_processes) + " threads") - - exit_flag = False - - try: - apikey = config["key"]["database"] - except: - send("database key field in config must be present") - exit_flag = True - try: - tbakey = config["key"]["tba"] - except: - send("tba key field in config must be present") - exit_flag = True - - if apikey == None or apikey == "": - send("database key field in config must not be empty, please populate the database key") - exit_flag = True - if tbakey == None or tbakey == "": - send("tba key field in config must not be empty, please populate the tba key") - exit_flag = True - - if exit_flag: - sys.exit(1) - - send("found and loaded database and tba keys") - - client = pymongo.MongoClient(apikey) - - previous_time = get_previous_time(client) - send("analysis backtimed to: " + str(previous_time)) - - start = time.time() - send("loading match data") - match_data = load_match(client, competition) - send("finished loading match data in " + str(time.time() - start) + " seconds") - - start = time.time() - send("performing analysis on match data") - results = matchloop(client, competition, match_data, match_tests, exec_threads) - send("finished match analysis in " + str(time.time() - start) + " seconds") - - start = time.time() - send("uploading match results to database") - push_match(client, competition, results) - send("finished uploading match results in " + str(time.time() - start) + " seconds") - - start = time.time() - send("performing analysis on team metrics") - results = metricloop(tbakey, client, competition, current_time, metrics_tests) - send("finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") - - start = time.time() - send("loading pit data") - pit_data = load_pit(client, competition) - send("finished loading pit data in " + str(time.time() - start) + " seconds") - - start = time.time() - send("performing analysis on pit data") - results = pitloop(client, competition, pit_data, pit_tests) - send("finished pit analysis in " + str(time.time() - start) + " seconds") - - start = time.time() - send("uploading pit results to database") - push_pit(client, competition, results) - send("finished uploading pit results in " + str(time.time() - start) + " seconds") - - set_current_time(client, current_time) - send("finished all tests in " + str(time.time() - loop_start) + " seconds, looping") - - except KeyboardInterrupt: - send("detected KeyboardInterrupt, killing threads") - if "exec_threads" in locals(): - exec_threads.terminate() - exec_threads.join() - exec_threads.close() - send("terminated threads, exiting") - loop_stored_exception = sys.exc_info() - loop_exit_code = 0 - break - except Exception as e: - send("encountered an exception while running") - send(str(e)) - loop_exit_code = 1 - break - - sys.exit(loop_exit_code) - -def main_win(): - - warnings.filterwarnings("ignore") - sys.stderr = open("errorlog.txt", "w") - - splash(__version__) - - loop_exit_code = 0 - loop_stored_exception = None + loop_exit_code = 0 + loop_stored_exception = None while True: @@ -430,33 +224,38 @@ def main_win(): loop_start = time.time() current_time = time.time() - log(stdout, INF, "current time: " + str(current_time)) + send(stdout, INF, "current time: " + str(current_time)) + + send(stdout, INF, "loading config at <" + config_path + ">", code = 0) config = {} if load_config(config_path, config) == 1: + send(stderr, ERR, "could not find config at <" + config_path + ">, generating blank config and exiting", code = 100) sys.exit(1) + send(stdout, INF, "found and opened config at <" + config_path + ">", code = 0) + error_flag = False try: competition = config["competition"] except: - log(stderr, ERR, "could not find competition field in config", code = 101) + send(stderr, ERR, "could not find competition field in config", code = 101) error_flag = True try: match_tests = config["statistics"]["match"] except: - log(stderr, ERR, "could not find match_tests field in config", code = 102) + send(stderr, ERR, "could not find match_tests field in config", code = 102) error_flag = True try: metrics_tests = config["statistics"]["metric"] except: - log(stderr, ERR, "could not find metrics_tests field in config", code = 103) + send(stderr, ERR, "could not find metrics_tests field in config", code = 103) error_flag = True try: pit_tests = config["statistics"]["pit"] except: - log(stderr, ERR, "could not find pit_tests field in config", code = 104) + send(stderr, ERR, "could not find pit_tests field in config", code = 104) error_flag = True if error_flag: @@ -464,28 +263,28 @@ def main_win(): error_flag = False if competition == None or competition == "": - log(stderr, ERR, "competition field in config must not be empty", code = 105) + send(stderr, ERR, "competition field in config must not be empty", code = 105) error_flag = True if match_tests == None: - log(stderr, ERR, "match_tests field in config must not be empty", code = 106) + send(stderr, ERR, "match_tests field in config must not be empty", code = 106) error_flag = True if metrics_tests == None: - log(stderr, ERR, "metrics_tests field in config must not be empty", code = 107) + send(stderr, ERR, "metrics_tests field in config must not be empty", code = 107) error_flag = True if pit_tests == None: - log(stderr, ERR, "pit_tests field in config must not be empty", code = 108) + send(stderr, ERR, "pit_tests field in config must not be empty", code = 108) error_flag = True if error_flag: sys.exit(1) - log(stdout, INF, "found and loaded competition, match_tests, metrics_tests, pit_tests from config") + send(stdout, INF, "found and loaded competition, match_tests, metrics_tests, pit_tests from config") sys_max_threads = os.cpu_count() try: cfg_max_threads = config["max-threads"] except: - log(stderr, ERR, "max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) + send(stderr, ERR, "max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) sys.exit(1) if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : @@ -497,99 +296,101 @@ def main_win(): elif cfg_max_threads == 0: alloc_processes = sys_max_threads else: - log(stderr, ERR, "max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, code = 110) + send(stderr, ERR, "max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, code = 110) sys.exit(1) - log(stdout, INF, "found and loaded max-threads from config") - log(stdout, INF, "attempting to start " + str(alloc_processes) + " threads") + send(stdout, INF, "found and loaded max-threads from config") + send(stdout, INF, "attempting to start " + str(alloc_processes) + " threads") try: exec_threads = Pool(processes = alloc_processes) except Exception as e: - log(stderr, ERR, "unable to start threads", code = 200) - log(stderr, INF, e) + send(stderr, ERR, "unable to start threads", code = 200) + send(stderr, INF, e) sys.exit(1) - log(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") + send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") exit_flag = False try: apikey = config["key"]["database"] except: - log(stderr, ERR, "database key field in config must be present", code = 111) + send(stderr, ERR, "database key field in config must be present", code = 111) exit_flag = True try: tbakey = config["key"]["tba"] except: - log(stderr, ERR, "tba key field in config must be present", code = 112) + send(stderr, ERR, "tba key field in config must be present", code = 112) exit_flag = True if apikey == None or apikey == "": - log(stderr, ERR, "database key field in config must not be empty, please populate the database key") + send(stderr, ERR, "database key field in config must not be empty, please populate the database key") exit_flag = True if tbakey == None or tbakey == "": - log(stderr, ERR, "tba key field in config must not be empty, please populate the tba key") + send(stderr, ERR, "tba key field in config must not be empty, please populate the tba key") exit_flag = True if exit_flag: sys.exit(1) - log(stdout, INF, "found and loaded database and tba keys") + send(stdout, INF, "found and loaded database and tba keys") client = pymongo.MongoClient(apikey) previous_time = get_previous_time(client) - log(stdout, INF, "analysis backtimed to: " + str(previous_time)) + send(stdout, INF, "analysis backtimed to: " + str(previous_time)) start = time.time() - log(stdout, INF, "loading match data") + send(stdout, INF, "loading match data") match_data = load_match(client, competition) - log(stdout, INF, "finished loading match data in " + str(time.time() - start) + " seconds") + send(stdout, INF, "finished loading match data in " + str(time.time() - start) + " seconds") start = time.time() - log(stdout, INF, "performing analysis on match data") + send(stdout, INF, "performing analysis on match data") results = matchloop(client, competition, match_data, match_tests, exec_threads) - log(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds") + send(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds") start = time.time() - log(stdout, INF, "uploading match results to database") + send(stdout, INF, "uploading match results to database") push_match(client, competition, results) - log(stdout, INF, "finished uploading match results in " + str(time.time() - start) + " seconds") + send(stdout, INF, "finished uploading match results in " + str(time.time() - start) + " seconds") start = time.time() - log(stdout, INF, "performing analysis on team metrics") + send(stdout, INF, "performing analysis on team metrics") results = metricloop(tbakey, client, competition, current_time, metrics_tests) - log(stdout, INF, "finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") + send(stdout, INF, "finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") start = time.time() - log(stdout, INF, "loading pit data") + send(stdout, INF, "loading pit data") pit_data = load_pit(client, competition) - log(stdout, INF, "finished loading pit data in " + str(time.time() - start) + " seconds") + send(stdout, INF, "finished loading pit data in " + str(time.time() - start) + " seconds") start = time.time() - log(stdout, INF, "performing analysis on pit data") + send(stdout, INF, "performing analysis on pit data") results = pitloop(client, competition, pit_data, pit_tests) - log(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds") + send(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds") start = time.time() - log(stdout, INF, "uploading pit results to database") + send(stdout, INF, "uploading pit results to database") push_pit(client, competition, results) - log(stdout, INF, "finished uploading pit results in " + str(time.time() - start) + " seconds") + send(stdout, INF, "finished uploading pit results in " + str(time.time() - start) + " seconds") + + client.close() set_current_time(client, current_time) - log(stdout, INF, "finished all tests in " + str(time.time() - loop_start) + " seconds, looping") + send(stdout, INF, "finished all tests in " + str(time.time() - loop_start) + " seconds, looping") except KeyboardInterrupt: - log(stdout, INF, "detected KeyboardInterrupt, killing threads") + send(stdout, INF, "detected KeyboardInterrupt, killing threads") if "exec_threads" in locals(): exec_threads.terminate() exec_threads.join() exec_threads.close() - log(stdout, INF, "terminated threads, exiting") + send(stdout, INF, "terminated threads, exiting") loop_stored_exception = sys.exc_info() loop_exit_code = 0 break except Exception as e: - log(stderr, ERR, "encountered an exception while running") + send(stderr, ERR, "encountered an exception while running") print(e, file = stderr) loop_exit_code = 1 break @@ -601,10 +402,8 @@ def load_config(path, config_vector): f = open(path, "r") config_vector.update(json.load(f)) f.close() - #socket.send("found and opened config at <" + path + ">") return 0 except: - #log(stderr, ERR, "could not find config at <" + path + ">, generating blank config and exiting", code = 100) f = open(path, "w") f.write(sample_json) f.close() @@ -619,8 +418,50 @@ def save_config(path, config_vector): except: return 1 -def start(pid_path): - main_lin(pid_path) +def start(pid_path, verbose = False): + + if not verbose: + + f = open('errorlog.txt', 'w+') + with daemon.DaemonContext( + working_directory=os.getcwd(), + pidfile=pidfile.TimeoutPIDLockFile(pid_path), + stderr=f + ): + + async def handler(client, path): + clients.append(client) + while True: + try: + pong_waiter = await client.ping() + await pong_waiter + time.sleep(3) + except Exception as e: + clients.remove(client) + break + + async def send_one(client, data): + await client.send(data) + + def send(target, level, message, code = 0): + message_clients = clients.copy() + for client in message_clients: + try: + asyncio.run(send_one(client, message)) + except: + pass + + clients = [] + start_server = websockets.serve(handler, "0.0.0.0", 5678) + + asyncio.get_event_loop().run_until_complete(start_server) + threading.Thread(target = asyncio.get_event_loop().run_forever).start() + + main(send) + + else: + + main(log, verbose=verbose) def stop(pid_path): try: @@ -652,7 +493,7 @@ if __name__ == "__main__": if sys.platform.startswith("win"): freeze_support() - main_win() + start(None, verbose = True) else: import daemon @@ -667,7 +508,7 @@ if __name__ == "__main__": elif 'restart' == sys.argv[1]: restart(pid_path) elif 'verbose' == sys.argv[1]: - main_win() + start(None, verbose = True) else: print("usage: %s start|stop|restart|verbose" % sys.argv[0]) sys.exit(2) From 11d3db4b44b873e28b7de5862c9c175c342b7868 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 19 Aug 2021 00:34:42 +0000 Subject: [PATCH 049/143] added profile option to superscript Former-commit-id: ea07d7c7094cd1469712e0d4e916ecd25dad5c23 --- .gitignore | 2 +- src/cli/superscript.py | 42 +++++++++++++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 97705c5..1573f33 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ **/*.pid -**/profile +**/profile.* **/errorlog.txt /dist/superscript.* diff --git a/src/cli/superscript.py b/src/cli/superscript.py index b375d2c..d3e77b8 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -13,6 +13,7 @@ __changelog__ = """changelog: - linux daemon now sends stderr to errorlog.txt - added verbose option to linux superscript to allow for interactive output - moved pymongo import to superscript.py + - added profile option to linux superscript to profile runtime of script 0.9.3: - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions @@ -205,9 +206,9 @@ sample_json = """{ } }""" -def main(send, verbose = False): +def main(send, verbose = False, profile = False): - if verbose : + if verbose or profile: warnings.filterwarnings("ignore") sys.stderr = open("errorlog.txt", "w") @@ -395,6 +396,9 @@ def main(send, verbose = False): loop_exit_code = 1 break + if profile: + return + sys.exit(loop_exit_code) def load_config(path, config_vector): @@ -418,9 +422,27 @@ def save_config(path, config_vector): except: return 1 -def start(pid_path, verbose = False): +def start(pid_path, verbose = False, profile = False): - if not verbose: + if profile: + + def send(target, level, message, code = 0): + pass + + import cProfile, pstats, io + profile = cProfile.Profile() + profile.enable() + main(send, profile = True) + profile.disable() + f = open("profile.txt", 'w+') + ps = pstats.Stats(profile, stream = f).sort_stats('cumtime') + ps.print_stats() + + elif verbose: + + main(log, verbose = verbose) + + else: f = open('errorlog.txt', 'w+') with daemon.DaemonContext( @@ -457,11 +479,7 @@ def start(pid_path, verbose = False): asyncio.get_event_loop().run_until_complete(start_server) threading.Thread(target = asyncio.get_event_loop().run_forever).start() - main(send) - - else: - - main(log, verbose=verbose) + main(send) def stop(pid_path): try: @@ -509,10 +527,12 @@ if __name__ == "__main__": restart(pid_path) elif 'verbose' == sys.argv[1]: start(None, verbose = True) + elif 'profile' == sys.argv[1]: + start(None, profile=True) else: - print("usage: %s start|stop|restart|verbose" % sys.argv[0]) + print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0]) sys.exit(2) sys.exit(0) else: - print("usage: %s start|stop|restart|verbose" % sys.argv[0]) + print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0]) sys.exit(2) \ No newline at end of file From 052788afb9a0592b0ce0d1599c9f476fe4056e43 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 19 Aug 2021 20:58:35 +0000 Subject: [PATCH 050/143] fixed verbose/profile output options Former-commit-id: ca399cf3500e066d5f3d448506dca7c5cebb27df --- src/cli/superscript.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index d3e77b8..58a5b07 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -208,16 +208,14 @@ sample_json = """{ def main(send, verbose = False, profile = False): - if verbose or profile: - - warnings.filterwarnings("ignore") - sys.stderr = open("errorlog.txt", "w") + warnings.filterwarnings("ignore") + sys.stderr = open("errorlog.txt", "w") + loop_exit_code = 0 + loop_stored_exception = None + if verbose: splash(__version__) - loop_exit_code = 0 - loop_stored_exception = None - while True: try: From 097fd2836b62552c5a6787bf597f794e22d67dd0 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sun, 22 Aug 2021 05:35:04 +0000 Subject: [PATCH 051/143] reorganized imports in superscript Former-commit-id: 18c26a00b622ad8f946ce589f556b085e678a413 --- src/cli/superscript.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 58a5b07..91d11b5 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -142,20 +142,20 @@ __all__ = [ # imports: +import asyncio import json -import os import math from multiprocessing import Pool, freeze_support +import os +import pymongo +import sys +import threading import time import warnings -import sys -import asyncio import websockets -import pymongo -import threading from interface import splash, log, ERR, INF, stdout, stderr -from data import get_previous_time, set_current_time, load_match, push_match, load_metric, push_metric, load_pit, push_pit +from data import get_previous_time, set_current_time, load_match, push_match, load_pit, push_pit from processing import matchloop, metricloop, pitloop config_path = "config.json" From 9be9008ae15942bba9da30397058cad5c8fbc912 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 26 Aug 2021 22:26:29 +0000 Subject: [PATCH 052/143] fixed and properly named build action Former-commit-id: 694733700a0d6933399e1040665ba4aa653f67b8 --- .github/workflows/build-cli.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-cli.yml b/.github/workflows/build-cli.yml index 681ab31..04e4e66 100644 --- a/.github/workflows/build-cli.yml +++ b/.github/workflows/build-cli.yml @@ -1,11 +1,11 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Superscript Unit Tests +name: Build Superscript Linux on: release: - types: [published, edited] + types: [published, created, edited] jobs: generate: @@ -14,4 +14,6 @@ jobs: steps: - name: Checkout master - uses: actions/checkout@master + uses: actions/checkout@master + - name: Echo test + run: echo "test" \ No newline at end of file From 2f90e7d11afad86d75d42ea7234f4acd55fe95fa Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 26 Aug 2021 22:48:48 +0000 Subject: [PATCH 053/143] added prerequisite and build steps Former-commit-id: 65a1720657e3d90a33621e55a413f07431d1c80f --- .github/workflows/build-cli.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-cli.yml b/.github/workflows/build-cli.yml index 04e4e66..175e095 100644 --- a/.github/workflows/build-cli.yml +++ b/.github/workflows/build-cli.yml @@ -11,9 +11,15 @@ jobs: generate: name: Build Linux runs-on: ubuntu-latest - steps: - name: Checkout master uses: actions/checkout@master - - name: Echo test - run: echo "test" \ No newline at end of file + - name: Install Dependencies + run: pip install -r requirements.txt + working-directory: src/ + - name: Give Execute Permission + run: chmod +x build/build-CLI.sh + working-directory: build/ + - name: Build Binary + run: ./build-CLI.sh + working-directory: build/ \ No newline at end of file From 66e00987c40bbc6dd975210753e4372b0bf77628 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 26 Aug 2021 22:53:21 +0000 Subject: [PATCH 054/143] fixed path Former-commit-id: 1106a0ffb1d393bd682fc70c3bf9dea7ea08575e --- .github/workflows/build-cli.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cli.yml b/.github/workflows/build-cli.yml index 175e095..ede3b4c 100644 --- a/.github/workflows/build-cli.yml +++ b/.github/workflows/build-cli.yml @@ -18,8 +18,8 @@ jobs: run: pip install -r requirements.txt working-directory: src/ - name: Give Execute Permission - run: chmod +x build/build-CLI.sh + run: chmod +x build-CLI.sh working-directory: build/ - name: Build Binary - run: ./build-CLI.sh + run: ./build-CLI.sh5 working-directory: build/ \ No newline at end of file From 3e212e4502e55baec9a33f350693779ad4f349cb Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 26 Aug 2021 22:57:15 +0000 Subject: [PATCH 055/143] fixed typo Former-commit-id: fc63d5d7e1acf874f70db955b581d229ed3a8322 --- .github/workflows/build-cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-cli.yml b/.github/workflows/build-cli.yml index ede3b4c..0828fa6 100644 --- a/.github/workflows/build-cli.yml +++ b/.github/workflows/build-cli.yml @@ -21,5 +21,5 @@ jobs: run: chmod +x build-CLI.sh working-directory: build/ - name: Build Binary - run: ./build-CLI.sh5 + run: ./build-CLI.sh working-directory: build/ \ No newline at end of file From 8000a7314d318f28eaf1b6eca60cef30dffaa6be Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 26 Aug 2021 23:14:09 +0000 Subject: [PATCH 056/143] add release asset automaticallt Former-commit-id: 42ca74b4ab541a9358cb5fe2d82ced0d48537f00 --- .github/workflows/build-cli.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-cli.yml b/.github/workflows/build-cli.yml index 0828fa6..0093a7c 100644 --- a/.github/workflows/build-cli.yml +++ b/.github/workflows/build-cli.yml @@ -22,4 +22,14 @@ jobs: working-directory: build/ - name: Build Binary run: ./build-CLI.sh - working-directory: build/ \ No newline at end of file + working-directory: build/ + - name: Copy Binary to Root Dir + run: cp superscript .. + working-directory: dist/ + - name: Upload Release Asset + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: superscript + asset_name: Superscript Linux Binary + tag: ${{ github.ref }} \ No newline at end of file From 2444963af9e675d16c30f64069c11c7aee043f89 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 26 Aug 2021 23:19:53 +0000 Subject: [PATCH 057/143] changed asset upload name removed create criteria, must be publish Former-commit-id: 73b5c393a016ed25ac49661e68f67e9aa495adaf --- .github/workflows/build-cli.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cli.yml b/.github/workflows/build-cli.yml index 0093a7c..7b9f8d7 100644 --- a/.github/workflows/build-cli.yml +++ b/.github/workflows/build-cli.yml @@ -5,7 +5,7 @@ name: Build Superscript Linux on: release: - types: [published, created, edited] + types: [published, edited] jobs: generate: @@ -31,5 +31,5 @@ jobs: with: repo_token: ${{ secrets.GITHUB_TOKEN }} file: superscript - asset_name: Superscript Linux Binary + asset_name: superscript tag: ${{ github.ref }} \ No newline at end of file From 56d3a0adcdc0f931818037ff2920fdfa51be02fb Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 27 Aug 2021 23:39:48 +0000 Subject: [PATCH 058/143] added event and time delay options (event delay unimplemented) Former-commit-id: fb4e5da1d4605a341056dc035b9db0695b51c24d --- src/cli/superscript.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 91d11b5..c10b09a 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -14,6 +14,9 @@ __changelog__ = """changelog: - added verbose option to linux superscript to allow for interactive output - moved pymongo import to superscript.py - added profile option to linux superscript to profile runtime of script + - added event and time delay options to config + - event delay pauses loop until even listener recieves an update + - time delay pauses loop until the time specified has elapsed since the BEGINNING of previous loop 0.9.3: - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions @@ -203,7 +206,9 @@ sample_json = """{ "climb-mechanism":true, "attitude":true } - } + }, + "even-delay":false, + "loop-delay":60 }""" def main(send, verbose = False, profile = False): @@ -378,6 +383,12 @@ def main(send, verbose = False, profile = False): set_current_time(client, current_time) send(stdout, INF, "finished all tests in " + str(time.time() - loop_start) + " seconds, looping") + loop_delay = float(config["loop-delay"]) + remaining_time = loop_delay - (time.time() - loop_start) + if remaining_time > 0: + send(stdout, INF, "loop delayed by " + str(remaining_time) + " seconds") + time.sleep(remaining_time) + except KeyboardInterrupt: send(stdout, INF, "detected KeyboardInterrupt, killing threads") if "exec_threads" in locals(): From 8112effbce5d028c3a1595a2db9ffe8ae8090fad Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 21 Sep 2021 04:43:32 +0000 Subject: [PATCH 059/143] added debug option, improved memory usage slightly, changed errorlog.txt to errorlog.log, updated .gitignore Former-commit-id: bd51efc6c2248edd987a9e5fc45113fa39af1c46 --- .gitignore | 1 + src/cli/processing.py | 27 ++++++++++----------------- src/cli/superscript.py | 26 ++++++++++++++++++++++---- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 1573f33..0c62b1f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ **/profile.* +**/*.log **/errorlog.txt /dist/superscript.* /dist/superscript \ No newline at end of file diff --git a/src/cli/processing.py b/src/cli/processing.py index fe028e5..73b1224 100644 --- a/src/cli/processing.py +++ b/src/cli/processing.py @@ -9,11 +9,11 @@ def simplestats(data_test): signal.signal(signal.SIGINT, signal.SIG_IGN) - data = np.array(data_test[0]) + data = np.array(data_test[3]) data = data[np.isfinite(data)] ranges = list(range(len(data))) - test = data_test[1] + test = data_test[2] if test == "basic_stats": return an.basic_stats(data) @@ -48,13 +48,7 @@ def matchloop(client, competition, data, tests, exec_threads): value = self[item] = type(self)() return value - return_vector = {} - - team_filtered = [] - variable_filtered = [] - variable_data = [] - test_filtered = [] - result_filtered = [] + input_vector = [] return_vector = AutoVivification() for team in data: @@ -65,25 +59,24 @@ def matchloop(client, competition, data, tests, exec_threads): for test in tests[variable]: - team_filtered.append(team) - variable_filtered.append(variable) - variable_data.append((data[team][variable], test)) - test_filtered.append(test) + input_vector.append((team, variable, test, data[team][variable])) - result_filtered = exec_threads.map(simplestats, variable_data) + result_filtered = exec_threads.map(simplestats, input_vector) + i = 0 result_filtered = list(result_filtered) for result in result_filtered: - filtered = test_filtered[i] + filtered = input_vector[i][2] try: short = short_mapping[filtered] - return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result[short] + return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result[short] except KeyError: # not in mapping - return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result + return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result + i += 1 return return_vector diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 91d11b5..ef15062 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -10,10 +10,12 @@ __changelog__ = """changelog: 1.0.0: - superscript now runs in PEP 3143 compliant well behaved daemon on Linux systems - linux superscript daemon has integrated websocket output to monitor progress/status remotely - - linux daemon now sends stderr to errorlog.txt + - linux daemon now sends stderr to errorlog.log - added verbose option to linux superscript to allow for interactive output - moved pymongo import to superscript.py - added profile option to linux superscript to profile runtime of script + - reduced memory usage slightly by consolidating the unwrapped input data + - added debug option, which performs one loop of analysis and dumps results to local files 0.9.3: - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions @@ -206,10 +208,10 @@ sample_json = """{ } }""" -def main(send, verbose = False, profile = False): +def main(send, verbose = False, profile = False, debug = False): warnings.filterwarnings("ignore") - sys.stderr = open("errorlog.txt", "w") + sys.stderr = open("errorlog.log", "w") loop_exit_code = 0 loop_stored_exception = None @@ -348,6 +350,11 @@ def main(send, verbose = False, profile = False): results = matchloop(client, competition, match_data, match_tests, exec_threads) send(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds") + if debug: + f = open("matchloop.log", "w+") + json.dump(results, f) + f.close() + start = time.time() send(stdout, INF, "uploading match results to database") push_match(client, competition, results) @@ -368,6 +375,11 @@ def main(send, verbose = False, profile = False): results = pitloop(client, competition, pit_data, pit_tests) send(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds") + if debug: + f = open("pitloop.log", "w+") + json.dump(results, f) + f.close() + start = time.time() send(stdout, INF, "uploading pit results to database") push_pit(client, competition, results) @@ -420,7 +432,7 @@ def save_config(path, config_vector): except: return 1 -def start(pid_path, verbose = False, profile = False): +def start(pid_path, verbose = False, profile = False, debug = False): if profile: @@ -440,6 +452,10 @@ def start(pid_path, verbose = False, profile = False): main(log, verbose = verbose) + elif debug: + + main(log, verbose = True, profile = True, debug = debug) + else: f = open('errorlog.txt', 'w+') @@ -527,6 +543,8 @@ if __name__ == "__main__": start(None, verbose = True) elif 'profile' == sys.argv[1]: start(None, profile=True) + elif 'debug' == sys.argv[1]: + start(None, debug = True) else: print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0]) sys.exit(2) From 2b8ecc5bee412ffceaaeb4b24660535c9dd59f19 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 21 Sep 2021 05:14:54 +0000 Subject: [PATCH 060/143] added databse config getter/setter to data.py, fixed args for data.py functions Former-commit-id: 74bcc116a221a6bc4865f8b19d4e5fd182aa9303 --- src/cli/data.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/cli/data.py b/src/cli/data.py index e5bfb5b..56f3054 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -114,13 +114,13 @@ def unkeyify_2l(layered_dict): out[i] = list(map(lambda x: x[1], add)) return out -def get_previous_time(apikey): +def get_previous_time(client): - previous_time = get_analysis_flags(apikey, "latest_update") + previous_time = get_analysis_flags(client, "latest_update") if previous_time == None: - set_analysis_flags(apikey, "latest_update", 0) + set_analysis_flags(client, "latest_update", 0) previous_time = 0 else: @@ -129,21 +129,29 @@ def get_previous_time(apikey): return previous_time -def set_current_time(apikey, current_time): +def set_current_time(client, current_time): - set_analysis_flags(apikey, "latest_update", {"latest_update":current_time}) + set_analysis_flags(client, "latest_update", {"latest_update":current_time}) -def load_match(apikey, competition): +def get_database_config(client): - return get_match_data_formatted(apikey, competition) + return get_analysis_flags(client, "config")["config"] -def load_metric(apikey, competition, match, group_name, metrics): +def set_database_config(client, config): + + set_analysis_flags(client, "config", {"config": config}) + +def load_match(client, competition): + + return get_match_data_formatted(client, competition) + +def load_metric(client, competition, match, group_name, metrics): group = {} for team in match[group_name]: - db_data = get_team_metrics_data(apikey, competition, team) + db_data = get_team_metrics_data(client, competition, team) if db_data == None: @@ -165,24 +173,24 @@ def load_metric(apikey, competition, match, group_name, metrics): return group -def load_pit(apikey, competition): +def load_pit(client, competition): - return get_pit_data_formatted(apikey, competition) + return get_pit_data_formatted(client, competition) -def push_match(apikey, competition, results): +def push_match(client, competition, results): for team in results: - push_team_tests_data(apikey, competition, team, results[team]) + push_team_tests_data(client, competition, team, results[team]) -def push_metric(apikey, competition, metric): +def push_metric(client, competition, metric): for team in metric: - push_team_metrics_data(apikey, competition, team, metric[team]) + push_team_metrics_data(client, competition, team, metric[team]) -def push_pit(apikey, competition, pit): +def push_pit(client, competition, pit): for variable in pit: - push_team_pit_data(apikey, competition, variable, pit[variable]) \ No newline at end of file + push_team_pit_data(client, competition, variable, pit[variable]) \ No newline at end of file From 16882a0a7549496c3cb1836a6fd13ea4b15c8ccd Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 21 Sep 2021 23:36:13 +0000 Subject: [PATCH 061/143] added debug option to usage description, set generic error code to 1 Former-commit-id: 9163244e2cb4f2d198d7a0bfa9f15ba32aebc627 --- src/cli/superscript.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index ef15062..6f00928 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -401,7 +401,7 @@ def main(send, verbose = False, profile = False, debug = False): loop_exit_code = 0 break except Exception as e: - send(stderr, ERR, "encountered an exception while running") + send(stderr, ERR, "encountered an exception while running", code = 1) print(e, file = stderr) loop_exit_code = 1 break @@ -546,9 +546,9 @@ if __name__ == "__main__": elif 'debug' == sys.argv[1]: start(None, debug = True) else: - print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0]) + print("usage: %s start|stop|restart|verbose|profile|debug" % sys.argv[0]) sys.exit(2) sys.exit(0) else: - print("usage: %s start|stop|restart|verbose|profile" % sys.argv[0]) + print("usage: %s start|stop|restart|verbose|profile|debug" % sys.argv[0]) sys.exit(2) \ No newline at end of file From 96ebb82085011050fc39a4f8d4c2c504249d3db9 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 23 Sep 2021 07:10:01 +0000 Subject: [PATCH 062/143] added opotion to pull config from database, major code refactor Former-commit-id: da913e639ba523dacd88ae5fce50d4037a9c9a57 --- src/cli/data.py | 3 +- src/cli/processing.py | 5 +- src/cli/superscript.py | 462 +++++++++++++++++++++++------------------ 3 files changed, 265 insertions(+), 205 deletions(-) diff --git a/src/cli/data.py b/src/cli/data.py index 56f3054..1fe53c5 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -135,7 +135,8 @@ def set_current_time(client, current_time): def get_database_config(client): - return get_analysis_flags(client, "config")["config"] + remote_config = get_analysis_flags(client, "config") + return remote_config["config"] if remote_config != None else None def set_database_config(client, config): diff --git a/src/cli/processing.py b/src/cli/processing.py index 73b1224..aa867f6 100644 --- a/src/cli/processing.py +++ b/src/cli/processing.py @@ -81,12 +81,13 @@ def matchloop(client, competition, data, tests, exec_threads): return return_vector -def metricloop(tbakey, client, competition, timestamp, metrics): # listener based metrics update +def metricloop(client, competition, data, metrics): # listener based metrics update elo_N = metrics["elo"]["N"] elo_K = metrics["elo"]["K"] - matches = pull_new_tba_matches(tbakey, competition, timestamp) + matches = data + #matches = pull_new_tba_matches(tbakey, competition, timestamp) red = {} blu = {} diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 5584032..d161fe7 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -19,6 +19,10 @@ __changelog__ = """changelog: - added event and time delay options to config - event delay pauses loop until even listener recieves an update - time delay pauses loop until the time specified has elapsed since the BEGINNING of previous loop + - added options to pull config information from database (reatins option to use local config file) + - config-preference option selects between prioritizing local config and prioritizing database config + - synchronize-config option selects whether to update the non prioritized config with the prioritized one + - divided config options between persistent ones (keys), and variable ones (everything else) 0.9.3: - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions @@ -160,65 +164,78 @@ import warnings import websockets from interface import splash, log, ERR, INF, stdout, stderr -from data import get_previous_time, set_current_time, load_match, push_match, load_pit, push_pit +from data import get_previous_time, pull_new_tba_matches, set_current_time, load_match, push_match, load_pit, push_pit, get_database_config, set_database_config from processing import matchloop, metricloop, pitloop config_path = "config.json" sample_json = """{ - "max-threads": 0.5, - "team": "", - "competition": "2020ilch", - "key":{ - "database":"", - "tba":"" - }, - "statistics":{ - "match":{ - "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] - + "persistent":{ + "key":{ + "database":"", + "tba":"" }, - "metric":{ - "elo":{ - "score":1500, - "N":400, - "K":24 + "config-preference":"local", + "synchronize-config":false + }, + "variable":{ + "max-threads":0.5, + "team":"", + "competition": "2020ilch", + "statistics":{ + "match":{ + "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] + }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 + "metric":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } }, - "ts":{ - "mu":25, - "sigma":8.33 + "pit":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true } }, - "pit":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true - } - }, - "even-delay":false, - "loop-delay":60 + "even-delay":false, + "loop-delay":60 + } }""" def main(send, verbose = False, profile = False, debug = False): + def close_all(): + if "exec_threads" in locals(): + exec_threads.terminate() + exec_threads.join() + exec_threads.close() + if "client" in locals(): + client.close() + warnings.filterwarnings("ignore") sys.stderr = open("errorlog.log", "w") - loop_exit_code = 0 - loop_stored_exception = None + exit_code = 0 if verbose: splash(__version__) @@ -229,173 +246,76 @@ def main(send, verbose = False, profile = False, debug = False): loop_start = time.time() - current_time = time.time() - send(stdout, INF, "current time: " + str(current_time)) - - send(stdout, INF, "loading config at <" + config_path + ">", code = 0) + send(stdout, INF, "current time: " + str(loop_start)) config = {} - if load_config(config_path, config) == 1: + + if load_config(config_path, config): send(stderr, ERR, "could not find config at <" + config_path + ">, generating blank config and exiting", code = 100) - sys.exit(1) - - send(stdout, INF, "found and opened config at <" + config_path + ">", code = 0) - - error_flag = False - - try: - competition = config["competition"] - except: - send(stderr, ERR, "could not find competition field in config", code = 101) - error_flag = True - try: - match_tests = config["statistics"]["match"] - except: - send(stderr, ERR, "could not find match_tests field in config", code = 102) - error_flag = True - try: - metrics_tests = config["statistics"]["metric"] - except: - send(stderr, ERR, "could not find metrics_tests field in config", code = 103) - error_flag = True - try: - pit_tests = config["statistics"]["pit"] - except: - send(stderr, ERR, "could not find pit_tests field in config", code = 104) - error_flag = True + exit_code = 1 + break - if error_flag: - sys.exit(1) - error_flag = False + send(stdout, INF, "found and loaded config at <" + config_path + ">") - if competition == None or competition == "": - send(stderr, ERR, "competition field in config must not be empty", code = 105) - error_flag = True - if match_tests == None: - send(stderr, ERR, "match_tests field in config must not be empty", code = 106) - error_flag = True - if metrics_tests == None: - send(stderr, ERR, "metrics_tests field in config must not be empty", code = 107) - error_flag = True - if pit_tests == None: - send(stderr, ERR, "pit_tests field in config must not be empty", code = 108) - error_flag = True - - if error_flag: - sys.exit(1) - - send(stdout, INF, "found and loaded competition, match_tests, metrics_tests, pit_tests from config") - - sys_max_threads = os.cpu_count() - try: - cfg_max_threads = config["max-threads"] - except: - send(stderr, ERR, "max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) - sys.exit(1) - - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - send(stderr, ERR, "max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, code = 110) - sys.exit(1) - - send(stdout, INF, "found and loaded max-threads from config") - send(stdout, INF, "attempting to start " + str(alloc_processes) + " threads") - try: - exec_threads = Pool(processes = alloc_processes) - except Exception as e: - send(stderr, ERR, "unable to start threads", code = 200) - send(stderr, INF, e) - sys.exit(1) - send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") - - exit_flag = False - - try: - apikey = config["key"]["database"] - except: - send(stderr, ERR, "database key field in config must be present", code = 111) - exit_flag = True - try: - tbakey = config["key"]["tba"] - except: - send(stderr, ERR, "tba key field in config must be present", code = 112) - exit_flag = True - - if apikey == None or apikey == "": - send(stderr, ERR, "database key field in config must not be empty, please populate the database key") - exit_flag = True - if tbakey == None or tbakey == "": - send(stderr, ERR, "tba key field in config must not be empty, please populate the tba key") - exit_flag = True - - if exit_flag: - sys.exit(1) - + flag, apikey, tbakey, preference, sync = parse_config_persistent(send, config) + if flag: + exit_code = 1 + break send(stdout, INF, "found and loaded database and tba keys") client = pymongo.MongoClient(apikey) - previous_time = get_previous_time(client) - send(stdout, INF, "analysis backtimed to: " + str(previous_time)) + send(stdout, INF, "established connection to database") + send(stdout, INF, "analysis backtimed to: " + str(get_previous_time(client))) + + resolve_config_conflicts(send, client, config, preference, sync) + if config == 1: + exit_code = 1 + break + flag, exec_threads, competition, match_tests, metrics_tests, pit_tests = parse_config_variable(send, config) + if flag: + exit_code = 1 + break start = time.time() - send(stdout, INF, "loading match data") + send(stdout, INF, "loading match, metric, pit data (this may take a few seconds)") match_data = load_match(client, competition) - send(stdout, INF, "finished loading match data in " + str(time.time() - start) + " seconds") + metrics_data = pull_new_tba_matches(tbakey, competition, loop_start) + pit_data = load_pit(client, competition) + send(stdout, INF, "finished loading match, metric, pit data in "+ str(time.time() - start) + " seconds") start = time.time() - send(stdout, INF, "performing analysis on match data") - results = matchloop(client, competition, match_data, match_tests, exec_threads) - send(stdout, INF, "finished match analysis in " + str(time.time() - start) + " seconds") + send(stdout, INF, "performing analysis on match, metrics, pit data") + match_results = matchloop(client, competition, match_data, match_tests, exec_threads) + metrics_results = metricloop(client, competition, metrics_data, metrics_tests) + pit_results = pitloop(client, competition, pit_data, pit_tests) + send(stdout, INF, "finished analysis in " + str(time.time() - start) + " seconds") + + start = time.time() + send(stdout, INF, "uploading match, metrics, pit results to database") + push_match(client, competition, match_results) + push_pit(client, competition, pit_results) + send(stdout, INF, "finished uploading results in " + str(time.time() - start) + " seconds") if debug: f = open("matchloop.log", "w+") - json.dump(results, f) + json.dump(match_results, f, ensure_ascii=False, indent=4) f.close() - start = time.time() - send(stdout, INF, "uploading match results to database") - push_match(client, competition, results) - send(stdout, INF, "finished uploading match results in " + str(time.time() - start) + " seconds") - - start = time.time() - send(stdout, INF, "performing analysis on team metrics") - results = metricloop(tbakey, client, competition, current_time, metrics_tests) - send(stdout, INF, "finished metric analysis and pushed to database in " + str(time.time() - start) + " seconds") - - start = time.time() - send(stdout, INF, "loading pit data") - pit_data = load_pit(client, competition) - send(stdout, INF, "finished loading pit data in " + str(time.time() - start) + " seconds") - - start = time.time() - send(stdout, INF, "performing analysis on pit data") - results = pitloop(client, competition, pit_data, pit_tests) - send(stdout, INF, "finished pit analysis in " + str(time.time() - start) + " seconds") - - if debug: f = open("pitloop.log", "w+") - json.dump(results, f) + json.dump(pit_results, f, ensure_ascii=False, indent=4) f.close() - start = time.time() - send(stdout, INF, "uploading pit results to database") - push_pit(client, competition, results) - send(stdout, INF, "finished uploading pit results in " + str(time.time() - start) + " seconds") + set_current_time(client, loop_start) + close_all() - client.close() + send(stdout, INF, "closed threads and database client") + send(stdout, INF, "finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") - set_current_time(client, current_time) - send(stdout, INF, "finished all tests in " + str(time.time() - loop_start) + " seconds, looping") + if profile: + return # return instead of break to avoid sys.exit - loop_delay = float(config["loop-delay"]) + loop_delay = float(config["variable"]["loop-delay"]) remaining_time = loop_delay - (time.time() - loop_start) if remaining_time > 0: send(stdout, INF, "loop delayed by " + str(remaining_time) + " seconds") @@ -403,24 +323,165 @@ def main(send, verbose = False, profile = False, debug = False): except KeyboardInterrupt: send(stdout, INF, "detected KeyboardInterrupt, killing threads") - if "exec_threads" in locals(): - exec_threads.terminate() - exec_threads.join() - exec_threads.close() + close_all() send(stdout, INF, "terminated threads, exiting") - loop_stored_exception = sys.exc_info() loop_exit_code = 0 break + except Exception as e: send(stderr, ERR, "encountered an exception while running", code = 1) print(e, file = stderr) - loop_exit_code = 1 + exit_code = 1 + close_all() break + + sys.exit(exit_code) - if profile: +def parse_config_persistent(send, config): + + exit_flag = False + try: + apikey = config["persistent"]["key"]["database"] + except: + send(stderr, ERR, "database key field in config must be present", code = 111) + exit_flag = True + try: + tbakey = config["persistent"]["key"]["tba"] + except: + send(stderr, ERR, "tba key field in config must be present", code = 112) + exit_flag = True + try: + preference = config["persistent"]["config-preference"] + except: + send(stderr, ERR, "config-preference field in config must be present", code = 113) + exit_flag = True + try: + sync = config["persistent"]["synchronize-config"] + except: + send(stderr, ERR, "synchronize-config field in config must be present", code = 114) + exit_flag = True + + if apikey == None or apikey == "": + send(stderr, ERR, "database key field in config must not be empty, please populate the database key", code = 115) + exit_flag = True + if tbakey == None or tbakey == "": + send(stderr, ERR, "tba key field in config must not be empty, please populate the tba key", code = 116) + exit_flag = True + if preference == None or preference == "": + send(stderr, ERR, "config-preference field in config must not be empty, please populate config-preference", code = 117) + exit_flag = True + if sync != True and sync != False: + send(stderr, ERR, "synchronize-config field in config must be a boolean, please populate synchronize-config", code = 118) + exit_flag = True + + return exit_flag, apikey, tbakey, preference, sync + +def parse_config_variable(send, config): + + exit_flag = False + + sys_max_threads = os.cpu_count() + try: + cfg_max_threads = config["variable"]["max-threads"] + except: + send(stderr, ERR, "max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) + exit_flag = True + + if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : + alloc_processes = sys_max_threads + cfg_max_threads + elif cfg_max_threads > 0 and cfg_max_threads < 1: + alloc_processes = math.floor(cfg_max_threads * sys_max_threads) + elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: + alloc_processes = cfg_max_threads + elif cfg_max_threads == 0: + alloc_processes = sys_max_threads + else: + send(stderr, ERR, "max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, code = 110) + exit_flag = True + + try: + exec_threads = Pool(processes = alloc_processes) + except Exception as e: + send(stderr, ERR, "unable to start threads", code = 200) + send(stderr, INF, e) + exit_flag = True + send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") + + try: + competition = config["variable"]["competition"] + except: + send(stderr, ERR, "could not find competition field in config", code = 101) + exit_flag = True + try: + match_tests = config["variable"]["statistics"]["match"] + except: + send(stderr, ERR, "could not find match field in config", code = 102) + exit_flag = True + try: + metrics_tests = config["variable"]["statistics"]["metric"] + except: + send(stderr, ERR, "could not find metrics field in config", code = 103) + exit_flag = True + try: + pit_tests = config["variable"]["statistics"]["pit"] + except: + send(stderr, ERR, "could not find pit field in config", code = 104) + exit_flag = True + + if competition == None or competition == "": + send(stderr, ERR, "competition field in config must not be empty", code = 105) + exit_flag = True + if match_tests == None: + send(stderr, ERR, "matchfield in config must not be empty", code = 106) + exit_flag = True + if metrics_tests == None: + send(stderr, ERR, "metrics field in config must not be empty", code = 107) + exit_flag = True + if pit_tests == None: + send(stderr, ERR, "pit field in config must not be empty", code = 108) + exit_flag = True + + send(stdout, INF, "found and loaded competition, match, metrics, pit from config") + + return exit_flag, exec_threads, competition, match_tests, metrics_tests, pit_tests + +def resolve_config_conflicts(send, client, config, preference, sync): + + if sync: + if preference == "local" or preference == "client": + send(stdout, INF, "config-preference set to local/client, loading local config information") + remote_config = get_database_config(client) + if remote_config != config["variable"]: + set_database_config(client, config["variable"]) + send(stdout, INF, "database config was different and was updated") + return + elif preference == "remote" or preference == "database": + send(stdout, INF, "config-preference set to remote/database, loading remote config information") + remote_config= get_database_config(client) + if remote_config != config["variable"]: + config["variable"] = remote_config + if save_config(config_path, config): + send(stderr, ERR, "local config was different but could not be updated") + config = 1 + return + send(stdout, INF, "local config was different and was updated") + return + else: + send(stderr, ERR, "config-preference field in config must be \"local\"/\"client\" or \"remote\"/\"database\"") + config = 1 + return + else: + if preference == "local" or preference == "client": + send(stdout, INF, "config-preference set to local/client, loading local config information") + return + elif preference == "remote" or preference == "database": + send(stdout, INF, "config-preference set to remote/database, loading database config information") + config["variable"] = get_database_config(client) + return + else: + send(stderr, ERR, "config-preference field in config must be \"local\"/\"client\" or \"remote\"/\"database\"") + config = 1 return - - sys.exit(loop_exit_code) def load_config(path, config_vector): try: @@ -435,13 +496,10 @@ def load_config(path, config_vector): return 1 def save_config(path, config_vector): - try: - f = open(path) - json.dump(config_vector) - f.close() - return 0 - except: - return 1 + f = open(path, "w+") + json.dump(config_vector, f, ensure_ascii=False, indent=4) + f.close() + return 0 def start(pid_path, verbose = False, profile = False, debug = False): From 32ae4fd6363061012d71c2b46b26e34ceb72dfcb Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sun, 10 Oct 2021 06:15:22 +0000 Subject: [PATCH 063/143] implemented event delay with dummy listener, event delay listener in data.py unimplemented Former-commit-id: dea5f3a37456c02b222052a62e80efc50670d3ff --- src/cli/data.py | 6 +++++- src/cli/superscript.py | 23 ++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/cli/data.py b/src/cli/data.py index e5bfb5b..926b589 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -185,4 +185,8 @@ def push_pit(apikey, competition, pit): for variable in pit: - push_team_pit_data(apikey, competition, variable, pit[variable]) \ No newline at end of file + push_team_pit_data(apikey, competition, variable, pit[variable]) + +def check_new_database_matches(client, competition): + + return True \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index c10b09a..f8859a5 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -158,7 +158,7 @@ import warnings import websockets from interface import splash, log, ERR, INF, stdout, stderr -from data import get_previous_time, set_current_time, load_match, push_match, load_pit, push_pit +from data import get_previous_time, set_current_time, load_match, push_match, load_pit, push_pit, check_new_database_matches from processing import matchloop, metricloop, pitloop config_path = "config.json" @@ -207,7 +207,7 @@ sample_json = """{ "attitude":true } }, - "even-delay":false, + "event-delay":false, "loop-delay":60 }""" @@ -383,11 +383,20 @@ def main(send, verbose = False, profile = False): set_current_time(client, current_time) send(stdout, INF, "finished all tests in " + str(time.time() - loop_start) + " seconds, looping") - loop_delay = float(config["loop-delay"]) - remaining_time = loop_delay - (time.time() - loop_start) - if remaining_time > 0: - send(stdout, INF, "loop delayed by " + str(remaining_time) + " seconds") - time.sleep(remaining_time) + event_delay = config["event-delay"] + if event_delay: + send(stdout, INF, "loop delayed until database returns new matches") + new_match = False + while not new_match: + time.sleep(1) + new_match = check_new_database_matches(client, competition) + send(stdout, INF, "database returned new matches") + else: + loop_delay = float(config["loop-delay"]) + remaining_time = loop_delay - (time.time() - loop_start) + if remaining_time > 0: + send(stdout, INF, "loop delayed by " + str(remaining_time) + " seconds") + time.sleep(remaining_time) except KeyboardInterrupt: send(stdout, INF, "detected KeyboardInterrupt, killing threads") From d89f99711727e47254fa3a28b18e9fe603c5c48c Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 11 Oct 2021 02:56:24 +0000 Subject: [PATCH 064/143] fixed sample config.json Former-commit-id: 24e666a23b2e121e711e715c389033321a86bf2f --- src/cli/superscript.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 791a960..4774902 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -171,8 +171,8 @@ config_path = "config.json" sample_json = """{ "persistent":{ "key":{ - "database":"", - "tba":"" + "database":"mongodb+srv://analysis:MU2gPeEjEurRt2n@2022-scouting-4vfuu.mongodb.net/?retryWrites=true&w=majority", + "tba":"UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5" }, "config-preference":"local", "synchronize-config":false @@ -216,10 +216,11 @@ sample_json = """{ "strategic-focus":true, "climb-mechanism":true, "attitude":true - }, - "event-delay":false, - "loop-delay":60 - } + } + }, + "event-delay":false, + "loop-delay":60 + } }""" def main(send, verbose = False, profile = False, debug = False): From 1e6bc1926ad0d8227abd26ad6c5c79d9221b33ab Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 11 Oct 2021 02:59:49 +0000 Subject: [PATCH 065/143] fixed minor push_pit bug Former-commit-id: 9603ca2be93bb8fcaec9552d037123ebbd6e24a3 --- src/cli/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/data.py b/src/cli/data.py index 95f5738..c223a79 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -194,7 +194,7 @@ def push_pit(client, competition, pit): for variable in pit: - push_team_pit_data(apikey, competition, variable, pit[variable]) + push_team_pit_data(client, competition, variable, pit[variable]) def check_new_database_matches(client, competition): From 6f28e507a1f62bb3efd8ed276e96ab61b3d9eca8 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 12 Oct 2021 21:42:09 +0000 Subject: [PATCH 066/143] fixed errorlog path Signed-off-by: Arthur Lu Former-commit-id: 0895eb3d747a808fb510dea5df6a67b0795a0fd2 --- src/cli/superscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 4774902..d2201e0 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -536,7 +536,7 @@ def start(pid_path, verbose = False, profile = False, debug = False): else: - f = open('errorlog.txt', 'w+') + f = open('errorlog.log', 'w+') with daemon.DaemonContext( working_directory=os.getcwd(), pidfile=pidfile.TimeoutPIDLockFile(pid_path), From 1151541976d1672129d9d8930d6a1598ba306933 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 12 Oct 2021 22:16:34 +0000 Subject: [PATCH 067/143] fixed key error in loop delay logic, improved error logging to include stack trace Signed-off-by: Arthur Lu Former-commit-id: 3f32104c725d484e737cbbd2f35a7d0a4639152e --- src/cli/superscript.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index d2201e0..e485f61 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -160,6 +160,7 @@ import pymongo import sys import threading import time +import traceback import warnings import websockets @@ -234,7 +235,6 @@ def main(send, verbose = False, profile = False, debug = False): client.close() warnings.filterwarnings("ignore") - sys.stderr = open("errorlog.log", "w") exit_code = 0 if verbose: @@ -315,7 +315,7 @@ def main(send, verbose = False, profile = False, debug = False): if profile: return # return instead of break to avoid sys.exit - event_delay = config["event-delay"] + event_delay = config["variable"]["event-delay"] if event_delay: send(stdout, INF, "loop delayed until database returns new matches") new_match = False @@ -324,7 +324,7 @@ def main(send, verbose = False, profile = False, debug = False): new_match = check_new_database_matches(client, competition) send(stdout, INF, "database returned new matches") else: - loop_delay = float(config["loop-delay"]) + loop_delay = float(config["variable"]["loop-delay"]) remaining_time = loop_delay - (time.time() - loop_start) if remaining_time > 0: send(stdout, INF, "loop delayed by " + str(remaining_time) + " seconds") @@ -334,17 +334,16 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "detected KeyboardInterrupt, killing threads") close_all() send(stdout, INF, "terminated threads, exiting") - loop_exit_code = 0 break except Exception as e: send(stderr, ERR, "encountered an exception while running", code = 1) - print(e, file = stderr) + traceback.print_exc(file = stderr) exit_code = 1 close_all() break - sys.exit(exit_code) + return exit_code def parse_config_persistent(send, config): @@ -520,19 +519,22 @@ def start(pid_path, verbose = False, profile = False, debug = False): import cProfile, pstats, io profile = cProfile.Profile() profile.enable() - main(send, profile = True) + exit_code = main(send, profile = True) profile.disable() f = open("profile.txt", 'w+') ps = pstats.Stats(profile, stream = f).sort_stats('cumtime') ps.print_stats() + sys.exit(exit_code) elif verbose: - main(log, verbose = verbose) + exit_code = main(log, verbose = verbose) + sys.exit(exit_code) elif debug: - main(log, verbose = True, profile = True, debug = debug) + exit_code = main(log, verbose = True, profile = True, debug = debug) + sys.exit(exit_code) else: @@ -571,7 +573,8 @@ def start(pid_path, verbose = False, profile = False, debug = False): asyncio.get_event_loop().run_until_complete(start_server) threading.Thread(target = asyncio.get_event_loop().run_forever).start() - main(send) + exit_code = main(send) + sys.exit(exit_code) def stop(pid_path): try: From e20a212bd997278b7529936b5a1861d57ba5fb78 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 13 Oct 2021 19:11:27 +0000 Subject: [PATCH 068/143] changes to stop() error reporting Signed-off-by: Arthur Lu Former-commit-id: bd77927421414660d3e7634d1ce70395552e2c96 --- src/cli/superscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index e485f61..b3610c8 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -595,7 +595,7 @@ def stop(pid_path): if os.path.exists(pid_path): os.remove(pid_path) else: - print(str(err)) + traceback.print_exc(file = stderr) sys.exit(1) def restart(pid_path): From 26d9d962c0dbdf976708f39c285100b984e85b4f Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 13 Oct 2021 22:32:42 +0000 Subject: [PATCH 069/143] added close_all when exiting due to error Former-commit-id: 9b614af85c93842f5a7f5882bd5337d8c6891b4f --- src/cli/superscript.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index b3610c8..0ef3395 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -252,6 +252,7 @@ def main(send, verbose = False, profile = False, debug = False): if load_config(config_path, config): send(stderr, ERR, "could not find config at <" + config_path + ">, generating blank config and exiting", code = 100) + close_all() exit_code = 1 break @@ -260,6 +261,7 @@ def main(send, verbose = False, profile = False, debug = False): flag, apikey, tbakey, preference, sync = parse_config_persistent(send, config) if flag: exit_code = 1 + close_all() break send(stdout, INF, "found and loaded database and tba keys") @@ -271,10 +273,12 @@ def main(send, verbose = False, profile = False, debug = False): resolve_config_conflicts(send, client, config, preference, sync) if config == 1: exit_code = 1 + close_all() break flag, exec_threads, competition, match_tests, metrics_tests, pit_tests = parse_config_variable(send, config) if flag: exit_code = 1 + close_all() break start = time.time() From a4a13c7cb511ce784140b3974e8f06cda6fedac0 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 14 Oct 2021 19:45:28 +0000 Subject: [PATCH 070/143] set return value for profile option Signed-off-by: Arthur Lu Former-commit-id: e15f87b2e5577a240d22498c24a20da3214db817 --- src/cli/superscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 0ef3395..3dd42cc 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -317,7 +317,7 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") if profile: - return # return instead of break to avoid sys.exit + return 0 # return instead of break to avoid sys.exit event_delay = config["variable"]["event-delay"] if event_delay: From 5d0fbc06c689974fffcb2c5fa786d651ab20eea2 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 14 Oct 2021 22:52:15 +0000 Subject: [PATCH 071/143] possible fix: UDP broadcast, not connecing from outside LAN Signed-off-by: Arthur Lu Former-commit-id: 531c8f80f36d2170bef582fbb6625377b58a7b2c --- src/cli/client.py | 19 +++++++++++++++++ src/cli/superscript.py | 46 ++++++++++++------------------------------ 2 files changed, 32 insertions(+), 33 deletions(-) create mode 100644 src/cli/client.py diff --git a/src/cli/client.py b/src/cli/client.py new file mode 100644 index 0000000..e3e7241 --- /dev/null +++ b/src/cli/client.py @@ -0,0 +1,19 @@ +import socket + +client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) # UDP + +# Enable port reusage so we will be able to run multiple clients and servers on single (host, port). +# Do not use socket.SO_REUSEADDR except you using linux(kernel<3.9): goto https://stackoverflow.com/questions/14388706/how-do-so-reuseaddr-and-so-reuseport-differ for more information. +# For linux hosts all sockets that want to share the same address and port combination must belong to processes that share the same effective user ID! +# So, on linux(kernel>=3.9) you have to run multiple servers and clients under one user to share the same (host, port). +# Thanks to @stevenreddie +client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + +# Enable broadcasting mode +client.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + +client.connect(("", 5678)) +while True: + # Thanks @seym45 for a fix + data, addr = client.recvfrom(1024) + print("received message: %s"%data) \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 3dd42cc..36432cb 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -151,18 +151,16 @@ __all__ = [ # imports: -import asyncio import json import math from multiprocessing import Pool, freeze_support import os import pymongo +import socket import sys -import threading import time import traceback import warnings -import websockets from interface import splash, log, ERR, INF, stdout, stderr from data import get_previous_time, pull_new_tba_matches, set_current_time, load_match, push_match, load_pit, push_pit, get_database_config, set_database_config, check_new_database_matches @@ -541,44 +539,26 @@ def start(pid_path, verbose = False, profile = False, debug = False): sys.exit(exit_code) else: - + f = open('errorlog.log', 'w+') with daemon.DaemonContext( - working_directory=os.getcwd(), - pidfile=pidfile.TimeoutPIDLockFile(pid_path), - stderr=f + working_directory = os.getcwd(), + pidfile = pidfile.TimeoutPIDLockFile(pid_path), + stderr = f ): - async def handler(client, path): - clients.append(client) - while True: - try: - pong_waiter = await client.ping() - await pong_waiter - time.sleep(3) - except Exception as e: - clients.remove(client) - break + server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + server.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + server.settimeout(0.2) - async def send_one(client, data): - await client.send(data) - def send(target, level, message, code = 0): - message_clients = clients.copy() - for client in message_clients: - try: - asyncio.run(send_one(client, message)) - except: - pass - - clients = [] - start_server = websockets.serve(handler, "0.0.0.0", 5678) - - asyncio.get_event_loop().run_until_complete(start_server) - threading.Thread(target = asyncio.get_event_loop().run_forever).start() + server.sendto(bytes(message, 'utf-8'), ('', 5678)) exit_code = main(send) - sys.exit(exit_code) + server.close() + f.close() + sys.exit() def stop(pid_path): try: From 8540642bef86a63c3a681c6bb9cf35b7696979c2 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 14 Oct 2021 23:30:15 +0000 Subject: [PATCH 072/143] works locally again Former-commit-id: 7b05707e3bf81f083805c13aed93e814f0373d30 --- src/cli/client.py | 2 +- src/cli/data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/client.py b/src/cli/client.py index e3e7241..e96620a 100644 --- a/src/cli/client.py +++ b/src/cli/client.py @@ -12,7 +12,7 @@ client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) # Enable broadcasting mode client.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) -client.connect(("", 5678)) +client.bind(("", 5678)) while True: # Thanks @seym45 for a fix data, addr = client.recvfrom(1024) diff --git a/src/cli/data.py b/src/cli/data.py index c223a79..c46e726 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -193,7 +193,7 @@ def push_metric(client, competition, metric): def push_pit(client, competition, pit): for variable in pit: - + push_team_pit_data(client, competition, variable, pit[variable]) def check_new_database_matches(client, competition): From 6cd092da37bd624213d8566568976692eb1ea1c8 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 15 Oct 2021 06:12:50 +0000 Subject: [PATCH 073/143] solution using zmq PUB/SUB, working locally Former-commit-id: c0c46a1b9deb5e43d6f3714a40440092ceb06e36 --- src/cli/client.py | 24 ++++++++++-------------- src/cli/superscript.py | 18 +++++++++++------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/cli/client.py b/src/cli/client.py index e96620a..bc4a15b 100644 --- a/src/cli/client.py +++ b/src/cli/client.py @@ -1,19 +1,15 @@ -import socket +import signal +import zmq -client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) # UDP -# Enable port reusage so we will be able to run multiple clients and servers on single (host, port). -# Do not use socket.SO_REUSEADDR except you using linux(kernel<3.9): goto https://stackoverflow.com/questions/14388706/how-do-so-reuseaddr-and-so-reuseport-differ for more information. -# For linux hosts all sockets that want to share the same address and port combination must belong to processes that share the same effective user ID! -# So, on linux(kernel>=3.9) you have to run multiple servers and clients under one user to share the same (host, port). -# Thanks to @stevenreddie -client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) +signal.signal(signal.SIGINT, signal.SIG_DFL) -# Enable broadcasting mode -client.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) +context = zmq.Context() + +socket = context.socket(zmq.SUB) +socket.connect('tcp://localhost:5678') +socket.setsockopt(zmq.SUBSCRIBE, b'status') -client.bind(("", 5678)) while True: - # Thanks @seym45 for a fix - data, addr = client.recvfrom(1024) - print("received message: %s"%data) \ No newline at end of file + message = socket.recv_multipart() + print(f'Received: {message}') \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 36432cb..371d18f 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -156,7 +156,6 @@ import math from multiprocessing import Pool, freeze_support import os import pymongo -import socket import sys import time import traceback @@ -314,6 +313,8 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "closed threads and database client") send(stdout, INF, "finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") + raise Exception("boop") + if profile: return 0 # return instead of break to avoid sys.exit @@ -547,16 +548,19 @@ def start(pid_path, verbose = False, profile = False, debug = False): stderr = f ): - server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - server.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - server.settimeout(0.2) + import zmq + + context = zmq.Context() + socket = context.socket(zmq.PUB) + socket.bind("tcp://*:5678") + + socket.send(b'status') def send(target, level, message, code = 0): - server.sendto(bytes(message, 'utf-8'), ('', 5678)) + socket.send(bytes("status: " + message, 'utf-8')) exit_code = main(send) - server.close() + socket.close() f.close() sys.exit() From 6bcb3cbff435482edbbce8758f3c6c13f4e29735 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 15 Oct 2021 06:20:05 +0000 Subject: [PATCH 074/143] verification of zmq solution working remotely Signed-off-by: Arthur Lu Former-commit-id: 41fb81cfac711a7af9f64dbbd1bc408158e058fd --- src/cli/superscript.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 371d18f..13ebc1b 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -160,6 +160,7 @@ import sys import time import traceback import warnings +import zmq from interface import splash, log, ERR, INF, stdout, stderr from data import get_previous_time, pull_new_tba_matches, set_current_time, load_match, push_match, load_pit, push_pit, get_database_config, set_database_config, check_new_database_matches @@ -313,8 +314,6 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "closed threads and database client") send(stdout, INF, "finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") - raise Exception("boop") - if profile: return 0 # return instead of break to avoid sys.exit @@ -548,8 +547,6 @@ def start(pid_path, verbose = False, profile = False, debug = False): stderr = f ): - import zmq - context = zmq.Context() socket = context.socket(zmq.PUB) socket.bind("tcp://*:5678") From dafd66447fd1fcde3606509b2537ff383f2a1138 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 16 Oct 2021 20:21:12 +0000 Subject: [PATCH 075/143] moved sample zmq SUB client to test/ Former-commit-id: ab13e10271d605dbbee9305d65526aaf7d0d1c07 --- {src/cli => test}/client.py | 1 - 1 file changed, 1 deletion(-) rename {src/cli => test}/client.py (88%) diff --git a/src/cli/client.py b/test/client.py similarity index 88% rename from src/cli/client.py rename to test/client.py index bc4a15b..df4c0e4 100644 --- a/src/cli/client.py +++ b/test/client.py @@ -1,7 +1,6 @@ import signal import zmq - signal.signal(signal.SIGINT, signal.SIG_DFL) context = zmq.Context() From 08f5ba987a3407f20f6048154b76b8fd2f77b3ea Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 16 Oct 2021 20:28:57 +0000 Subject: [PATCH 076/143] removed websockets from requirements.txr, fixed return code of daemon, removed websockets from hidden imports of superscript.spec Former-commit-id: 828e45844eaeeb70ed90b4cc85a2c114b20b6d5e --- src/cli/superscript.py | 2 +- src/cli/superscript.spec | 2 -- src/requirements.txt | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 13ebc1b..70f497f 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -559,7 +559,7 @@ def start(pid_path, verbose = False, profile = False, debug = False): exit_code = main(send) socket.close() f.close() - sys.exit() + sys.exit(exit_code) def stop(pid_path): try: diff --git a/src/cli/superscript.spec b/src/cli/superscript.spec index 7664ae0..04cafa7 100644 --- a/src/cli/superscript.spec +++ b/src/cli/superscript.spec @@ -10,8 +10,6 @@ a = Analysis(['superscript.py'], "dnspython", "sklearn.utils._weight_vector", "requests", - "websockets.legacy", - "websockets.legacy.server", ], hookspath=[], runtime_hooks=[], diff --git a/src/requirements.txt b/src/requirements.txt index 644ff65..88cc58b 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -15,5 +15,5 @@ pandas kivy==2.0.0rc2 -websockets +pyzmq python-daemon \ No newline at end of file From 36081e1239db9cc700754fec7b31f732d3a10481 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 20:05:07 +0000 Subject: [PATCH 077/143] removed keys Signed-off-by: Arthur Lu Former-commit-id: 4a1d7f0db1a0ddf7790ee94868ceeaf1c273d315 --- src/cli/module.py | 44 ++++++++++++++++++++++ src/cli/superscript.py | 4 +- src/cli/testing.ipynb | 83 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 src/cli/module.py create mode 100644 src/cli/testing.ipynb diff --git a/src/cli/module.py b/src/cli/module.py new file mode 100644 index 0000000..8f0e6d6 --- /dev/null +++ b/src/cli/module.py @@ -0,0 +1,44 @@ +import data as d +class Module: + config = None + data = None + results = None + def __init__(self, config, apikey, tbakey, timestamp): + pass + def validate_config(self): + pass + def load_data(self): + pass + def process_data(self, exec_threads): + pass + def push_results(self): + pass + +class Match: + config = None + apikey = None + tbakey = None + timestamp = None + teams = None + + data = None + results = None + + def __init__(self, config, apikey, tbakey, timestamp, teams): + self.config = config + self.apikey = apikey + self.tbakey = tbakey + self.timestamp = timestamp + self.teams = teams + + def validate_config(self): + return True + + def load_data(self): + pass + + def process_data(self, exec_threads): + pass + + def push_results(self): + pass \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 70f497f..0a139c8 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -170,8 +170,8 @@ config_path = "config.json" sample_json = """{ "persistent":{ "key":{ - "database":"mongodb+srv://analysis:MU2gPeEjEurRt2n@2022-scouting-4vfuu.mongodb.net/?retryWrites=true&w=majority", - "tba":"UDvKmPjPRfwwUdDX1JxbmkyecYBJhCtXeyVk9vmO2i7K0Zn4wqQPMfzuEINXJ7e5" + "database":"", + "tba":"" }, "config-preference":"local", "synchronize-config":false diff --git a/src/cli/testing.ipynb b/src/cli/testing.ipynb new file mode 100644 index 0000000..83cbfa6 --- /dev/null +++ b/src/cli/testing.ipynb @@ -0,0 +1,83 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "class C1:\n", + "\tval = 0\n", + "\tdef __init__(self, i):\n", + "\t\tself.val = i\n", + "\n", + "class C2:\n", + "\tval = 0\n", + "\tdef __init__(self, i):\n", + "\t\tself.val = 2*i\n", + "\n", + "class C3:\n", + "\tval = 0\n", + "\tdef __init__(self, i):\n", + "\t\tself.val = 3*i" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[<__main__.C1 at 0x7fa84827afd0>,\n", + " <__main__.C2 at 0x7fa84827ae80>,\n", + " <__main__.C3 at 0x7fa8482020d0>]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "classes = [C1, C2, C3]\n", + "initialized = []\n", + "for c in classes:\n", + "\tinitialized.append(c(10))\n", + "initialized" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" + }, + "kernelspec": { + "display_name": "Python 3.8.5 64-bit", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 07965cba4b3eec94fd810ae520942bfb441fd118 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 20:28:29 +0000 Subject: [PATCH 078/143] added working validate_config for Match module Signed-off-by: Arthur Lu Former-commit-id: b9e2de2dc673987ac514cc019196522017a3ba2e --- src/cli/module.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cli/module.py b/src/cli/module.py index 8f0e6d6..e1ed08d 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -32,7 +32,18 @@ class Match: self.teams = teams def validate_config(self): - return True + if self.config == None: + return "config cannot be empty" + elif self.apikey == None or self.apikey == "": + return "apikey cannot be empty" + elif self.tbakey == None or self.tbakey == "": + return "tbakey cannot be empty" + elif not(self.config["scope"] in ["competition", "season", "none"]): + return "scope must be one of: (competition, season, none)" + elif not(self.config["agglomeration"] in ["none", "mean"]): + return "agglomeration must be one of: (none, mean)" + else: + return None def load_data(self): pass From d2ced284ea3a2d1e085471cb777e50bfa68cfe6f Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 20:42:47 +0000 Subject: [PATCH 079/143] added working load_data for Match module Signed-off-by: Arthur Lu Former-commit-id: 8e8c19c5fd279a6a32e8379652edc5f00d36c3d7 --- src/cli/module.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index e1ed08d..518ebf7 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -1,4 +1,13 @@ import data as d + +class AutoVivification(dict): + def __getitem__(self, item): + try: + return dict.__getitem__(self, item) + except KeyError: + value = self[item] = type(self)() + return value + class Module: config = None data = None @@ -21,8 +30,7 @@ class Match: timestamp = None teams = None - data = None - results = None + data = [] def __init__(self, config, apikey, tbakey, timestamp, teams): self.config = config @@ -32,21 +40,30 @@ class Match: self.teams = teams def validate_config(self): + return True, "" + """ if self.config == None: - return "config cannot be empty" + return False, "config cannot be empty" elif self.apikey == None or self.apikey == "": - return "apikey cannot be empty" + return False, "apikey cannot be empty" elif self.tbakey == None or self.tbakey == "": - return "tbakey cannot be empty" + return False, "tbakey cannot be empty" elif not(self.config["scope"] in ["competition", "season", "none"]): - return "scope must be one of: (competition, season, none)" + return False, "scope must be one of: (competition, season, none)" elif not(self.config["agglomeration"] in ["none", "mean"]): - return "agglomeration must be one of: (none, mean)" + return False, "agglomeration must be one of: (none, mean)" else: - return None + return True, "" + """ def load_data(self): - pass + scope = self.config["scope"] + for team in self.teams: + competitions = d.get_team_conpetitions(self.apikey, team, scope) # unimplemented + for competition in competitions: + for variable in self.config["tests"]: + match_data = d.get_team_match_data(self.apikey, competition, team) # needs modified implementation + self.data.append((team, competition, variable, match_data)) def process_data(self, exec_threads): pass From 79dd2c3479763a02e70324eee1d5eabc37fcaed2 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 20:56:06 +0000 Subject: [PATCH 080/143] minor fix Signed-off-by: Arthur Lu Former-commit-id: 72a14bfac9a02e342eaa7091cd98abfb9a118aec --- src/cli/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/module.py b/src/cli/module.py index 518ebf7..6fdfb2e 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -62,7 +62,7 @@ class Match: competitions = d.get_team_conpetitions(self.apikey, team, scope) # unimplemented for competition in competitions: for variable in self.config["tests"]: - match_data = d.get_team_match_data(self.apikey, competition, team) # needs modified implementation + match_data = d.get_team_match_data(self.apikey, competition, team, variable) # needs modified implementation self.data.append((team, competition, variable, match_data)) def process_data(self, exec_threads): From 375b5501538cd154243499401758d9b11dac0b4e Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 21:27:35 +0000 Subject: [PATCH 081/143] finished rough outline for Match module Signed-off-by: Arthur Lu Former-commit-id: d5ebb0348bec81a3c7ce0a908c2ca5eb7ea57fd5 --- src/cli/module.py | 53 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index 6fdfb2e..e9d85a7 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -1,4 +1,7 @@ import data as d +import signal +import numpy as np +import tra_analysis as an class AutoVivification(dict): def __getitem__(self, item): @@ -31,6 +34,7 @@ class Match: teams = None data = [] + results = [] def __init__(self, config, apikey, tbakey, timestamp, teams): self.config = config @@ -63,10 +67,53 @@ class Match: for competition in competitions: for variable in self.config["tests"]: match_data = d.get_team_match_data(self.apikey, competition, team, variable) # needs modified implementation - self.data.append((team, competition, variable, match_data)) + variable_tests = self.config["tests"][variable] + self.data.append({"team": team, "competition": competition, "variable": variable, "tests": variable_tests, "data": match_data}) + + def tests(test_data): + signal.signal(signal.SIGINT, signal.SIG_IGN) + + if(test_data["data"] == None): + return None + + data = np.array(test_data["data"]) + data = data[np.isfinite(data)] + ranges = list(range(len(data))) + + tests = test_data["tests"] + + results = AutoVivification() + + if "basic_stats" in tests: + results["basic_stats"] = an.basic_stats(data) + if "historical_analysis" in tests: + results["historical_analysis"] = an.histo_analysis([ranges, data]) + if "regression_linear" in tests: + results["regression_linear"] = an.regression(ranges, data, ['lin']) + if "regression_logarithmic" in tests: + results["regression_logarithmic"] = an.regression(ranges, data, ['log']) + if "regression_exponential" in tests: + results["regression_exponential"] = an.regression(ranges, data, ['exp']) + if "regression_polynomial" in tests: + results["regression_polynomial"] = an.regression(ranges, data, ['ply']) + if "regression_sigmoidal" in tests: + results["regression_sigmoidal"] = an.regression(ranges, data, ['sig']) + + return results def process_data(self, exec_threads): - pass + self.results = list(exec_threads.map(self.tests, self.data)) def push_results(self): - pass \ No newline at end of file + short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} + results_short = AutoVivification() + i = 0 + for result in self.results: + for variable in result: + if variable in short_mapping: + short = short_mapping[variable] + else: + short = variable + d.push_team_match_results(self.data[i]["team"], self.data[i]["competition"], self.data[i]["variable"], short, result[variable]) + #results_short[ self.data["team"] ][ self.data["competition"] ][ self.data["variable"] ][short] = result[variable] + i+=1 \ No newline at end of file From 66bf3a3a3ef70b0b47e088939b889fd0114a276a Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 21:29:44 +0000 Subject: [PATCH 082/143] removed autovivification (likely not required) Signed-off-by: Arthur Lu Former-commit-id: f40939d69d0c1441cd1b2867e51481af19bf88e9 --- src/cli/module.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index e9d85a7..2910daa 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -3,14 +3,6 @@ import signal import numpy as np import tra_analysis as an -class AutoVivification(dict): - def __getitem__(self, item): - try: - return dict.__getitem__(self, item) - except KeyError: - value = self[item] = type(self)() - return value - class Module: config = None data = None @@ -82,7 +74,7 @@ class Match: tests = test_data["tests"] - results = AutoVivification() + results = {} if "basic_stats" in tests: results["basic_stats"] = an.basic_stats(data) @@ -106,7 +98,6 @@ class Match: def push_results(self): short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} - results_short = AutoVivification() i = 0 for result in self.results: for variable in result: @@ -114,6 +105,5 @@ class Match: short = short_mapping[variable] else: short = variable - d.push_team_match_results(self.data[i]["team"], self.data[i]["competition"], self.data[i]["variable"], short, result[variable]) - #results_short[ self.data["team"] ][ self.data["competition"] ][ self.data["variable"] ][short] = result[variable] + d.push_team_match_results(self.data[i]["team"], self.data[i]["competition"], self.data[i]["variable"], short, result[variable]) # needs implementation i+=1 \ No newline at end of file From 88282aa18bf6c60967b47f1e23a4082d22e5b00b Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 21:51:14 +0000 Subject: [PATCH 083/143] modified config validation for Match module Signed-off-by: Arthur Lu Former-commit-id: de81a03e3da925bbcdc4e3691be95c4d0b772e80 --- src/cli/module.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index 2910daa..555c12b 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -36,7 +36,6 @@ class Match: self.teams = teams def validate_config(self): - return True, "" """ if self.config == None: return False, "config cannot be empty" @@ -46,8 +45,10 @@ class Match: return False, "tbakey cannot be empty" elif not(self.config["scope"] in ["competition", "season", "none"]): return False, "scope must be one of: (competition, season, none)" - elif not(self.config["agglomeration"] in ["none", "mean"]): - return False, "agglomeration must be one of: (none, mean)" + elif self.config["agglomeration"] != "none": + return False, "agglomeration must be 'none', there are currently no supported Agglomeration methods" + elif self.config["tests"] == None: + return False, "tests must not be None, it may be empty {}" else: return True, "" """ From 4ded02b34f3e9432ac7f4eede893f6e455baa2d4 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 21:56:44 +0000 Subject: [PATCH 084/143] added sample initializaition loop Signed-off-by: Arthur Lu Former-commit-id: d16fe61a79601e70e0b1fd2134f31c5310e16207 --- src/cli/testing.ipynb | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/cli/testing.ipynb b/src/cli/testing.ipynb index 83cbfa6..904b8ac 100644 --- a/src/cli/testing.ipynb +++ b/src/cli/testing.ipynb @@ -48,6 +48,39 @@ "initialized" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import module\n", + "module_references = {\"match\": module.Match}\n", + "module_initialized = []\n", + "for m in [\"match\"]:\n", + "\tmodule_initialized.append(module_references[m](\"\", \"\", \"\", 0, []))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "module_initialized" + ] + }, { "cell_type": "code", "execution_count": null, From a1fb295a1a394399eddf1b8812d60c0ca5d047bf Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 21:58:02 +0000 Subject: [PATCH 085/143] minor fix Signed-off-by: Arthur Lu Former-commit-id: 7f0d3fc7539314521b2a0d1c714bd5234fb20e21 --- src/cli/module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli/module.py b/src/cli/module.py index 555c12b..e394167 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -36,6 +36,7 @@ class Match: self.teams = teams def validate_config(self): + return True, "" """ if self.config == None: return False, "config cannot be empty" From f8f8543ea27ef2b3835dbe4ae91b9f7a053e6bf7 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 21 Oct 2021 22:33:26 +0000 Subject: [PATCH 086/143] changed sample json Former-commit-id: 8966900341508677b1b78d0ac3bd6b1ae6eae9c2 --- src/cli/superscript.py | 80 +++++++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 0a139c8..56d484c 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -177,48 +177,64 @@ sample_json = """{ "synchronize-config":false }, "variable":{ + "max-threads":0.5, + + "competition":"", "team":"", - "competition": "2020ilch", - "statistics":{ + + "event-delay":false, + "loop-delay":0, + "reportable":true, + + "teams":[], + + "modules":{ + "match":{ - "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] + "tests":{ + "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] + } }, + "metric":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 - }, - "ts":{ - "mu":25, - "sigma":8.33 + "tests":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } } }, + "pit":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true + "tests":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true + } } - }, - "event-delay":false, - "loop-delay":60 + } } }""" From 0eccc320960d2cb8922474779b97adc29fa017d9 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 22 Oct 2021 01:40:56 +0000 Subject: [PATCH 087/143] changed Match module to use current data bindings Signed-off-by: Arthur Lu Former-commit-id: afab67f8ef21ca066576eed5f1b1c5feb3fb2c2f --- src/cli/module.py | 133 ++++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 58 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index e394167..4021989 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -23,89 +23,106 @@ class Match: apikey = None tbakey = None timestamp = None - teams = None + competition = None data = [] results = [] - def __init__(self, config, apikey, tbakey, timestamp, teams): + def __init__(self, config, apikey, tbakey, timestamp, competition): self.config = config self.apikey = apikey self.tbakey = tbakey self.timestamp = timestamp - self.teams = teams + self.competition = competition def validate_config(self): return True, "" - """ - if self.config == None: - return False, "config cannot be empty" - elif self.apikey == None or self.apikey == "": - return False, "apikey cannot be empty" - elif self.tbakey == None or self.tbakey == "": - return False, "tbakey cannot be empty" - elif not(self.config["scope"] in ["competition", "season", "none"]): - return False, "scope must be one of: (competition, season, none)" - elif self.config["agglomeration"] != "none": - return False, "agglomeration must be 'none', there are currently no supported Agglomeration methods" - elif self.config["tests"] == None: - return False, "tests must not be None, it may be empty {}" - else: - return True, "" - """ def load_data(self): - scope = self.config["scope"] - for team in self.teams: - competitions = d.get_team_conpetitions(self.apikey, team, scope) # unimplemented - for competition in competitions: - for variable in self.config["tests"]: - match_data = d.get_team_match_data(self.apikey, competition, team, variable) # needs modified implementation - variable_tests = self.config["tests"][variable] - self.data.append({"team": team, "competition": competition, "variable": variable, "tests": variable_tests, "data": match_data}) + self.data = d.load_match(self.apikey, self.competition) + + def simplestats(data_test): - def tests(test_data): signal.signal(signal.SIGINT, signal.SIG_IGN) - if(test_data["data"] == None): - return None - - data = np.array(test_data["data"]) + data = np.array(data_test[3]) data = data[np.isfinite(data)] ranges = list(range(len(data))) - tests = test_data["tests"] + test = data_test[2] - results = {} + if test == "basic_stats": + return an.basic_stats(data) - if "basic_stats" in tests: - results["basic_stats"] = an.basic_stats(data) - if "historical_analysis" in tests: - results["historical_analysis"] = an.histo_analysis([ranges, data]) - if "regression_linear" in tests: - results["regression_linear"] = an.regression(ranges, data, ['lin']) - if "regression_logarithmic" in tests: - results["regression_logarithmic"] = an.regression(ranges, data, ['log']) - if "regression_exponential" in tests: - results["regression_exponential"] = an.regression(ranges, data, ['exp']) - if "regression_polynomial" in tests: - results["regression_polynomial"] = an.regression(ranges, data, ['ply']) - if "regression_sigmoidal" in tests: - results["regression_sigmoidal"] = an.regression(ranges, data, ['sig']) + if test == "historical_analysis": + return an.histo_analysis([ranges, data]) - return results + if test == "regression_linear": + return an.regression(ranges, data, ['lin']) + + if test == "regression_logarithmic": + return an.regression(ranges, data, ['log']) + + if test == "regression_exponential": + return an.regression(ranges, data, ['exp']) + + if test == "regression_polynomial": + return an.regression(ranges, data, ['ply']) + + if test == "regression_sigmoidal": + return an.regression(ranges, data, ['sig']) def process_data(self, exec_threads): - self.results = list(exec_threads.map(self.tests, self.data)) + + tests = self.config["tests"] + data = self.data + + input_vector = [] + + for team in data: + + for variable in data[team]: + + if variable in tests: + + for test in tests[variable]: + + input_vector.append((team, variable, test, data[team][variable])) + + self.data = input_vector + self.results = list(exec_threads.map(self.simplestats, self.data)) def push_results(self): + short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} + + class AutoVivification(dict): + def __getitem__(self, item): + try: + return dict.__getitem__(self, item) + except KeyError: + value = self[item] = type(self)() + return value + + result_filtered = self.results + input_vector = self.data + + return_vector = AutoVivification() + i = 0 - for result in self.results: - for variable in result: - if variable in short_mapping: - short = short_mapping[variable] - else: - short = variable - d.push_team_match_results(self.data[i]["team"], self.data[i]["competition"], self.data[i]["variable"], short, result[variable]) # needs implementation - i+=1 \ No newline at end of file + + for result in result_filtered: + + filtered = input_vector[i][2] + + try: + short = short_mapping[filtered] + return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result[short] + except KeyError: # not in mapping + return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result + + i += 1 + + self.results = return_vector + + d.push_match(self.apikey, self.competition, self.results) \ No newline at end of file From 6c8e738420612b08f3e3c394ca2db4c3d73abc33 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 28 Oct 2021 20:13:57 +0000 Subject: [PATCH 088/143] added Metric, Pit modules (theoretically working) Signed-off-by: Arthur Lu Former-commit-id: 3ac4e96d2dd63cc0c2061a78e6e15e26a4538e8d --- src/cli/module.py | 165 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 162 insertions(+), 3 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index 4021989..5d0fcbf 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -19,14 +19,15 @@ class Module: pass class Match: + config = None apikey = None tbakey = None timestamp = None competition = None - data = [] - results = [] + data = None + results = None def __init__(self, config, apikey, tbakey, timestamp, competition): self.config = config @@ -125,4 +126,162 @@ class Match: self.results = return_vector - d.push_match(self.apikey, self.competition, self.results) \ No newline at end of file + d.push_match(self.apikey, self.competition, self.results) + +class Metric: + + config = None + apikey = None + tbakey = None + timestamp = None + competition = None + + data = None + results = None + + def __init__(self, config, apikey, tbakey, timestamp, competition): + self.config = config + self.apikey = apikey + self.tbakey = tbakey + self.timestamp = timestamp + self.competition = competition + + def validate_config(self): + return True, "" + + def load_data(self): + self.data = d.pull_new_tba_matches(self.apikey, self.competition, self.timestamp) + + def process_data(self, exec_threads): + + elo_N = self.config["elo"]["N"] + elo_K = self.config["elo"]["K"] + + matches = self.data + + red = {} + blu = {} + + for match in matches: + + red = d.load_metric(self.apikey, self.competition, match, "red", self.config) + blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config) + + elo_red_total = 0 + elo_blu_total = 0 + + gl2_red_score_total = 0 + gl2_blu_score_total = 0 + + gl2_red_rd_total = 0 + gl2_blu_rd_total = 0 + + gl2_red_vol_total = 0 + gl2_blu_vol_total = 0 + + for team in red: + + elo_red_total += red[team]["elo"]["score"] + + gl2_red_score_total += red[team]["gl2"]["score"] + gl2_red_rd_total += red[team]["gl2"]["rd"] + gl2_red_vol_total += red[team]["gl2"]["vol"] + + for team in blu: + + elo_blu_total += blu[team]["elo"]["score"] + + gl2_blu_score_total += blu[team]["gl2"]["score"] + gl2_blu_rd_total += blu[team]["gl2"]["rd"] + gl2_blu_vol_total += blu[team]["gl2"]["vol"] + + red_elo = {"score": elo_red_total / len(red)} + blu_elo = {"score": elo_blu_total / len(blu)} + + red_gl2 = {"score": gl2_red_score_total / len(red), "rd": gl2_red_rd_total / len(red), "vol": gl2_red_vol_total / len(red)} + blu_gl2 = {"score": gl2_blu_score_total / len(blu), "rd": gl2_blu_rd_total / len(blu), "vol": gl2_blu_vol_total / len(blu)} + + + if match["winner"] == "red": + + observations = {"red": 1, "blu": 0} + + elif match["winner"] == "blue": + + observations = {"red": 0, "blu": 1} + + else: + + observations = {"red": 0.5, "blu": 0.5} + + red_elo_delta = an.Metric().elo(red_elo["score"], blu_elo["score"], observations["red"], elo_N, elo_K) - red_elo["score"] + blu_elo_delta = an.Metric().elo(blu_elo["score"], red_elo["score"], observations["blu"], elo_N, elo_K) - blu_elo["score"] + + new_red_gl2_score, new_red_gl2_rd, new_red_gl2_vol = an.Metric().glicko2(red_gl2["score"], red_gl2["rd"], red_gl2["vol"], [blu_gl2["score"]], [blu_gl2["rd"]], [observations["red"], observations["blu"]]) + new_blu_gl2_score, new_blu_gl2_rd, new_blu_gl2_vol = an.Metric().glicko2(blu_gl2["score"], blu_gl2["rd"], blu_gl2["vol"], [red_gl2["score"]], [red_gl2["rd"]], [observations["blu"], observations["red"]]) + + red_gl2_delta = {"score": new_red_gl2_score - red_gl2["score"], "rd": new_red_gl2_rd - red_gl2["rd"], "vol": new_red_gl2_vol - red_gl2["vol"]} + blu_gl2_delta = {"score": new_blu_gl2_score - blu_gl2["score"], "rd": new_blu_gl2_rd - blu_gl2["rd"], "vol": new_blu_gl2_vol - blu_gl2["vol"]} + + for team in red: + + red[team]["elo"]["score"] = red[team]["elo"]["score"] + red_elo_delta + + red[team]["gl2"]["score"] = red[team]["gl2"]["score"] + red_gl2_delta["score"] + red[team]["gl2"]["rd"] = red[team]["gl2"]["rd"] + red_gl2_delta["rd"] + red[team]["gl2"]["vol"] = red[team]["gl2"]["vol"] + red_gl2_delta["vol"] + + for team in blu: + + blu[team]["elo"]["score"] = blu[team]["elo"]["score"] + blu_elo_delta + + blu[team]["gl2"]["score"] = blu[team]["gl2"]["score"] + blu_gl2_delta["score"] + blu[team]["gl2"]["rd"] = blu[team]["gl2"]["rd"] + blu_gl2_delta["rd"] + blu[team]["gl2"]["vol"] = blu[team]["gl2"]["vol"] + blu_gl2_delta["vol"] + + temp_vector = {} + temp_vector.update(red) + temp_vector.update(blu) + + d.push_metric(self.client, self.competition, temp_vector) + + def push_results(self): + pass + +class Pit: + + config = None + apikey = None + tbakey = None + timestamp = None + competition = None + + data = None + results = None + + def __init__(self, config, apikey, tbakey, timestamp, competition): + self.config = config + self.apikey = apikey + self.tbakey = tbakey + self.timestamp = timestamp + self.competition = competition + + def validate_config(self): + return True, "" + + def load_data(self): + self.data = d.load_pit(self.apikey, self.competition) + + def process_data(self, exec_threads): + return_vector = {} + for team in self.data: + for variable in self.data[team]: + if variable in self.config: + if not variable in return_vector: + return_vector[variable] = [] + return_vector[variable].append(self.data[team][variable]) + + self.results = return_vector + + def push_results(self): + d.push_pit(self.apikey, self.competition, self.results) \ No newline at end of file From 0af93238bee8317f6e1c7e02034ee692a4ce2f09 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 28 Oct 2021 20:40:56 +0000 Subject: [PATCH 089/143] implemented dynamic module loading/running, fixed minor issue in metric module Signed-off-by: Arthur Lu Former-commit-id: 7be48af85eed7bc5544671144cc83b4c9ce8649c --- src/cli/module.py | 10 +++---- src/cli/superscript.py | 65 ++++++++++++++---------------------------- 2 files changed, 27 insertions(+), 48 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index 5d0fcbf..dfc9a16 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -150,12 +150,12 @@ class Metric: return True, "" def load_data(self): - self.data = d.pull_new_tba_matches(self.apikey, self.competition, self.timestamp) + self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp) def process_data(self, exec_threads): - elo_N = self.config["elo"]["N"] - elo_K = self.config["elo"]["K"] + elo_N = self.config["tests"]["elo"]["N"] + elo_K = self.config["tests"]["elo"]["K"] matches = self.data @@ -164,8 +164,8 @@ class Metric: for match in matches: - red = d.load_metric(self.apikey, self.competition, match, "red", self.config) - blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config) + red = d.load_metric(self.apikey, self.competition, match, "red", self.config["tests"]) + blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config["tests"]) elo_red_total = 0 elo_blu_total = 0 diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 56d484c..7a75f09 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -163,8 +163,9 @@ import warnings import zmq from interface import splash, log, ERR, INF, stdout, stderr -from data import get_previous_time, pull_new_tba_matches, set_current_time, load_match, push_match, load_pit, push_pit, get_database_config, set_database_config, check_new_database_matches -from processing import matchloop, metricloop, pitloop +from data import get_previous_time, set_current_time, get_database_config, set_database_config, check_new_database_matches +from module import Match, Metric, Pit +#from processing import matchloop, metricloop, pitloop config_path = "config.json" sample_json = """{ @@ -254,6 +255,8 @@ def main(send, verbose = False, profile = False, debug = False): if verbose: splash(__version__) + modules = {"match": Match, "metric": Metric, "pit": Pit} + while True: try: @@ -289,31 +292,23 @@ def main(send, verbose = False, profile = False, debug = False): exit_code = 1 close_all() break - flag, exec_threads, competition, match_tests, metrics_tests, pit_tests = parse_config_variable(send, config) + flag, exec_threads, competition, config_modules = parse_config_variable(send, config) if flag: exit_code = 1 close_all() break - start = time.time() - send(stdout, INF, "loading match, metric, pit data (this may take a few seconds)") - match_data = load_match(client, competition) - metrics_data = pull_new_tba_matches(tbakey, competition, loop_start) - pit_data = load_pit(client, competition) - send(stdout, INF, "finished loading match, metric, pit data in "+ str(time.time() - start) + " seconds") - - start = time.time() - send(stdout, INF, "performing analysis on match, metrics, pit data") - match_results = matchloop(client, competition, match_data, match_tests, exec_threads) - metrics_results = metricloop(client, competition, metrics_data, metrics_tests) - pit_results = pitloop(client, competition, pit_data, pit_tests) - send(stdout, INF, "finished analysis in " + str(time.time() - start) + " seconds") - - start = time.time() - send(stdout, INF, "uploading match, metrics, pit results to database") - push_match(client, competition, match_results) - push_pit(client, competition, pit_results) - send(stdout, INF, "finished uploading results in " + str(time.time() - start) + " seconds") + for m in config_modules: + if m in modules: + start = time.time() + current_module = modules[m](config_modules[m], client, tbakey, loop_start, competition) + valid = current_module.validate_config() + if not valid: + continue + current_module.load_data() + current_module.process_data(exec_threads) + current_module.push_results() + print(m + " module finished in " + str(time.time() - start) + " seconds") if debug: f = open("matchloop.log", "w+") @@ -439,37 +434,21 @@ def parse_config_variable(send, config): send(stderr, ERR, "could not find competition field in config", code = 101) exit_flag = True try: - match_tests = config["variable"]["statistics"]["match"] + modules = config["variable"]["modules"] except: - send(stderr, ERR, "could not find match field in config", code = 102) - exit_flag = True - try: - metrics_tests = config["variable"]["statistics"]["metric"] - except: - send(stderr, ERR, "could not find metrics field in config", code = 103) - exit_flag = True - try: - pit_tests = config["variable"]["statistics"]["pit"] - except: - send(stderr, ERR, "could not find pit field in config", code = 104) + send(stderr, ERR, "could not find modules field in config", code = 102) exit_flag = True if competition == None or competition == "": send(stderr, ERR, "competition field in config must not be empty", code = 105) exit_flag = True - if match_tests == None: - send(stderr, ERR, "matchfield in config must not be empty", code = 106) - exit_flag = True - if metrics_tests == None: - send(stderr, ERR, "metrics field in config must not be empty", code = 107) - exit_flag = True - if pit_tests == None: - send(stderr, ERR, "pit field in config must not be empty", code = 108) + if modules == None: + send(stderr, ERR, "modules in config must not be empty", code = 106) exit_flag = True send(stdout, INF, "found and loaded competition, match, metrics, pit from config") - return exit_flag, exec_threads, competition, match_tests, metrics_tests, pit_tests + return exit_flag, exec_threads, competition, modules def resolve_config_conflicts(send, client, config, preference, sync): From 9ddf61621e9f0b84db48566c13c55e108b7b82b6 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 28 Oct 2021 21:12:28 +0000 Subject: [PATCH 090/143] fixed debug logging Signed-off-by: Arthur Lu Former-commit-id: bd87111e08709e825af2abad0d94894274a354c6 --- src/cli/superscript.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 7a75f09..6df57f9 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -308,16 +308,11 @@ def main(send, verbose = False, profile = False, debug = False): current_module.load_data() current_module.process_data(exec_threads) current_module.push_results() - print(m + " module finished in " + str(time.time() - start) + " seconds") - - if debug: - f = open("matchloop.log", "w+") - json.dump(match_results, f, ensure_ascii=False, indent=4) - f.close() - - f = open("pitloop.log", "w+") - json.dump(pit_results, f, ensure_ascii=False, indent=4) - f.close() + send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds") + if debug: + f = open(m + ".log", "w+") + json.dump({"data": current_module.data, "results":current_module.results}, f, ensure_ascii=False, indent=4) + f.close() set_current_time(client, loop_start) close_all() From bd0c0d99f96668fd7b48171da9a1378f05f3dcdb Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 30 Oct 2021 03:11:10 +0000 Subject: [PATCH 091/143] module replaces processing Former-commit-id: d798bfe7d2ba3369aef5d6ed2a3f3c0b1a66b482 --- src/cli/processing.py | 188 ----------------------------------------- src/cli/superscript.py | 1 - 2 files changed, 189 deletions(-) delete mode 100644 src/cli/processing.py diff --git a/src/cli/processing.py b/src/cli/processing.py deleted file mode 100644 index aa867f6..0000000 --- a/src/cli/processing.py +++ /dev/null @@ -1,188 +0,0 @@ -import numpy as np - -from tra_analysis import Analysis as an -from data import pull_new_tba_matches, push_metric, load_metric - -import signal - -def simplestats(data_test): - - signal.signal(signal.SIGINT, signal.SIG_IGN) - - data = np.array(data_test[3]) - data = data[np.isfinite(data)] - ranges = list(range(len(data))) - - test = data_test[2] - - if test == "basic_stats": - return an.basic_stats(data) - - if test == "historical_analysis": - return an.histo_analysis([ranges, data]) - - if test == "regression_linear": - return an.regression(ranges, data, ['lin']) - - if test == "regression_logarithmic": - return an.regression(ranges, data, ['log']) - - if test == "regression_exponential": - return an.regression(ranges, data, ['exp']) - - if test == "regression_polynomial": - return an.regression(ranges, data, ['ply']) - - if test == "regression_sigmoidal": - return an.regression(ranges, data, ['sig']) - -def matchloop(client, competition, data, tests, exec_threads): - - short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} - - class AutoVivification(dict): - def __getitem__(self, item): - try: - return dict.__getitem__(self, item) - except KeyError: - value = self[item] = type(self)() - return value - - input_vector = [] - return_vector = AutoVivification() - - for team in data: - - for variable in data[team]: - - if variable in tests: - - for test in tests[variable]: - - input_vector.append((team, variable, test, data[team][variable])) - - result_filtered = exec_threads.map(simplestats, input_vector) - - i = 0 - - result_filtered = list(result_filtered) - - for result in result_filtered: - - filtered = input_vector[i][2] - - try: - short = short_mapping[filtered] - return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result[short] - except KeyError: # not in mapping - return_vector[input_vector[i][0]][input_vector[i][1]][input_vector[i][2]] = result - - i += 1 - - return return_vector - -def metricloop(client, competition, data, metrics): # listener based metrics update - - elo_N = metrics["elo"]["N"] - elo_K = metrics["elo"]["K"] - - matches = data - #matches = pull_new_tba_matches(tbakey, competition, timestamp) - - red = {} - blu = {} - - for match in matches: - - red = load_metric(client, competition, match, "red", metrics) - blu = load_metric(client, competition, match, "blue", metrics) - - elo_red_total = 0 - elo_blu_total = 0 - - gl2_red_score_total = 0 - gl2_blu_score_total = 0 - - gl2_red_rd_total = 0 - gl2_blu_rd_total = 0 - - gl2_red_vol_total = 0 - gl2_blu_vol_total = 0 - - for team in red: - - elo_red_total += red[team]["elo"]["score"] - - gl2_red_score_total += red[team]["gl2"]["score"] - gl2_red_rd_total += red[team]["gl2"]["rd"] - gl2_red_vol_total += red[team]["gl2"]["vol"] - - for team in blu: - - elo_blu_total += blu[team]["elo"]["score"] - - gl2_blu_score_total += blu[team]["gl2"]["score"] - gl2_blu_rd_total += blu[team]["gl2"]["rd"] - gl2_blu_vol_total += blu[team]["gl2"]["vol"] - - red_elo = {"score": elo_red_total / len(red)} - blu_elo = {"score": elo_blu_total / len(blu)} - - red_gl2 = {"score": gl2_red_score_total / len(red), "rd": gl2_red_rd_total / len(red), "vol": gl2_red_vol_total / len(red)} - blu_gl2 = {"score": gl2_blu_score_total / len(blu), "rd": gl2_blu_rd_total / len(blu), "vol": gl2_blu_vol_total / len(blu)} - - - if match["winner"] == "red": - - observations = {"red": 1, "blu": 0} - - elif match["winner"] == "blue": - - observations = {"red": 0, "blu": 1} - - else: - - observations = {"red": 0.5, "blu": 0.5} - - red_elo_delta = an.Metric().elo(red_elo["score"], blu_elo["score"], observations["red"], elo_N, elo_K) - red_elo["score"] - blu_elo_delta = an.Metric().elo(blu_elo["score"], red_elo["score"], observations["blu"], elo_N, elo_K) - blu_elo["score"] - - new_red_gl2_score, new_red_gl2_rd, new_red_gl2_vol = an.Metric().glicko2(red_gl2["score"], red_gl2["rd"], red_gl2["vol"], [blu_gl2["score"]], [blu_gl2["rd"]], [observations["red"], observations["blu"]]) - new_blu_gl2_score, new_blu_gl2_rd, new_blu_gl2_vol = an.Metric().glicko2(blu_gl2["score"], blu_gl2["rd"], blu_gl2["vol"], [red_gl2["score"]], [red_gl2["rd"]], [observations["blu"], observations["red"]]) - - red_gl2_delta = {"score": new_red_gl2_score - red_gl2["score"], "rd": new_red_gl2_rd - red_gl2["rd"], "vol": new_red_gl2_vol - red_gl2["vol"]} - blu_gl2_delta = {"score": new_blu_gl2_score - blu_gl2["score"], "rd": new_blu_gl2_rd - blu_gl2["rd"], "vol": new_blu_gl2_vol - blu_gl2["vol"]} - - for team in red: - - red[team]["elo"]["score"] = red[team]["elo"]["score"] + red_elo_delta - - red[team]["gl2"]["score"] = red[team]["gl2"]["score"] + red_gl2_delta["score"] - red[team]["gl2"]["rd"] = red[team]["gl2"]["rd"] + red_gl2_delta["rd"] - red[team]["gl2"]["vol"] = red[team]["gl2"]["vol"] + red_gl2_delta["vol"] - - for team in blu: - - blu[team]["elo"]["score"] = blu[team]["elo"]["score"] + blu_elo_delta - - blu[team]["gl2"]["score"] = blu[team]["gl2"]["score"] + blu_gl2_delta["score"] - blu[team]["gl2"]["rd"] = blu[team]["gl2"]["rd"] + blu_gl2_delta["rd"] - blu[team]["gl2"]["vol"] = blu[team]["gl2"]["vol"] + blu_gl2_delta["vol"] - - temp_vector = {} - temp_vector.update(red) - temp_vector.update(blu) - - push_metric(client, competition, temp_vector) - -def pitloop(client, competition, pit, tests): - - return_vector = {} - for team in pit: - for variable in pit[team]: - if variable in tests: - if not variable in return_vector: - return_vector[variable] = [] - return_vector[variable].append(pit[team][variable]) - - return return_vector \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 6df57f9..84b8e32 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -165,7 +165,6 @@ import zmq from interface import splash, log, ERR, INF, stdout, stderr from data import get_previous_time, set_current_time, get_database_config, set_database_config, check_new_database_matches from module import Match, Metric, Pit -#from processing import matchloop, metricloop, pitloop config_path = "config.json" sample_json = """{ From 9c2946718f821283b2b576c5510d593427540acf Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 30 Oct 2021 03:51:03 +0000 Subject: [PATCH 092/143] removed testing notebook, anticipated removal of pandas requirement, added pandas and matplotlib to pyinstaller exclusion, removed unessasary option in build scripts Former-commit-id: 0fd24d42e12a16fc929945cc5cdd6745ce023171 --- build/build-CLI.bat | 2 +- build/build-CLI.sh | 2 +- src/cli/data.py | 1 - src/cli/superscript.spec | 5 +- src/cli/testing.ipynb | 116 --------------------------------------- src/requirements.txt | 2 - 6 files changed, 6 insertions(+), 122 deletions(-) delete mode 100644 src/cli/testing.ipynb diff --git a/build/build-CLI.bat b/build/build-CLI.bat index 22dd93c..29199e2 100644 --- a/build/build-CLI.bat +++ b/build/build-CLI.bat @@ -2,4 +2,4 @@ set pathtospec="../src/cli/superscript.spec" set pathtodist="../dist/" set pathtowork="temp/" -pyinstaller --onefile --clean --distpath %pathtodist% --workpath %pathtowork% %pathtospec% \ No newline at end of file +pyinstaller --clean --distpath %pathtodist% --workpath %pathtowork% %pathtospec% \ No newline at end of file diff --git a/build/build-CLI.sh b/build/build-CLI.sh index 302265b..e328718 100644 --- a/build/build-CLI.sh +++ b/build/build-CLI.sh @@ -2,4 +2,4 @@ pathtospec="../src/cli/superscript.spec" pathtodist="../dist/" pathtowork="temp/" -pyinstaller --onefile --clean --distpath ${pathtodist} --workpath ${pathtowork} ${pathtospec} \ No newline at end of file +pyinstaller --clean --distpath ${pathtodist} --workpath ${pathtowork} ${pathtospec} \ No newline at end of file diff --git a/src/cli/data.py b/src/cli/data.py index c46e726..1d9c056 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -1,5 +1,4 @@ import requests -import pandas as pd def pull_new_tba_matches(apikey, competition, cutoff): api_key= apikey diff --git a/src/cli/superscript.spec b/src/cli/superscript.spec index 04cafa7..5d09620 100644 --- a/src/cli/superscript.spec +++ b/src/cli/superscript.spec @@ -13,7 +13,10 @@ a = Analysis(['superscript.py'], ], hookspath=[], runtime_hooks=[], - excludes=[], + excludes=[ + "matplotlib", + "pandas" + ], win_no_prefer_redirects=False, win_private_assemblies=False, cipher=block_cipher, diff --git a/src/cli/testing.ipynb b/src/cli/testing.ipynb deleted file mode 100644 index 904b8ac..0000000 --- a/src/cli/testing.ipynb +++ /dev/null @@ -1,116 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "class C1:\n", - "\tval = 0\n", - "\tdef __init__(self, i):\n", - "\t\tself.val = i\n", - "\n", - "class C2:\n", - "\tval = 0\n", - "\tdef __init__(self, i):\n", - "\t\tself.val = 2*i\n", - "\n", - "class C3:\n", - "\tval = 0\n", - "\tdef __init__(self, i):\n", - "\t\tself.val = 3*i" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[<__main__.C1 at 0x7fa84827afd0>,\n", - " <__main__.C2 at 0x7fa84827ae80>,\n", - " <__main__.C3 at 0x7fa8482020d0>]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "classes = [C1, C2, C3]\n", - "initialized = []\n", - "for c in classes:\n", - "\tinitialized.append(c(10))\n", - "initialized" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import module\n", - "module_references = {\"match\": module.Match}\n", - "module_initialized = []\n", - "for m in [\"match\"]:\n", - "\tmodule_initialized.append(module_references[m](\"\", \"\", \"\", 0, []))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "module_initialized" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "interpreter": { - "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" - }, - "kernelspec": { - "display_name": "Python 3.8.5 64-bit", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/requirements.txt b/src/requirements.txt index 88cc58b..52854df 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,6 +1,5 @@ requests pymongo -pandas tra-analysis dnspython @@ -11,7 +10,6 @@ scipy scikit-learn six pyparsing -pandas kivy==2.0.0rc2 From 60e88a59874f7fad8c3b54a55cd027ba3d97ee06 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 2 Nov 2021 05:37:27 +0000 Subject: [PATCH 093/143] formalized module interface Former-commit-id: 108284a94b2e63c3fc10b02b9f60739f4ccbbaba --- src/cli/module.py | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index dfc9a16..233e1aa 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -1,24 +1,37 @@ +import abc import data as d import signal import numpy as np import tra_analysis as an -class Module: - config = None - data = None - results = None - def __init__(self, config, apikey, tbakey, timestamp): - pass - def validate_config(self): - pass - def load_data(self): - pass - def process_data(self, exec_threads): - pass - def push_results(self): - pass +class Module(metaclass = abc.ABCMeta): -class Match: + @classmethod + def __subclasshook__(cls, subclass): + return (hasattr(subclass, 'validate_config') and + callable(subclass.validate_config) and + hasattr(subclass, 'load_data') and + callable(subclass.load_data) and + hasattr(subclass, 'process_data') and + callable(subclass.process_data) and + hasattr(subclass, 'push_results') and + callable(subclass.push_results) + ) + + @abc.abstractmethod + def validate_config(self): + raise NotImplementedError + @abc.abstractmethod + def load_data(self): + raise NotImplementedError + @abc.abstractmethod + def process_data(self, exec_threads): + raise NotImplementedError + @abc.abstractmethod + def push_results(self): + raise NotImplementedError + +class Match (Module): config = None apikey = None @@ -128,7 +141,7 @@ class Match: d.push_match(self.apikey, self.competition, self.results) -class Metric: +class Metric (Module): config = None apikey = None @@ -248,7 +261,7 @@ class Metric: def push_results(self): pass -class Pit: +class Pit (Module): config = None apikey = None From c90a35ff51088c4a1fd2deb7db5eed8384d18ff4 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 5 Nov 2021 23:17:42 +0000 Subject: [PATCH 094/143] added empty modules for future additions Former-commit-id: d234302cdc26cce03833877133f25a144d7d1f58 --- src/cli/module.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/cli/module.py b/src/cli/module.py index 233e1aa..b3cac88 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -297,4 +297,13 @@ class Pit (Module): self.results = return_vector def push_results(self): - d.push_pit(self.apikey, self.competition, self.results) \ No newline at end of file + d.push_pit(self.apikey, self.competition, self.results) + +class Rating (Module): + pass + +class Heatmap (Module): + pass + +class Sentiment (Module): + pass \ No newline at end of file From b9610b7b46b6e8a1ab0c8c6118d5349648f49232 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 6 Nov 2021 00:04:11 +0000 Subject: [PATCH 095/143] improved and streamlined config error handling Former-commit-id: 99efefd3025f99ca65cf73dd694baa86a2a65e73 --- src/cli/superscript.py | 117 +++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 70 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 84b8e32..755b3b5 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -166,6 +166,12 @@ from interface import splash, log, ERR, INF, stdout, stderr from data import get_previous_time, set_current_time, get_database_config, set_database_config, check_new_database_matches from module import Match, Metric, Pit +class ConfigurationError (Exception): + code = None + def __init__(self, str, code): + super().__init__(str) + self.code = code + config_path = "config.json" sample_json = """{ "persistent":{ @@ -247,6 +253,8 @@ def main(send, verbose = False, profile = False, debug = False): exec_threads.close() if "client" in locals(): client.close() + if "f" in locals(): + f.close() warnings.filterwarnings("ignore") exit_code = 0 @@ -267,18 +275,12 @@ def main(send, verbose = False, profile = False, debug = False): config = {} if load_config(config_path, config): - send(stderr, ERR, "could not find config at <" + config_path + ">, generating blank config and exiting", code = 100) - close_all() - exit_code = 1 - break + raise ConfigurationError("could not find config at <" + config_path + ">, generating blank config and exiting", 110) send(stdout, INF, "found and loaded config at <" + config_path + ">") - flag, apikey, tbakey, preference, sync = parse_config_persistent(send, config) - if flag: - exit_code = 1 - close_all() - break + apikey, tbakey, preference, sync = parse_config_persistent(send, config) + send(stdout, INF, "found and loaded database and tba keys") client = pymongo.MongoClient(apikey) @@ -286,16 +288,9 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "established connection to database") send(stdout, INF, "analysis backtimed to: " + str(get_previous_time(client))) - resolve_config_conflicts(send, client, config, preference, sync) - if config == 1: - exit_code = 1 - close_all() - break - flag, exec_threads, competition, config_modules = parse_config_variable(send, config) - if flag: - exit_code = 1 - close_all() - break + config = resolve_config_conflicts(send, client, config, preference, sync) + + exec_threads, competition, config_modules = parse_config_variable(send, config) for m in config_modules: if m in modules: @@ -320,7 +315,8 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") if profile: - return 0 # return instead of break to avoid sys.exit + exit_code = 0 + break event_delay = config["variable"]["event-delay"] if event_delay: @@ -343,6 +339,13 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "terminated threads, exiting") break + except ConfigurationError as e: + send(stderr, ERR, "encountered a configuration error: " + str(e), code = e.code) + traceback.print_exc(file = stderr) + exit_code = 1 + close_all() + break + except Exception as e: send(stderr, ERR, "encountered an exception while running", code = 1) traceback.print_exc(file = stderr) @@ -354,54 +357,41 @@ def main(send, verbose = False, profile = False, debug = False): def parse_config_persistent(send, config): - exit_flag = False try: apikey = config["persistent"]["key"]["database"] except: - send(stderr, ERR, "database key field in config must be present", code = 111) - exit_flag = True + raise ConfigurationError("persistent/key/database field is invalid or missing", 111) try: tbakey = config["persistent"]["key"]["tba"] except: - send(stderr, ERR, "tba key field in config must be present", code = 112) - exit_flag = True + raise ConfigurationError("persistent/key/tba field is invalid or missing", 112) try: preference = config["persistent"]["config-preference"] except: - send(stderr, ERR, "config-preference field in config must be present", code = 113) - exit_flag = True + raise ConfigurationError("persistent/config-preference field is invalid or missing", 113) try: sync = config["persistent"]["synchronize-config"] except: - send(stderr, ERR, "synchronize-config field in config must be present", code = 114) - exit_flag = True + raise ConfigurationError("persistent/synchronize-config field is invalid or missing", 114) if apikey == None or apikey == "": - send(stderr, ERR, "database key field in config must not be empty, please populate the database key", code = 115) - exit_flag = True + raise ConfigurationError("persistent/key/database field is empty", 115) if tbakey == None or tbakey == "": - send(stderr, ERR, "tba key field in config must not be empty, please populate the tba key", code = 116) - exit_flag = True + raise ConfigurationError("persistent/key/tba field is empty", 116) if preference == None or preference == "": - send(stderr, ERR, "config-preference field in config must not be empty, please populate config-preference", code = 117) - exit_flag = True + raise ConfigurationError("persistent/config-preference field is empty", 117) if sync != True and sync != False: - send(stderr, ERR, "synchronize-config field in config must be a boolean, please populate synchronize-config", code = 118) - exit_flag = True + raise ConfigurationError("persistent/synchronize-config field is empty", 118) - return exit_flag, apikey, tbakey, preference, sync + return apikey, tbakey, preference, sync def parse_config_variable(send, config): - exit_flag = False - sys_max_threads = os.cpu_count() try: cfg_max_threads = config["variable"]["max-threads"] except: - send(stderr, ERR, "max-threads field in config must not be empty, refer to documentation for configuration options", code = 109) - exit_flag = True - + raise ConfigurationError("variable/max-threads field is invalid or missing, refer to documentation for configuration options", 109) if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : alloc_processes = sys_max_threads + cfg_max_threads elif cfg_max_threads > 0 and cfg_max_threads < 1: @@ -411,38 +401,31 @@ def parse_config_variable(send, config): elif cfg_max_threads == 0: alloc_processes = sys_max_threads else: - send(stderr, ERR, "max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, code = 110) - exit_flag = True - + raise ConfigurationError("variable/max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, 110) try: exec_threads = Pool(processes = alloc_processes) except Exception as e: - send(stderr, ERR, "unable to start threads", code = 200) send(stderr, INF, e) - exit_flag = True + raise ConfigurationError("unable to start threads", 200) send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") try: competition = config["variable"]["competition"] except: - send(stderr, ERR, "could not find competition field in config", code = 101) - exit_flag = True + raise ConfigurationError("variable/competition field is invalid or missing", 101) try: modules = config["variable"]["modules"] except: - send(stderr, ERR, "could not find modules field in config", code = 102) - exit_flag = True + raise ConfigurationError("variable/modules field is invalid or missing", 102) if competition == None or competition == "": - send(stderr, ERR, "competition field in config must not be empty", code = 105) - exit_flag = True + raise ConfigurationError("variable/competition field is empty", 105) if modules == None: - send(stderr, ERR, "modules in config must not be empty", code = 106) - exit_flag = True + raise ConfigurationError("variable/modules field is empty", 106) send(stdout, INF, "found and loaded competition, match, metrics, pit from config") - return exit_flag, exec_threads, competition, modules + return exec_threads, competition, modules def resolve_config_conflicts(send, client, config, preference, sync): @@ -453,34 +436,28 @@ def resolve_config_conflicts(send, client, config, preference, sync): if remote_config != config["variable"]: set_database_config(client, config["variable"]) send(stdout, INF, "database config was different and was updated") - return + return config elif preference == "remote" or preference == "database": send(stdout, INF, "config-preference set to remote/database, loading remote config information") remote_config= get_database_config(client) if remote_config != config["variable"]: config["variable"] = remote_config if save_config(config_path, config): - send(stderr, ERR, "local config was different but could not be updated") - config = 1 - return + raise ConfigurationError("local config was different but could not be updated", 121) send(stdout, INF, "local config was different and was updated") - return + return config else: - send(stderr, ERR, "config-preference field in config must be \"local\"/\"client\" or \"remote\"/\"database\"") - config = 1 - return + raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) else: if preference == "local" or preference == "client": send(stdout, INF, "config-preference set to local/client, loading local config information") - return + return config elif preference == "remote" or preference == "database": send(stdout, INF, "config-preference set to remote/database, loading database config information") config["variable"] = get_database_config(client) - return + return config else: - send(stderr, ERR, "config-preference field in config must be \"local\"/\"client\" or \"remote\"/\"database\"") - config = 1 - return + raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) def load_config(path, config_vector): try: From 4b5db1aba86ec295b61a6a6588e245407e67739b Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 6 Nov 2021 01:50:55 +0000 Subject: [PATCH 096/143] moved config parsing functions to config.py Former-commit-id: a251fef0242410685ad6d6daaa1ad4d5287b3d06 --- src/cli/config.py | 208 +++++++++++++++++++++++++++++++++++++++++ src/cli/superscript.py | 204 +--------------------------------------- 2 files changed, 211 insertions(+), 201 deletions(-) create mode 100644 src/cli/config.py diff --git a/src/cli/config.py b/src/cli/config.py new file mode 100644 index 0000000..ee2b122 --- /dev/null +++ b/src/cli/config.py @@ -0,0 +1,208 @@ +import math +import json +from multiprocessing import Pool +import os + +from data import set_database_config, get_database_config +from interface import stderr, stdout, INF, ERR + +config_path = "config.json" + +sample_json = """{ + "persistent":{ + "key":{ + "database":"", + "tba":"" + }, + "config-preference":"local", + "synchronize-config":false + }, + "variable":{ + + "max-threads":0.5, + + "competition":"", + "team":"", + + "event-delay":false, + "loop-delay":0, + "reportable":true, + + "teams":[], + + "modules":{ + + "match":{ + "tests":{ + "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], + "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] + } + + }, + + "metric":{ + "tests":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + } + }, + + "pit":{ + "tests":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true + } + } + } + } +}""" + +class ConfigurationError (Exception): + code = None + def __init__(self, str, code): + super().__init__(str) + self.code = code + +def parse_config_persistent(send, config): + + try: + apikey = config["persistent"]["key"]["database"] + except: + raise ConfigurationError("persistent/key/database field is invalid or missing", 111) + try: + tbakey = config["persistent"]["key"]["tba"] + except: + raise ConfigurationError("persistent/key/tba field is invalid or missing", 112) + try: + preference = config["persistent"]["config-preference"] + except: + raise ConfigurationError("persistent/config-preference field is invalid or missing", 113) + try: + sync = config["persistent"]["synchronize-config"] + except: + raise ConfigurationError("persistent/synchronize-config field is invalid or missing", 114) + + if apikey == None or apikey == "": + raise ConfigurationError("persistent/key/database field is empty", 115) + if tbakey == None or tbakey == "": + raise ConfigurationError("persistent/key/tba field is empty", 116) + if preference == None or preference == "": + raise ConfigurationError("persistent/config-preference field is empty", 117) + if sync != True and sync != False: + raise ConfigurationError("persistent/synchronize-config field is empty", 118) + + return apikey, tbakey, preference, sync + +def parse_config_variable(send, config): + + sys_max_threads = os.cpu_count() + try: + cfg_max_threads = config["variable"]["max-threads"] + except: + raise ConfigurationError("variable/max-threads field is invalid or missing, refer to documentation for configuration options", 109) + if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : + alloc_processes = sys_max_threads + cfg_max_threads + elif cfg_max_threads > 0 and cfg_max_threads < 1: + alloc_processes = math.floor(cfg_max_threads * sys_max_threads) + elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: + alloc_processes = cfg_max_threads + elif cfg_max_threads == 0: + alloc_processes = sys_max_threads + else: + raise ConfigurationError("variable/max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, 110) + try: + exec_threads = Pool(processes = alloc_processes) + except Exception as e: + send(stderr, INF, e) + raise ConfigurationError("unable to start threads", 200) + send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") + + try: + competition = config["variable"]["competition"] + except: + raise ConfigurationError("variable/competition field is invalid or missing", 101) + try: + modules = config["variable"]["modules"] + except: + raise ConfigurationError("variable/modules field is invalid or missing", 102) + + if competition == None or competition == "": + raise ConfigurationError("variable/competition field is empty", 105) + if modules == None: + raise ConfigurationError("variable/modules field is empty", 106) + + send(stdout, INF, "found and loaded competition, match, metrics, pit from config") + + return exec_threads, competition, modules + +def resolve_config_conflicts(send, client, config, preference, sync): + + if sync: + if preference == "local" or preference == "client": + send(stdout, INF, "config-preference set to local/client, loading local config information") + remote_config = get_database_config(client) + if remote_config != config["variable"]: + set_database_config(client, config["variable"]) + send(stdout, INF, "database config was different and was updated") + return config + elif preference == "remote" or preference == "database": + send(stdout, INF, "config-preference set to remote/database, loading remote config information") + remote_config= get_database_config(client) + if remote_config != config["variable"]: + config["variable"] = remote_config + if save_config(config_path, config): + raise ConfigurationError("local config was different but could not be updated", 121) + send(stdout, INF, "local config was different and was updated") + return config + else: + raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) + else: + if preference == "local" or preference == "client": + send(stdout, INF, "config-preference set to local/client, loading local config information") + return config + elif preference == "remote" or preference == "database": + send(stdout, INF, "config-preference set to remote/database, loading database config information") + config["variable"] = get_database_config(client) + return config + else: + raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) + +def load_config(path, config_vector): + try: + f = open(path, "r") + config_vector.update(json.load(f)) + f.close() + return 0 + except: + f = open(path, "w") + f.write(sample_json) + f.close() + return 1 + +def save_config(path, config_vector): + f = open(path, "w+") + json.dump(config_vector, f, ensure_ascii=False, indent=4) + f.close() + return 0 \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 755b3b5..e3f138f 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -152,8 +152,7 @@ __all__ = [ # imports: import json -import math -from multiprocessing import Pool, freeze_support +from multiprocessing import freeze_support import os import pymongo import sys @@ -162,87 +161,12 @@ import traceback import warnings import zmq +from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError +from data import get_previous_time, set_current_time, check_new_database_matches from interface import splash, log, ERR, INF, stdout, stderr -from data import get_previous_time, set_current_time, get_database_config, set_database_config, check_new_database_matches from module import Match, Metric, Pit -class ConfigurationError (Exception): - code = None - def __init__(self, str, code): - super().__init__(str) - self.code = code - config_path = "config.json" -sample_json = """{ - "persistent":{ - "key":{ - "database":"", - "tba":"" - }, - "config-preference":"local", - "synchronize-config":false - }, - "variable":{ - - "max-threads":0.5, - - "competition":"", - "team":"", - - "event-delay":false, - "loop-delay":0, - "reportable":true, - - "teams":[], - - "modules":{ - - "match":{ - "tests":{ - "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] - } - - }, - - "metric":{ - "tests":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 - }, - "ts":{ - "mu":25, - "sigma":8.33 - } - } - }, - - "pit":{ - "tests":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true - } - } - } - } -}""" def main(send, verbose = False, profile = False, debug = False): @@ -355,128 +279,6 @@ def main(send, verbose = False, profile = False, debug = False): return exit_code -def parse_config_persistent(send, config): - - try: - apikey = config["persistent"]["key"]["database"] - except: - raise ConfigurationError("persistent/key/database field is invalid or missing", 111) - try: - tbakey = config["persistent"]["key"]["tba"] - except: - raise ConfigurationError("persistent/key/tba field is invalid or missing", 112) - try: - preference = config["persistent"]["config-preference"] - except: - raise ConfigurationError("persistent/config-preference field is invalid or missing", 113) - try: - sync = config["persistent"]["synchronize-config"] - except: - raise ConfigurationError("persistent/synchronize-config field is invalid or missing", 114) - - if apikey == None or apikey == "": - raise ConfigurationError("persistent/key/database field is empty", 115) - if tbakey == None or tbakey == "": - raise ConfigurationError("persistent/key/tba field is empty", 116) - if preference == None or preference == "": - raise ConfigurationError("persistent/config-preference field is empty", 117) - if sync != True and sync != False: - raise ConfigurationError("persistent/synchronize-config field is empty", 118) - - return apikey, tbakey, preference, sync - -def parse_config_variable(send, config): - - sys_max_threads = os.cpu_count() - try: - cfg_max_threads = config["variable"]["max-threads"] - except: - raise ConfigurationError("variable/max-threads field is invalid or missing, refer to documentation for configuration options", 109) - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - raise ConfigurationError("variable/max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, 110) - try: - exec_threads = Pool(processes = alloc_processes) - except Exception as e: - send(stderr, INF, e) - raise ConfigurationError("unable to start threads", 200) - send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") - - try: - competition = config["variable"]["competition"] - except: - raise ConfigurationError("variable/competition field is invalid or missing", 101) - try: - modules = config["variable"]["modules"] - except: - raise ConfigurationError("variable/modules field is invalid or missing", 102) - - if competition == None or competition == "": - raise ConfigurationError("variable/competition field is empty", 105) - if modules == None: - raise ConfigurationError("variable/modules field is empty", 106) - - send(stdout, INF, "found and loaded competition, match, metrics, pit from config") - - return exec_threads, competition, modules - -def resolve_config_conflicts(send, client, config, preference, sync): - - if sync: - if preference == "local" or preference == "client": - send(stdout, INF, "config-preference set to local/client, loading local config information") - remote_config = get_database_config(client) - if remote_config != config["variable"]: - set_database_config(client, config["variable"]) - send(stdout, INF, "database config was different and was updated") - return config - elif preference == "remote" or preference == "database": - send(stdout, INF, "config-preference set to remote/database, loading remote config information") - remote_config= get_database_config(client) - if remote_config != config["variable"]: - config["variable"] = remote_config - if save_config(config_path, config): - raise ConfigurationError("local config was different but could not be updated", 121) - send(stdout, INF, "local config was different and was updated") - return config - else: - raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) - else: - if preference == "local" or preference == "client": - send(stdout, INF, "config-preference set to local/client, loading local config information") - return config - elif preference == "remote" or preference == "database": - send(stdout, INF, "config-preference set to remote/database, loading database config information") - config["variable"] = get_database_config(client) - return config - else: - raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) - -def load_config(path, config_vector): - try: - f = open(path, "r") - config_vector.update(json.load(f)) - f.close() - return 0 - except: - f = open(path, "w") - f.write(sample_json) - f.close() - return 1 - -def save_config(path, config_vector): - f = open(path, "w+") - json.dump(config_vector, f, ensure_ascii=False, indent=4) - f.close() - return 0 - def start(pid_path, verbose = False, profile = False, debug = False): if profile: From ac66545226d5d25d7df8e80f2a4e3c2e12215fb4 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 23 Nov 2021 22:23:59 +0000 Subject: [PATCH 097/143] changed Module interface to 3 functions Signed-off-by: Arthur Lu Former-commit-id: 356b71be62ff0c9ca786bc53f0df91d637b802a5 --- src/cli/module.py | 63 +++++++++++++++++++++++++++--------------- src/cli/superscript.py | 4 +-- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index b3cac88..3c89fc4 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -8,20 +8,24 @@ class Module(metaclass = abc.ABCMeta): @classmethod def __subclasshook__(cls, subclass): - return (hasattr(subclass, 'validate_config') and - callable(subclass.validate_config) and - hasattr(subclass, 'load_data') and - callable(subclass.load_data) and - hasattr(subclass, 'process_data') and - callable(subclass.process_data) and - hasattr(subclass, 'push_results') and - callable(subclass.push_results) + return (hasattr(subclass, '__init__') and + callable(subclass.__init__) and + hasattr(subclass, 'validate_config') and + callable(subclass.validate_config) and + hasattr(subclass, 'run') and + callable(subclass.run) ) - @abc.abstractmethod - def validate_config(self): + def __init__(self, config, apikey, tbakey, timestamp, competition, *args, **kwargs): raise NotImplementedError @abc.abstractmethod + def validate_config(self, *args, **kwargs): + raise NotImplementedError + @abc.abstractmethod + def run(self, exec_threads, *args, **kwargs): + raise NotImplementedError + """ + @abc.abstractmethod def load_data(self): raise NotImplementedError @abc.abstractmethod @@ -29,7 +33,7 @@ class Module(metaclass = abc.ABCMeta): raise NotImplementedError @abc.abstractmethod def push_results(self): - raise NotImplementedError + raise NotImplementedError""" class Match (Module): @@ -52,10 +56,15 @@ class Match (Module): def validate_config(self): return True, "" - def load_data(self): + def run(self, exec_threads): + self._load_data() + self._process_data(exec_threads) + self._push_results() + + def _load_data(self): self.data = d.load_match(self.apikey, self.competition) - def simplestats(data_test): + def _simplestats(data_test): signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -86,7 +95,7 @@ class Match (Module): if test == "regression_sigmoidal": return an.regression(ranges, data, ['sig']) - def process_data(self, exec_threads): + def _process_data(self, exec_threads): tests = self.config["tests"] data = self.data @@ -104,9 +113,9 @@ class Match (Module): input_vector.append((team, variable, test, data[team][variable])) self.data = input_vector - self.results = list(exec_threads.map(self.simplestats, self.data)) + self.results = list(exec_threads.map(self._simplestats, self.data)) - def push_results(self): + def _push_results(self): short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} @@ -162,10 +171,15 @@ class Metric (Module): def validate_config(self): return True, "" - def load_data(self): + def run(self, exec_threads): + self._load_data() + self._process_data(exec_threads) + self._push_results() + + def _load_data(self): self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp) - def process_data(self, exec_threads): + def _process_data(self, exec_threads): elo_N = self.config["tests"]["elo"]["N"] elo_K = self.config["tests"]["elo"]["K"] @@ -258,7 +272,7 @@ class Metric (Module): d.push_metric(self.client, self.competition, temp_vector) - def push_results(self): + def _push_results(self): pass class Pit (Module): @@ -282,10 +296,15 @@ class Pit (Module): def validate_config(self): return True, "" - def load_data(self): + def run(self, exec_threads): + self._load_data() + self._process_data(exec_threads) + self._push_results() + + def _load_data(self): self.data = d.load_pit(self.apikey, self.competition) - def process_data(self, exec_threads): + def _process_data(self, exec_threads): return_vector = {} for team in self.data: for variable in self.data[team]: @@ -296,7 +315,7 @@ class Pit (Module): self.results = return_vector - def push_results(self): + def _push_results(self): d.push_pit(self.apikey, self.competition, self.results) class Rating (Module): diff --git a/src/cli/superscript.py b/src/cli/superscript.py index e3f138f..064a371 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -223,9 +223,7 @@ def main(send, verbose = False, profile = False, debug = False): valid = current_module.validate_config() if not valid: continue - current_module.load_data() - current_module.process_data(exec_threads) - current_module.push_results() + current_module.run(exec_threads) send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds") if debug: f = open(m + ".log", "w+") From af74a69579953ae5161ac161c810fd5f6e95b25c Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 24 Nov 2021 02:27:03 +0000 Subject: [PATCH 098/143] removed commented out code Signed-off-by: Arthur Lu Former-commit-id: 3669712bec78402e948d67b7b236466a0368b260 --- src/cli/module.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/cli/module.py b/src/cli/module.py index 3c89fc4..e6ef29a 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -24,16 +24,6 @@ class Module(metaclass = abc.ABCMeta): @abc.abstractmethod def run(self, exec_threads, *args, **kwargs): raise NotImplementedError - """ - @abc.abstractmethod - def load_data(self): - raise NotImplementedError - @abc.abstractmethod - def process_data(self, exec_threads): - raise NotImplementedError - @abc.abstractmethod - def push_results(self): - raise NotImplementedError""" class Match (Module): From b4c9ef22b6d957119f39699b694a1511acfb84ff Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 9 Jan 2022 22:19:50 -0600 Subject: [PATCH 099/143] pull some data from API, fix some datatype compats Former-commit-id: 03ec0dd8fd4d1540a5061b00b3758a866d5a24ce --- src/cli/config.py | 202 +++++++++++++++++++++++++---------------- src/cli/data.py | 22 ++--- src/cli/exceptions.py | 5 + src/cli/pull.py | 66 ++++++++++++++ src/cli/superscript.py | 6 +- 5 files changed, 209 insertions(+), 92 deletions(-) create mode 100644 src/cli/exceptions.py create mode 100644 src/cli/pull.py diff --git a/src/cli/config.py b/src/cli/config.py index ee2b122..f837133 100644 --- a/src/cli/config.py +++ b/src/cli/config.py @@ -8,76 +8,130 @@ from interface import stderr, stdout, INF, ERR config_path = "config.json" -sample_json = """{ - "persistent":{ - "key":{ - "database":"", - "tba":"" - }, - "config-preference":"local", - "synchronize-config":false - }, - "variable":{ - - "max-threads":0.5, - - "competition":"", - "team":"", - - "event-delay":false, - "loop-delay":0, - "reportable":true, - - "teams":[], - - "modules":{ - - "match":{ - "tests":{ - "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] - } - - }, - - "metric":{ - "tests":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 - }, - "ts":{ - "mu":25, - "sigma":8.33 - } - } - }, - - "pit":{ - "tests":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true - } - } - } - } -}""" +sample_json = """ +{ + "persistent":{ + "key":{ + "database":"", + "tba":"", + "tra":{ + "CLIENT_ID":"", + "CLIENT_SECRET":"" + } + }, + "config-preference":"local", + "synchronize-config":false + }, + "variable":{ + "max-threads":0.5, + "team":"", + "event-delay":false, + "loop-delay":0, + "reportable":true, + "teams":[ + + ], + "modules":{ + "match":{ + "tests":{ + "balls-blocked":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-collected":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-started":[ + "basic_stats", + "historical_analyss", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ] + } + }, + "metric":{ + "tests":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + } + }, + "pit":{ + "tests":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true + } + } + } + } +} +""" class ConfigurationError (Exception): code = None @@ -139,23 +193,17 @@ def parse_config_variable(send, config): raise ConfigurationError("unable to start threads", 200) send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") - try: - competition = config["variable"]["competition"] - except: - raise ConfigurationError("variable/competition field is invalid or missing", 101) try: modules = config["variable"]["modules"] except: raise ConfigurationError("variable/modules field is invalid or missing", 102) - if competition == None or competition == "": - raise ConfigurationError("variable/competition field is empty", 105) if modules == None: raise ConfigurationError("variable/modules field is empty", 106) send(stdout, INF, "found and loaded competition, match, metrics, pit from config") - return exec_threads, competition, modules + return exec_threads, modules def resolve_config_conflicts(send, client, config, preference, sync): diff --git a/src/cli/data.py b/src/cli/data.py index 1d9c056..d262bd3 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -1,4 +1,6 @@ import requests +import pull +import pandas as pd def pull_new_tba_matches(apikey, competition, cutoff): api_key= apikey @@ -13,7 +15,7 @@ def get_team_match_data(client, competition, team_num): db = client.data_scouting mdata = db.matchdata out = {} - for i in mdata.find({"competition" : competition, "team_scouted": team_num}): + for i in mdata.find({"competition" : competition, "team_scouted": str(team_num)}): out[i['match']] = i['data'] return pd.DataFrame(out) @@ -21,7 +23,7 @@ def get_team_pit_data(client, competition, team_num): db = client.data_scouting mdata = db.pitdata out = {} - return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"] + return mdata.find_one({"competition" : competition, "team_scouted": str(team_num)})["data"] def get_team_metrics_data(client, competition, team_num): db = client.data_processing @@ -29,25 +31,21 @@ def get_team_metrics_data(client, competition, team_num): return mdata.find_one({"competition" : competition, "team": team_num}) def get_match_data_formatted(client, competition): - db = client.data_scouting - mdata = db.teamlist - x=mdata.find_one({"competition":competition}) + teams_at_comp = pull.get_teams_at_competition(competition) out = {} - for i in x: + for team in teams_at_comp: try: - out[int(i)] = unkeyify_2l(get_team_match_data(client, competition, int(i)).transpose().to_dict()) + out[int(team)] = unkeyify_2l(get_team_match_data(client, competition, team).transpose().to_dict()) except: pass return out def get_metrics_data_formatted(client, competition): - db = client.data_scouting - mdata = db.teamlist - x=mdata.find_one({"competition":competition}) + teams_at_comp = pull.get_teams_at_competition(competition) out = {} - for i in x: + for team in teams_at_comp: try: - out[int(i)] = get_team_metrics_data(client, competition, int(i)) + out[int(team)] = get_team_metrics_data(client, competition, int(team)) except: pass return out diff --git a/src/cli/exceptions.py b/src/cli/exceptions.py new file mode 100644 index 0000000..891668b --- /dev/null +++ b/src/cli/exceptions.py @@ -0,0 +1,5 @@ +class APIError(Exception): + code = None + def __init__(self, str, endpoint): + super().__init__(str) + self.endpoint = endpoint \ No newline at end of file diff --git a/src/cli/pull.py b/src/cli/pull.py new file mode 100644 index 0000000..db2772f --- /dev/null +++ b/src/cli/pull.py @@ -0,0 +1,66 @@ +import requests +import json +from exceptions import APIError + +def load_config(path): + with open(path, "r") as f: + return json.load(f) + +url = "https://titanscouting.epochml.org" +config_tra = load_config("config.json") +trakey = config_tra['persistent']['key']['tra'] + +def get_team_competition(): + endpoint = '/api/fetchTeamCompetition' + params = { + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return json['competition'] + else: + raise APIError(json, endpoint) + +def get_team(): + endpoint = '/api/fetchTeamCompetition' + params = { + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return json['team'] + else: + raise APIError(json, endpoint) + +def get_team_match_data(competition, team_num): + endpoint = '/api/fetchAllTeamMatchData' + params = { + "competition": competition, + "teamScouted": team_num, + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return json['data'][team_num] + else: + raise APIError(json, endpoint) + +def get_teams_at_competition(competition): + endpoint = '/api/fetchAllTeamNicknamesAtCompetition' + params = { + "competition": competition, + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return list(json['data'].keys()) + else: + raise APIError(json, endpoint) \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 064a371..a362e4a 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -160,7 +160,7 @@ import time import traceback import warnings import zmq - +import pull from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError from data import get_previous_time, set_current_time, check_new_database_matches from interface import splash, log, ERR, INF, stdout, stderr @@ -214,8 +214,8 @@ def main(send, verbose = False, profile = False, debug = False): config = resolve_config_conflicts(send, client, config, preference, sync) - exec_threads, competition, config_modules = parse_config_variable(send, config) - + exec_threads, config_modules = parse_config_variable(send, config) + competition = pull.get_team_competition() for m in config_modules: if m in modules: start = time.time() From 74542c1d8fb13e0c69f3c5f545a4821459b6d7b1 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 10 Jan 2022 09:21:46 -0600 Subject: [PATCH 100/143] add validation schema for persistent keys Former-commit-id: ceaf3f289170ec9ec6810856d965ce1495caeeb1 --- src/cli/config.py | 40 ++++++++++++++-------------------- src/cli/validation-schema.json | 26 ++++++++++++++++++++++ src/requirements.txt | 4 +++- 3 files changed, 45 insertions(+), 25 deletions(-) create mode 100644 src/cli/validation-schema.json diff --git a/src/cli/config.py b/src/cli/config.py index f837133..3160c4b 100644 --- a/src/cli/config.py +++ b/src/cli/config.py @@ -2,6 +2,7 @@ import math import json from multiprocessing import Pool import os +from cerberus import Validator from data import set_database_config, get_database_config from interface import stderr, stdout, INF, ERR @@ -140,32 +141,16 @@ class ConfigurationError (Exception): self.code = code def parse_config_persistent(send, config): + v = Validator(load_validation_schema(), allow_unknown = True) + isValidated = v.validate(config) - try: - apikey = config["persistent"]["key"]["database"] - except: - raise ConfigurationError("persistent/key/database field is invalid or missing", 111) - try: - tbakey = config["persistent"]["key"]["tba"] - except: - raise ConfigurationError("persistent/key/tba field is invalid or missing", 112) - try: - preference = config["persistent"]["config-preference"] - except: - raise ConfigurationError("persistent/config-preference field is invalid or missing", 113) - try: - sync = config["persistent"]["synchronize-config"] - except: - raise ConfigurationError("persistent/synchronize-config field is invalid or missing", 114) + if not isValidated: + raise ConfigurationError(v.errors, 101) - if apikey == None or apikey == "": - raise ConfigurationError("persistent/key/database field is empty", 115) - if tbakey == None or tbakey == "": - raise ConfigurationError("persistent/key/tba field is empty", 116) - if preference == None or preference == "": - raise ConfigurationError("persistent/config-preference field is empty", 117) - if sync != True and sync != False: - raise ConfigurationError("persistent/synchronize-config field is empty", 118) + apikey = config["persistent"]["key"]["database"] + tbakey = config["persistent"]["key"]["tba"] + preference = config["persistent"]["config-preference"] + sync = config["persistent"]["synchronize-config"] return apikey, tbakey, preference, sync @@ -249,6 +234,13 @@ def load_config(path, config_vector): f.close() return 1 +def load_validation_schema(): + try: + with open("validation-schema.json", "r") as f: + return json.load(f) + except: + raise FileNotFoundError("Validation schema not found at validation-schema.json") + def save_config(path, config_vector): f = open(path, "w+") json.dump(config_vector, f, ensure_ascii=False, indent=4) diff --git a/src/cli/validation-schema.json b/src/cli/validation-schema.json new file mode 100644 index 0000000..c0e02b8 --- /dev/null +++ b/src/cli/validation-schema.json @@ -0,0 +1,26 @@ +{ + "persistent": { + "type": "dict", + "require_all": true, + "schema": { + "key": { + "type": "dict", + "require_all":true, + "schema": { + "database": {"type":"string"}, + "tba": {"type": "string"}, + "tra": { + "type": "dict", + "require_all": true, + "schema": { + "CLIENT_ID": {"type": "string"}, + "CLIENT_SECRET": {"type": "string"} + } + } + } + }, + "config-preference": {"type": "string", "required": true}, + "synchronize-config": {"type": "boolean", "required": true} + } + } +} diff --git a/src/requirements.txt b/src/requirements.txt index 52854df..c89383a 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -14,4 +14,6 @@ pyparsing kivy==2.0.0rc2 pyzmq -python-daemon \ No newline at end of file +python-daemon + +cerebrus \ No newline at end of file From c77dd1ea5f3bb1898d79aafdb2dd60cfa7263d22 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 10 Jan 2022 09:23:43 -0600 Subject: [PATCH 101/143] set tab config Former-commit-id: 1c3fc9a87268e883127cf2adf092b73e992649f0 --- .devcontainer/devcontainer.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c380475..bc14fe9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,7 +17,9 @@ "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", - "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest" + "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest", + "editor.tabSize": 2, + "editor.insertSpaces": false }, "extensions": [ "mhutchie.git-graph", From 18d4438f7f8b7a82bdf3a87f958b30347e232b92 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 10 Jan 2022 15:35:06 +0000 Subject: [PATCH 102/143] fix typo Former-commit-id: e16295da2b8303134888e13201c88b89a05aac8d --- src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index c89383a..c295882 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -16,4 +16,4 @@ kivy==2.0.0rc2 pyzmq python-daemon -cerebrus \ No newline at end of file +cerberus \ No newline at end of file From db4b0208517debf968bc0387bcbbbd9ff052d669 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 10 Jan 2022 15:40:58 +0000 Subject: [PATCH 103/143] allow overriding competition in config Former-commit-id: 40252dd74cbbea6ac34f2adcb482505f2e8278fb --- src/cli/superscript.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index a362e4a..25c2518 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -215,7 +215,10 @@ def main(send, verbose = False, profile = False, debug = False): config = resolve_config_conflicts(send, client, config, preference, sync) exec_threads, config_modules = parse_config_variable(send, config) - competition = pull.get_team_competition() + if 'competition' in config['variable']: + competition = config['variable']['competition'] + else: + competition = pull.get_team_competition() for m in config_modules: if m in modules: start = time.time() From a940f3ffeb059334d6445cd3256133310624bac0 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 10 Jan 2022 20:02:58 +0000 Subject: [PATCH 104/143] use tab size 4 Former-commit-id: 2f83604e14d38a83e0ef1de528f51a9b30ce76d4 --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bc14fe9..81b21b5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,9 +16,9 @@ "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", + "python.linting.pylintPath": "/usr/local/bin/pylint", "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest", - "editor.tabSize": 2, + "editor.tabSize": 4, "editor.insertSpaces": false }, "extensions": [ From 7d081946362fb5d6042f25a0d463d00008a092bc Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 10 Jan 2022 20:23:14 +0000 Subject: [PATCH 105/143] patched issue in multiprocessing, moved ConfigurationError to exceptions.py, made pull use load_config in config.py Former-commit-id: aed03369c9136b674c91138e2d15e6ba99164e87 --- src/cli/config.py | 7 +-- src/cli/exceptions.py | 8 +++- src/cli/module.py | 9 ++-- src/cli/pull.py | 102 +++++++++++++++++++++--------------------- 4 files changed, 64 insertions(+), 62 deletions(-) diff --git a/src/cli/config.py b/src/cli/config.py index 3160c4b..900d151 100644 --- a/src/cli/config.py +++ b/src/cli/config.py @@ -3,6 +3,7 @@ import json from multiprocessing import Pool import os from cerberus import Validator +from exceptions import ConfigurationError from data import set_database_config, get_database_config from interface import stderr, stdout, INF, ERR @@ -134,12 +135,6 @@ sample_json = """ } """ -class ConfigurationError (Exception): - code = None - def __init__(self, str, code): - super().__init__(str) - self.code = code - def parse_config_persistent(send, config): v = Validator(load_validation_schema(), allow_unknown = True) isValidated = v.validate(config) diff --git a/src/cli/exceptions.py b/src/cli/exceptions.py index 891668b..13d4103 100644 --- a/src/cli/exceptions.py +++ b/src/cli/exceptions.py @@ -2,4 +2,10 @@ class APIError(Exception): code = None def __init__(self, str, endpoint): super().__init__(str) - self.endpoint = endpoint \ No newline at end of file + self.endpoint = endpoint + +class ConfigurationError (Exception): + code = None + def __init__(self, str, code): + super().__init__(str) + self.code = code \ No newline at end of file diff --git a/src/cli/module.py b/src/cli/module.py index e6ef29a..763a348 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -2,7 +2,7 @@ import abc import data as d import signal import numpy as np -import tra_analysis as an +from tra_analysis import Analysis as an class Module(metaclass = abc.ABCMeta): @@ -54,7 +54,7 @@ class Match (Module): def _load_data(self): self.data = d.load_match(self.apikey, self.competition) - def _simplestats(data_test): + def _simplestats(self, data_test): signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -103,7 +103,10 @@ class Match (Module): input_vector.append((team, variable, test, data[team][variable])) self.data = input_vector - self.results = list(exec_threads.map(self._simplestats, self.data)) + #self.results = list(exec_threads.map(self._simplestats, self.data)) + self.results = [] + for test_var_data in self.data: + self.results.append(self._simplestats(test_var_data)) def _push_results(self): diff --git a/src/cli/pull.py b/src/cli/pull.py index db2772f..5a95de6 100644 --- a/src/cli/pull.py +++ b/src/cli/pull.py @@ -1,66 +1,64 @@ import requests import json from exceptions import APIError - -def load_config(path): - with open(path, "r") as f: - return json.load(f) +from config import load_config url = "https://titanscouting.epochml.org" -config_tra = load_config("config.json") +config_tra = {} +load_config("config.json", config_tra) trakey = config_tra['persistent']['key']['tra'] def get_team_competition(): - endpoint = '/api/fetchTeamCompetition' - params = { - "CLIENT_ID": trakey['CLIENT_ID'], - "CLIENT_SECRET": trakey['CLIENT_SECRET'] - } - response = requests.request("GET", url + endpoint, params=params) - json = response.json() - if json['success']: - return json['competition'] - else: - raise APIError(json, endpoint) + endpoint = '/api/fetchTeamCompetition' + params = { + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return json['competition'] + else: + raise APIError(json, endpoint) def get_team(): - endpoint = '/api/fetchTeamCompetition' - params = { - "CLIENT_ID": trakey['CLIENT_ID'], - "CLIENT_SECRET": trakey['CLIENT_SECRET'] - } - response = requests.request("GET", url + endpoint, params=params) - json = response.json() - if json['success']: - return json['team'] - else: - raise APIError(json, endpoint) + endpoint = '/api/fetchTeamCompetition' + params = { + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return json['team'] + else: + raise APIError(json, endpoint) def get_team_match_data(competition, team_num): - endpoint = '/api/fetchAllTeamMatchData' - params = { - "competition": competition, - "teamScouted": team_num, - "CLIENT_ID": trakey['CLIENT_ID'], - "CLIENT_SECRET": trakey['CLIENT_SECRET'] - } - response = requests.request("GET", url + endpoint, params=params) - json = response.json() - if json['success']: - return json['data'][team_num] - else: - raise APIError(json, endpoint) + endpoint = '/api/fetchAllTeamMatchData' + params = { + "competition": competition, + "teamScouted": team_num, + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return json['data'][team_num] + else: + raise APIError(json, endpoint) def get_teams_at_competition(competition): - endpoint = '/api/fetchAllTeamNicknamesAtCompetition' - params = { - "competition": competition, - "CLIENT_ID": trakey['CLIENT_ID'], - "CLIENT_SECRET": trakey['CLIENT_SECRET'] - } - response = requests.request("GET", url + endpoint, params=params) - json = response.json() - if json['success']: - return list(json['data'].keys()) - else: - raise APIError(json, endpoint) \ No newline at end of file + endpoint = '/api/fetchAllTeamNicknamesAtCompetition' + params = { + "competition": competition, + "CLIENT_ID": trakey['CLIENT_ID'], + "CLIENT_SECRET": trakey['CLIENT_SECRET'] + } + response = requests.request("GET", url + endpoint, params=params) + json = response.json() + if json['success']: + return list(json['data'].keys()) + else: + raise APIError(json, endpoint) \ No newline at end of file From 40673169b2361ece78fa07a2a8ad551c7a937d0c Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Fri, 4 Feb 2022 02:44:48 +0000 Subject: [PATCH 106/143] update validation schema Former-commit-id: 982cb0aa6b4a80fd74ecc32cd69269fdb8888141 --- src/cli/validation-schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cli/validation-schema.json b/src/cli/validation-schema.json index c0e02b8..3b4c95c 100644 --- a/src/cli/validation-schema.json +++ b/src/cli/validation-schema.json @@ -14,7 +14,8 @@ "require_all": true, "schema": { "CLIENT_ID": {"type": "string"}, - "CLIENT_SECRET": {"type": "string"} + "CLIENT_SECRET": {"type": "string"}, + "url": {"type": "string"} } } } From 6cfe4e3f33de92286dafa84243b8051d177c3ebf Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 4 Feb 2022 03:06:09 +0000 Subject: [PATCH 107/143] fixed bugs in data.py to work with database, fixed object field in module.py: Metric, fixed timestamp input in superscripy.py Former-commit-id: 1e18cddc478fb00c675a34e6cb2b250e95f82b3d --- src/cli/data.py | 9 +++++---- src/cli/module.py | 2 +- src/cli/superscript.py | 5 +++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/cli/data.py b/src/cli/data.py index d262bd3..5df9ac9 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -4,7 +4,7 @@ import pandas as pd def pull_new_tba_matches(apikey, competition, cutoff): api_key= apikey - x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth_Key":api_key}, verify=False) + x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth-Key":api_key}, verify=False) out = [] for i in x.json(): if i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm": @@ -52,8 +52,9 @@ def get_metrics_data_formatted(client, competition): def get_pit_data_formatted(client, competition): db = client.data_scouting - mdata = db.teamlist - x=mdata.find_one({"competition":competition}) + mdata = db.teamlist.competition + x=mdata.find_one({"competition":competition}) # x should be a list of teams given the soecified competition + print(x) out = {} for i in x: try: @@ -155,7 +156,7 @@ def load_metric(client, competition, match, group_name, metrics): elo = {"score": metrics["elo"]["score"]} gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} - ts = {"mu": metrics["ts"]["mu"], "sigm+a": metrics["ts"]["sigma"]} + ts = {"mu": metrics["ts"]["mu"], "sigma": metrics["ts"]["sigma"]} group[team] = {"elo": elo, "gl2": gl2, "ts": ts} diff --git a/src/cli/module.py b/src/cli/module.py index 763a348..217c832 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -263,7 +263,7 @@ class Metric (Module): temp_vector.update(red) temp_vector.update(blu) - d.push_metric(self.client, self.competition, temp_vector) + d.push_metric(self.apikey, self.competition, temp_vector) def _push_results(self): pass diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 25c2518..970ab52 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -210,7 +210,8 @@ def main(send, verbose = False, profile = False, debug = False): client = pymongo.MongoClient(apikey) send(stdout, INF, "established connection to database") - send(stdout, INF, "analysis backtimed to: " + str(get_previous_time(client))) + previous_time = get_previous_time(client) + send(stdout, INF, "analysis backtimed to: " + str(previous_time)) config = resolve_config_conflicts(send, client, config, preference, sync) @@ -222,7 +223,7 @@ def main(send, verbose = False, profile = False, debug = False): for m in config_modules: if m in modules: start = time.time() - current_module = modules[m](config_modules[m], client, tbakey, loop_start, competition) + current_module = modules[m](config_modules[m], client, tbakey, previous_time, competition) valid = current_module.validate_config() if not valid: continue From c46e74036bdc7194a91da637af0f92881650bfb6 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Fri, 4 Feb 2022 03:16:45 +0000 Subject: [PATCH 108/143] get team nums at comp Former-commit-id: 028f6f775d4fe1327975f9688d7433e67f2623e0 --- src/cli/config.py | 3 ++- src/cli/data.py | 15 ++++----------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/cli/config.py b/src/cli/config.py index 900d151..332607b 100644 --- a/src/cli/config.py +++ b/src/cli/config.py @@ -18,7 +18,8 @@ sample_json = """ "tba":"", "tra":{ "CLIENT_ID":"", - "CLIENT_SECRET":"" + "CLIENT_SECRET":"", + "url": "" } }, "config-preference":"local", diff --git a/src/cli/data.py b/src/cli/data.py index 5df9ac9..9469c21 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -51,17 +51,10 @@ def get_metrics_data_formatted(client, competition): return out def get_pit_data_formatted(client, competition): - db = client.data_scouting - mdata = db.teamlist.competition - x=mdata.find_one({"competition":competition}) # x should be a list of teams given the soecified competition - print(x) - out = {} - for i in x: - try: - out[int(i)] = get_team_pit_data(client, competition, int(i)) - except: - pass - return out + x=requests.get("https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) + output = x.json() + data = output['data'] + return list(data.keys()) def get_pit_variable_data(client, competition): db = client.data_processing From c4bb633e08c4a8604d85bfa52d5baae69187aab5 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 4 Feb 2022 03:24:47 +0000 Subject: [PATCH 109/143] fixed pit data fetching Former-commit-id: d017ec52a7166a1e64bba88553915ca6d0f64789 --- src/cli/data.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/cli/data.py b/src/cli/data.py index 9469c21..f8d80fa 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -52,9 +52,16 @@ def get_metrics_data_formatted(client, competition): def get_pit_data_formatted(client, competition): x=requests.get("https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) - output = x.json() - data = output['data'] - return list(data.keys()) + x = x.json() + x = x['data'] + x = x.keys() + out = {} + for i in x: + try: + out[int(i)] = get_team_pit_data(client, competition, int(i)) + except: + pass + return out def get_pit_variable_data(client, competition): db = client.data_processing From b0ffdbab9a9a1a0d9d1db54cd8139723e00a52b7 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 4 Feb 2022 03:28:08 +0000 Subject: [PATCH 110/143] fixed pit processing issue due to config change Former-commit-id: 15f95b778bbddf687accc61c28a4a8975f5d89ae --- src/cli/module.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cli/module.py b/src/cli/module.py index 217c832..c1bb145 100644 --- a/src/cli/module.py +++ b/src/cli/module.py @@ -298,10 +298,12 @@ class Pit (Module): self.data = d.load_pit(self.apikey, self.competition) def _process_data(self, exec_threads): + tests = self.config["tests"] + print(tests) return_vector = {} for team in self.data: for variable in self.data[team]: - if variable in self.config: + if variable in tests: if not variable in return_vector: return_vector[variable] = [] return_vector[variable].append(self.data[team][variable]) From 491508a400e0f3c6109d7989ecd600002b8b3436 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 4 Feb 2022 08:38:03 +0000 Subject: [PATCH 111/143] switch docker image to python:slim, move requirements.txt to .devcontainer/ Former-commit-id: 427a37e514f689b9246ce4395bf179e079a5a903 --- .devcontainer/Dockerfile | 11 +++++------ .devcontainer/dev-dockerfile | 2 -- .devcontainer/devcontainer.json | 18 +++++------------ {src => .devcontainer}/requirements.txt | 26 +++++++++++-------------- 4 files changed, 21 insertions(+), 36 deletions(-) delete mode 100644 .devcontainer/dev-dockerfile rename {src => .devcontainer}/requirements.txt (70%) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 160a9b9..f762d03 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,6 @@ -FROM ubuntu:20.04 +FROM python:slim WORKDIR / -RUN apt-get -y update -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata -RUN apt-get install -y python3 python3-dev git python3-pip python3-kivy python-is-python3 libgl1-mesa-dev build-essential -RUN ln -s $(which pip3) /usr/bin/pip -RUN pip install pymongo pandas numpy scipy scikit-learn matplotlib pylint kivy \ No newline at end of file +RUN apt-get -y update; apt-get -y upgrade +RUN apt-get -y install git +COPY requirements.txt . +RUN pip install -r requirements.txt \ No newline at end of file diff --git a/.devcontainer/dev-dockerfile b/.devcontainer/dev-dockerfile deleted file mode 100644 index 74659d4..0000000 --- a/.devcontainer/dev-dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM titanscout2022/tra-analysis-base:latest -WORKDIR / \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 81b21b5..f62f520 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,23 +1,15 @@ { "name": "TRA Analysis Development Environment", "build": { - "dockerfile": "dev-dockerfile", + "dockerfile": "Dockerfile", }, "settings": { "terminal.integrated.shell.linux": "/bin/bash", - "python.pythonPath": "/usr/local/bin/python", + "python.pythonPath": "", "python.linting.enabled": true, "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", - "python.formatting.blackPath": "/usr/local/py-utils/bin/black", - "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", - "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", - "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", - "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", - "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", - "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/usr/local/bin/pylint", - "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest", + "python.linting.pylintPath": "", + "python.testing.pytestPath": "", "editor.tabSize": 4, "editor.insertSpaces": false }, @@ -26,5 +18,5 @@ "ms-python.python", "waderyan.gitblame" ], - "postCreateCommand": "/usr/bin/pip3 install -r ${containerWorkspaceFolder}/src/requirements.txt && /usr/bin/pip3 install --no-cache-dir pylint && /usr/bin/pip3 install pytest" + "postCreateCommand": "" } \ No newline at end of file diff --git a/src/requirements.txt b/.devcontainer/requirements.txt similarity index 70% rename from src/requirements.txt rename to .devcontainer/requirements.txt index c295882..56ec5dc 100644 --- a/src/requirements.txt +++ b/.devcontainer/requirements.txt @@ -1,19 +1,15 @@ -requests -pymongo -tra-analysis - +cerberus dnspython -pyinstaller - numpy -scipy -scikit-learn -six +pyinstaller +pylint +pymongo pyparsing - -kivy==2.0.0rc2 - -pyzmq +pytest python-daemon - -cerberus \ No newline at end of file +pyzmq +requests +scikit-learn +scipy +six +tra-analysis \ No newline at end of file From 524a0a211dbf33b3d3f4e61f13b6a6b77d35f355 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 9 Feb 2022 05:36:19 +0000 Subject: [PATCH 112/143] removed gui (last commit tagged "gui"), removed print statement in pit module Former-commit-id: 4978aee142eaf9431913b44eabfc0dfb79c7b600 --- src/{cli => }/config.py | 0 src/{cli => }/data.py | 0 src/{cli => }/exceptions.py | 0 src/gui/data.py | 129 ------ src/gui/design.kv | 151 ------- src/gui/main.py | 58 --- src/gui/superscript.py | 627 --------------------------- src/{cli => }/interface.py | 0 src/{cli => }/module.py | 1 - src/{cli => }/pull.py | 0 src/{cli => }/superscript.py | 0 src/{cli => }/superscript.spec | 0 src/{cli => }/validation-schema.json | 0 13 files changed, 966 deletions(-) rename src/{cli => }/config.py (100%) rename src/{cli => }/data.py (100%) rename src/{cli => }/exceptions.py (100%) delete mode 100644 src/gui/data.py delete mode 100644 src/gui/design.kv delete mode 100644 src/gui/main.py delete mode 100644 src/gui/superscript.py rename src/{cli => }/interface.py (100%) rename src/{cli => }/module.py (99%) rename src/{cli => }/pull.py (100%) rename src/{cli => }/superscript.py (100%) rename src/{cli => }/superscript.spec (100%) rename src/{cli => }/validation-schema.json (100%) diff --git a/src/cli/config.py b/src/config.py similarity index 100% rename from src/cli/config.py rename to src/config.py diff --git a/src/cli/data.py b/src/data.py similarity index 100% rename from src/cli/data.py rename to src/data.py diff --git a/src/cli/exceptions.py b/src/exceptions.py similarity index 100% rename from src/cli/exceptions.py rename to src/exceptions.py diff --git a/src/gui/data.py b/src/gui/data.py deleted file mode 100644 index 641aba7..0000000 --- a/src/gui/data.py +++ /dev/null @@ -1,129 +0,0 @@ -import requests -import pymongo -import pandas as pd -import time - -def pull_new_tba_matches(apikey, competition, cutoff): - api_key= apikey - x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth_Key":api_key}) - out = [] - for i in x.json(): - if i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm": - out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) - return out - -def get_team_match_data(apikey, competition, team_num): - client = pymongo.MongoClient(apikey) - db = client.data_scouting - mdata = db.matchdata - out = {} - for i in mdata.find({"competition" : competition, "team_scouted": team_num}): - out[i['match']] = i['data'] - return pd.DataFrame(out) - -def get_team_pit_data(apikey, competition, team_num): - client = pymongo.MongoClient(apikey) - db = client.data_scouting - mdata = db.pitdata - out = {} - return mdata.find_one({"competition" : competition, "team_scouted": team_num})["data"] - -def get_team_metrics_data(apikey, competition, team_num): - client = pymongo.MongoClient(apikey) - db = client.data_processing - mdata = db.team_metrics - return mdata.find_one({"competition" : competition, "team": team_num}) - -def get_match_data_formatted(apikey, competition): - client = pymongo.MongoClient(apikey) - db = client.data_scouting - mdata = db.teamlist - x=mdata.find_one({"competition":competition}) - out = {} - for i in x: - try: - out[int(i)] = unkeyify_2l(get_team_match_data(apikey, competition, int(i)).transpose().to_dict()) - except: - pass - return out - -def get_metrics_data_formatted(apikey, competition): - client = pymongo.MongoClient(apikey) - db = client.data_scouting - mdata = db.teamlist - x=mdata.find_one({"competition":competition}) - out = {} - for i in x: - try: - out[int(i)] = d.get_team_metrics_data(apikey, competition, int(i)) - except: - pass - return out - -def get_pit_data_formatted(apikey, competition): - client = pymongo.MongoClient(apikey) - db = client.data_scouting - mdata = db.teamlist - x=mdata.find_one({"competition":competition}) - out = {} - for i in x: - try: - out[int(i)] = get_team_pit_data(apikey, competition, int(i)) - except: - pass - return out - -def get_pit_variable_data(apikey, competition): - client = pymongo.MongoClient(apikey) - db = client.data_processing - mdata = db.team_pit - out = {} - return mdata.find() - -def get_pit_variable_formatted(apikey, competition): - temp = get_pit_variable_data(apikey, competition) - out = {} - for i in temp: - out[i["variable"]] = i["data"] - return out - -def push_team_tests_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_tests"): - client = pymongo.MongoClient(apikey) - db = client[dbname] - mdata = db[colname] - mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "data" : data}, True) - -def push_team_metrics_data(apikey, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"): - client = pymongo.MongoClient(apikey) - db = client[dbname] - mdata = db[colname] - mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True) - -def push_team_pit_data(apikey, competition, variable, data, dbname = "data_processing", colname = "team_pit"): - client = pymongo.MongoClient(apikey) - db = client[dbname] - mdata = db[colname] - mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True) - -def get_analysis_flags(apikey, flag): - client = pymongo.MongoClient(apikey) - db = client.data_processing - mdata = db.flags - return mdata.find_one({flag:{"$exists":True}}) - -def set_analysis_flags(apikey, flag, data): - client = pymongo.MongoClient(apikey) - db = client.data_processing - mdata = db.flags - return mdata.replace_one({flag:{"$exists":True}}, data, True) - -def unkeyify_2l(layered_dict): - out = {} - for i in layered_dict.keys(): - add = [] - sortkey = [] - for j in layered_dict[i].keys(): - add.append([j,layered_dict[i][j]]) - add.sort(key = lambda x: x[0]) - out[i] = list(map(lambda x: x[1], add)) - return out \ No newline at end of file diff --git a/src/gui/design.kv b/src/gui/design.kv deleted file mode 100644 index 177a926..0000000 --- a/src/gui/design.kv +++ /dev/null @@ -1,151 +0,0 @@ -: - orientation: "vertical" - - NavigationLayout: - ScreenManager: - id: screen_manager - HomeScreen: - name: "Home" - BoxLayout: - orientation: "vertical" - MDToolbar: - title: screen_manager.current - elevation: 10 - left_action_items: [['menu', lambda x: nav_drawer.toggle_nav_drawer()]] - - GridLayout: - cols: 1 - padding: 15, 15 - spacing: 20, 20 - MDTextFieldRect: - hint_text: "Console Log" - # size_hint: .8, None - # align: 'center' - # Widget: - SettingsScreen: - name: "Settings" - BoxLayout: - orientation: 'vertical' - MDToolbar: - title: screen_manager.current - elevation: 10 - left_action_items: [['menu', lambda x: nav_drawer.toggle_nav_drawer()]] - Widget: - InfoScreen: - name: "Info" - BoxLayout: - orientation: 'vertical' - MDToolbar: - title: screen_manager.current - elevation: 10 - left_action_items: [['menu', lambda x: nav_drawer.toggle_nav_drawer()]] - # GridLayout: - # cols: 2 - # padding: 15, 15 - # spacing: 20, 20 - BoxLayout: - orientation: "horizontal" - MDLabel: - text: "DB Key:" - halign: 'center' - MDTextField: - hint_text: "placeholder" - pos_hint: {"center_y": .5} - - BoxLayout: - orientation: "horizontal" - MDLabel: - text: "TBA Key:" - halign: 'center' - MDTextField: - hint_text: "placeholder" - pos_hint: {"center_y": .5} - BoxLayout: - orientation: "horizontal" - MDLabel: - text: "CPU Use:" - halign: 'center' - MDLabel: - text: "placeholder" - halign: 'center' - BoxLayout: - orientation: "horizontal" - MDLabel: - text: "Network:" - halign: 'center' - MDLabel: - text: "placeholder" - halign: 'center' - Widget: - BoxLayout: - orientation: "horizontal" - MDLabel: - text: "Progress" - halign: 'center' - MDProgressBar: - id: progress - value: 50 - StatsScreen: - name: "Stats" - MDCheckbox: - size_hint: None, None - size: "48dp", "48dp" - pos_hint: {'center_x': .5, 'center_y': .5} - on_active: Screen.test() - -#Navigation Drawer ------------------------- - MDNavigationDrawer: - id: nav_drawer - BoxLayout: - orientation: "vertical" - padding: "8dp" - spacing: "8dp" - MDLabel: - text: "Titan Scouting" - font_style: "Button" - size_hint_y: None - height: self.texture_size[1] - - MDLabel: - text: "Data Analysis" - font_style: "Caption" - size_hint_y: None - height: self.texture_size[1] - ScrollView: - MDList: - OneLineAvatarListItem: - text: "Home" - on_press: - # nav_drawer.set_state("close") - # screen_manager.transition.direction = "left" - screen_manager.current = "Home" - IconLeftWidget: - icon: "home" - - OneLineAvatarListItem: - text: "Settings" - on_press: - # nav_drawer.set_state("close") - # screen_manager.transition.direction = "right" - # screen_manager.fade - screen_manager.current = "Settings" - IconLeftWidget: - icon: "cog" - OneLineAvatarListItem: - text: "Info" - on_press: - # nav_drawer.set_state("close") - # screen_manager.transition.direction = "right" - # screen_manager.fade - screen_manager.current = "Info" - IconLeftWidget: - icon: "cog" - OneLineAvatarListItem: - text: "Stats" - on_press: - # nav_drawer.set_state("close") - # screen_manager.transition.direction = "right" - # screen_manager.fade - screen_manager.current = "Stats" - IconLeftWidget: - icon: "cog" \ No newline at end of file diff --git a/src/gui/main.py b/src/gui/main.py deleted file mode 100644 index a57421e..0000000 --- a/src/gui/main.py +++ /dev/null @@ -1,58 +0,0 @@ -from kivy.lang import Builder - -from kivymd.uix.screen import Screen -from kivymd.uix.list import OneLineListItem, MDList, TwoLineListItem, ThreeLineListItem -from kivymd.uix.list import OneLineIconListItem, IconLeftWidget -from kivy.uix.scrollview import ScrollView - - -from kivy.uix.boxlayout import BoxLayout -from kivy.uix.screenmanager import ScreenManager, Screen -from kivy.uix.dropdown import DropDown -from kivy.uix.button import Button -from kivy.base import runTouchApp -from kivymd.uix.menu import MDDropdownMenu, MDMenuItem - -from kivymd.app import MDApp -# import superscript as ss - -# from tra_analysis import analysis as an -import data as d -from collections import defaultdict -import json -import math -import numpy as np -import os -from os import system, name -from pathlib import Path -from multiprocessing import Pool -import matplotlib.pyplot as plt -from concurrent.futures import ThreadPoolExecutor -import time -import warnings - -# global exec_threads - - -# Screens -class HomeScreen(Screen): - pass -class SettingsScreen(Screen): - pass -class InfoScreen(Screen): - pass - -class StatsScreen(Screen): - pass - - -class MyApp(MDApp): - def build(self): - self.theme_cls.primary_palette = "Red" - return Builder.load_file("design.kv") - def test(): - print("test") - - -if __name__ == "__main__": - MyApp().run() \ No newline at end of file diff --git a/src/gui/superscript.py b/src/gui/superscript.py deleted file mode 100644 index e912951..0000000 --- a/src/gui/superscript.py +++ /dev/null @@ -1,627 +0,0 @@ -# Titan Robotics Team 2022: Superscript Script -# Written by Arthur Lu, Jacob Levine, and Dev Singh -# Notes: -# setup: - -__version__ = "0.8.6" - -# changelog should be viewed using print(analysis.__changelog__) -__changelog__ = """changelog: - 0.8.6: - - added proper main function - 0.8.5: - - added more gradeful KeyboardInterrupt exiting - - redirected stderr to errorlog.txt - 0.8.4: - - added better error message for missing config.json - - added automatic config.json creation - - added splash text with version and system info - 0.8.3: - - updated matchloop with new regression format (requires tra_analysis 3.x) - 0.8.2: - - readded while true to main function - - added more thread config options - 0.8.1: - - optimized matchloop further by bypassing GIL - 0.8.0: - - added multithreading to matchloop - - tweaked user log - 0.7.0: - - finished implementing main function - 0.6.2: - - integrated get_team_rankings.py as get_team_metrics() function - - integrated visualize_pit.py as graph_pit_histogram() function - 0.6.1: - - bug fixes with analysis.Metric() calls - - modified metric functions to use config.json defined default values - 0.6.0: - - removed main function - - changed load_config function - - added save_config function - - added load_match function - - renamed simpleloop to matchloop - - moved simplestats function inside matchloop - - renamed load_metrics to load_metric - - renamed metricsloop to metricloop - - split push to database functions amon push_match, push_metric, push_pit - - moved - 0.5.2: - - made changes due to refactoring of analysis - 0.5.1: - - text fixes - - removed matplotlib requirement - 0.5.0: - - improved user interface - 0.4.2: - - removed unessasary code - 0.4.1: - - fixed bug where X range for regression was determined before sanitization - - better sanitized data - 0.4.0: - - fixed spelling issue in __changelog__ - - addressed nan bug in regression - - fixed errors on line 335 with metrics calling incorrect key "glicko2" - - fixed errors in metrics computing - 0.3.0: - - added analysis to pit data - 0.2.1: - - minor stability patches - - implemented db syncing for timestamps - - fixed bugs - 0.2.0: - - finalized testing and small fixes - 0.1.4: - - finished metrics implement, trueskill is bugged - 0.1.3: - - working - 0.1.2: - - started implement of metrics - 0.1.1: - - cleaned up imports - 0.1.0: - - tested working, can push to database - 0.0.9: - - tested working - - prints out stats for the time being, will push to database later - 0.0.8: - - added data import - - removed tba import - - finished main method - 0.0.7: - - added load_config - - optimized simpleloop for readibility - - added __all__ entries - - added simplestats engine - - pending testing - 0.0.6: - - fixes - 0.0.5: - - imported pickle - - created custom database object - 0.0.4: - - fixed simpleloop to actually return a vector - 0.0.3: - - added metricsloop which is unfinished - 0.0.2: - - added simpleloop which is untested until data is provided - 0.0.1: - - created script - - added analysis, numba, numpy imports -""" - -__author__ = ( - "Arthur Lu ", - "Jacob Levine ", -) - -__all__ = [ - "load_config", - "save_config", - "get_previous_time", - "load_match", - "matchloop", - "load_metric", - "metricloop", - "load_pit", - "pitloop", - "push_match", - "push_metric", - "push_pit", -] - -# imports: - -from tra_analysis import analysis as an -import data as d -from collections import defaultdict -import json -import math -import numpy as np -import os -from os import system, name -from pathlib import Path -from multiprocessing import Pool -import platform -import sys -import time -import warnings - -global exec_threads - -def main(): - - global exec_threads - - sys.stderr = open("errorlog.txt", "w") - - warnings.filterwarnings("ignore") - - splash() - - while (True): - - try: - - current_time = time.time() - print("[OK] time: " + str(current_time)) - - config = load_config("config.json") - competition = config["competition"] - match_tests = config["statistics"]["match"] - pit_tests = config["statistics"]["pit"] - metrics_tests = config["statistics"]["metric"] - print("[OK] configs loaded") - - print("[OK] starting threads") - cfg_max_threads = config["max-threads"] - sys_max_threads = os.cpu_count() - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - print("[ERROR] Invalid number of processes, must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads)) - exit() - exec_threads = Pool(processes = alloc_processes) - print("[OK] " + str(alloc_processes) + " threads started") - - apikey = config["key"]["database"] - tbakey = config["key"]["tba"] - print("[OK] loaded keys") - - previous_time = get_previous_time(apikey) - print("[OK] analysis backtimed to: " + str(previous_time)) - - print("[OK] loading data") - start = time.time() - match_data = load_match(apikey, competition) - pit_data = load_pit(apikey, competition) - print("[OK] loaded data in " + str(time.time() - start) + " seconds") - - print("[OK] running match stats") - start = time.time() - matchloop(apikey, competition, match_data, match_tests) - print("[OK] finished match stats in " + str(time.time() - start) + " seconds") - - print("[OK] running team metrics") - start = time.time() - metricloop(tbakey, apikey, competition, previous_time, metrics_tests) - print("[OK] finished team metrics in " + str(time.time() - start) + " seconds") - - print("[OK] running pit analysis") - start = time.time() - pitloop(apikey, competition, pit_data, pit_tests) - print("[OK] finished pit analysis in " + str(time.time() - start) + " seconds") - - set_current_time(apikey, current_time) - print("[OK] finished all tests, looping") - - print_hrule() - - except KeyboardInterrupt: - print("\n[OK] caught KeyboardInterrupt, killing processes") - exec_threads.terminate() - print("[OK] processes killed, exiting") - exit() - - else: - pass - - #clear() - -def clear(): - - # for windows - if name == 'nt': - _ = system('cls') - - # for mac and linux(here, os.name is 'posix') - else: - _ = system('clear') - -def print_hrule(): - - print("#"+38*"-"+"#") - -def print_box(s): - - temp = "|" - temp += s - temp += (40-len(s)-2)*" " - temp += "|" - print(temp) - -def splash(): - - print_hrule() - print_box(" superscript version: " + __version__) - print_box(" os: " + platform.system()) - print_box(" python: " + platform.python_version()) - print_hrule() - -def load_config(file): - - config_vector = {} - - try: - f = open(file) - except: - print("[ERROR] could not locate config.json, generating blank config.json and exiting") - f = open(file, "w") - f.write(sample_json) - exit() - - config_vector = json.load(f) - - return config_vector - -def save_config(file, config_vector): - - with open(file) as f: - json.dump(config_vector, f) - -def get_previous_time(apikey): - - previous_time = d.get_analysis_flags(apikey, "latest_update") - - if previous_time == None: - - d.set_analysis_flags(apikey, "latest_update", 0) - previous_time = 0 - - else: - - previous_time = previous_time["latest_update"] - - return previous_time - -def set_current_time(apikey, current_time): - - d.set_analysis_flags(apikey, "latest_update", {"latest_update":current_time}) - -def load_match(apikey, competition): - - return d.get_match_data_formatted(apikey, competition) - -def simplestats(data_test): - - data = np.array(data_test[0]) - data = data[np.isfinite(data)] - ranges = list(range(len(data))) - - test = data_test[1] - - if test == "basic_stats": - return an.basic_stats(data) - - if test == "historical_analysis": - return an.histo_analysis([ranges, data]) - - if test == "regression_linear": - return an.regression(ranges, data, ['lin']) - - if test == "regression_logarithmic": - return an.regression(ranges, data, ['log']) - - if test == "regression_exponential": - return an.regression(ranges, data, ['exp']) - - if test == "regression_polynomial": - return an.regression(ranges, data, ['ply']) - - if test == "regression_sigmoidal": - return an.regression(ranges, data, ['sig']) - -def matchloop(apikey, competition, data, tests): # expects 3D array with [Team][Variable][Match] - - global exec_threads - - short_mapping = {"regression_linear": "lin", "regression_logarithmic": "log", "regression_exponential": "exp", "regression_polynomial": "ply", "regression_sigmoidal": "sig"} - - class AutoVivification(dict): - def __getitem__(self, item): - try: - return dict.__getitem__(self, item) - except KeyError: - value = self[item] = type(self)() - return value - - return_vector = {} - - team_filtered = [] - variable_filtered = [] - variable_data = [] - test_filtered = [] - result_filtered = [] - return_vector = AutoVivification() - - for team in data: - - for variable in data[team]: - - if variable in tests: - - for test in tests[variable]: - - team_filtered.append(team) - variable_filtered.append(variable) - variable_data.append((data[team][variable], test)) - test_filtered.append(test) - - result_filtered = exec_threads.map(simplestats, variable_data) - i = 0 - - result_filtered = list(result_filtered) - - for result in result_filtered: - - filtered = test_filtered[i] - - try: - short = short_mapping[filtered] - return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result[short] - except KeyError: # not in mapping - return_vector[team_filtered[i]][variable_filtered[i]][test_filtered[i]] = result - i += 1 - - push_match(apikey, competition, return_vector) - -def load_metric(apikey, competition, match, group_name, metrics): - - group = {} - - for team in match[group_name]: - - db_data = d.get_team_metrics_data(apikey, competition, team) - - if d.get_team_metrics_data(apikey, competition, team) == None: - - elo = {"score": metrics["elo"]["score"]} - gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} - ts = {"mu": metrics["ts"]["mu"], "sigma": metrics["ts"]["sigma"]} - - group[team] = {"elo": elo, "gl2": gl2, "ts": ts} - - else: - - metrics = db_data["metrics"] - - elo = metrics["elo"] - gl2 = metrics["gl2"] - ts = metrics["ts"] - - group[team] = {"elo": elo, "gl2": gl2, "ts": ts} - - return group - -def metricloop(tbakey, apikey, competition, timestamp, metrics): # listener based metrics update - - elo_N = metrics["elo"]["N"] - elo_K = metrics["elo"]["K"] - - matches = d.pull_new_tba_matches(tbakey, competition, timestamp) - - red = {} - blu = {} - - for match in matches: - - red = load_metric(apikey, competition, match, "red", metrics) - blu = load_metric(apikey, competition, match, "blue", metrics) - - elo_red_total = 0 - elo_blu_total = 0 - - gl2_red_score_total = 0 - gl2_blu_score_total = 0 - - gl2_red_rd_total = 0 - gl2_blu_rd_total = 0 - - gl2_red_vol_total = 0 - gl2_blu_vol_total = 0 - - for team in red: - - elo_red_total += red[team]["elo"]["score"] - - gl2_red_score_total += red[team]["gl2"]["score"] - gl2_red_rd_total += red[team]["gl2"]["rd"] - gl2_red_vol_total += red[team]["gl2"]["vol"] - - for team in blu: - - elo_blu_total += blu[team]["elo"]["score"] - - gl2_blu_score_total += blu[team]["gl2"]["score"] - gl2_blu_rd_total += blu[team]["gl2"]["rd"] - gl2_blu_vol_total += blu[team]["gl2"]["vol"] - - red_elo = {"score": elo_red_total / len(red)} - blu_elo = {"score": elo_blu_total / len(blu)} - - red_gl2 = {"score": gl2_red_score_total / len(red), "rd": gl2_red_rd_total / len(red), "vol": gl2_red_vol_total / len(red)} - blu_gl2 = {"score": gl2_blu_score_total / len(blu), "rd": gl2_blu_rd_total / len(blu), "vol": gl2_blu_vol_total / len(blu)} - - - if match["winner"] == "red": - - observations = {"red": 1, "blu": 0} - - elif match["winner"] == "blue": - - observations = {"red": 0, "blu": 1} - - else: - - observations = {"red": 0.5, "blu": 0.5} - - red_elo_delta = an.Metric().elo(red_elo["score"], blu_elo["score"], observations["red"], elo_N, elo_K) - red_elo["score"] - blu_elo_delta = an.Metric().elo(blu_elo["score"], red_elo["score"], observations["blu"], elo_N, elo_K) - blu_elo["score"] - - new_red_gl2_score, new_red_gl2_rd, new_red_gl2_vol = an.Metric().glicko2(red_gl2["score"], red_gl2["rd"], red_gl2["vol"], [blu_gl2["score"]], [blu_gl2["rd"]], [observations["red"], observations["blu"]]) - new_blu_gl2_score, new_blu_gl2_rd, new_blu_gl2_vol = an.Metric().glicko2(blu_gl2["score"], blu_gl2["rd"], blu_gl2["vol"], [red_gl2["score"]], [red_gl2["rd"]], [observations["blu"], observations["red"]]) - - red_gl2_delta = {"score": new_red_gl2_score - red_gl2["score"], "rd": new_red_gl2_rd - red_gl2["rd"], "vol": new_red_gl2_vol - red_gl2["vol"]} - blu_gl2_delta = {"score": new_blu_gl2_score - blu_gl2["score"], "rd": new_blu_gl2_rd - blu_gl2["rd"], "vol": new_blu_gl2_vol - blu_gl2["vol"]} - - for team in red: - - red[team]["elo"]["score"] = red[team]["elo"]["score"] + red_elo_delta - - red[team]["gl2"]["score"] = red[team]["gl2"]["score"] + red_gl2_delta["score"] - red[team]["gl2"]["rd"] = red[team]["gl2"]["rd"] + red_gl2_delta["rd"] - red[team]["gl2"]["vol"] = red[team]["gl2"]["vol"] + red_gl2_delta["vol"] - - for team in blu: - - blu[team]["elo"]["score"] = blu[team]["elo"]["score"] + blu_elo_delta - - blu[team]["gl2"]["score"] = blu[team]["gl2"]["score"] + blu_gl2_delta["score"] - blu[team]["gl2"]["rd"] = blu[team]["gl2"]["rd"] + blu_gl2_delta["rd"] - blu[team]["gl2"]["vol"] = blu[team]["gl2"]["vol"] + blu_gl2_delta["vol"] - - temp_vector = {} - temp_vector.update(red) - temp_vector.update(blu) - - push_metric(apikey, competition, temp_vector) - -def load_pit(apikey, competition): - - return d.get_pit_data_formatted(apikey, competition) - -def pitloop(apikey, competition, pit, tests): - - return_vector = {} - for team in pit: - for variable in pit[team]: - if variable in tests: - if not variable in return_vector: - return_vector[variable] = [] - return_vector[variable].append(pit[team][variable]) - - push_pit(apikey, competition, return_vector) - -def push_match(apikey, competition, results): - - for team in results: - - d.push_team_tests_data(apikey, competition, team, results[team]) - -def push_metric(apikey, competition, metric): - - for team in metric: - - d.push_team_metrics_data(apikey, competition, team, metric[team]) - -def push_pit(apikey, competition, pit): - - for variable in pit: - - d.push_team_pit_data(apikey, competition, variable, pit[variable]) - -def get_team_metrics(apikey, tbakey, competition): - - metrics = d.get_metrics_data_formatted(apikey, competition) - - elo = {} - gl2 = {} - - for team in metrics: - - elo[team] = metrics[team]["metrics"]["elo"]["score"] - gl2[team] = metrics[team]["metrics"]["gl2"]["score"] - - elo = {k: v for k, v in sorted(elo.items(), key=lambda item: item[1])} - gl2 = {k: v for k, v in sorted(gl2.items(), key=lambda item: item[1])} - - elo_ranked = [] - - for team in elo: - - elo_ranked.append({"team": str(team), "elo": str(elo[team])}) - - gl2_ranked = [] - - for team in gl2: - - gl2_ranked.append({"team": str(team), "gl2": str(gl2[team])}) - - return {"elo-ranks": elo_ranked, "glicko2-ranks": gl2_ranked} - -sample_json = """{ - "max-threads": 0.5, - "team": "", - "competition": "2020ilch", - "key":{ - "database":"", - "tba":"" - }, - "statistics":{ - "match":{ - "balls-blocked":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-collected":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-lower-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-started":["basic_stats","historical_analyss","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-teleop":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"], - "balls-upper-auto":["basic_stats","historical_analysis","regression_linear","regression_logarithmic","regression_exponential","regression_polynomial","regression_sigmoidal"] - - }, - "metric":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 - }, - "ts":{ - "mu":25, - "sigma":8.33 - } - }, - "pit":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true - } - } -}""" - -if __name__ == "__main__": - if sys.platform.startswith('win'): - multiprocessing.freeze_support() - main() \ No newline at end of file diff --git a/src/cli/interface.py b/src/interface.py similarity index 100% rename from src/cli/interface.py rename to src/interface.py diff --git a/src/cli/module.py b/src/module.py similarity index 99% rename from src/cli/module.py rename to src/module.py index c1bb145..a059e1a 100644 --- a/src/cli/module.py +++ b/src/module.py @@ -299,7 +299,6 @@ class Pit (Module): def _process_data(self, exec_threads): tests = self.config["tests"] - print(tests) return_vector = {} for team in self.data: for variable in self.data[team]: diff --git a/src/cli/pull.py b/src/pull.py similarity index 100% rename from src/cli/pull.py rename to src/pull.py diff --git a/src/cli/superscript.py b/src/superscript.py similarity index 100% rename from src/cli/superscript.py rename to src/superscript.py diff --git a/src/cli/superscript.spec b/src/superscript.spec similarity index 100% rename from src/cli/superscript.spec rename to src/superscript.spec diff --git a/src/cli/validation-schema.json b/src/validation-schema.json similarity index 100% rename from src/cli/validation-schema.json rename to src/validation-schema.json From b43836899d8a0fabda53b636379b4e2e83ca172b Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 19 Feb 2022 06:19:13 +0000 Subject: [PATCH 113/143] moved config functions into Configuration class, simplified exception class by removing error codes, removed exec_threads from module parameters Former-commit-id: 545ef765653970e1cdebac692eccd227effb2508 --- src/config.py | 466 +++++++++++++++++++------------------ src/dep.py | 141 +++++++++++ src/exceptions.py | 10 +- src/module.py | 21 +- src/pull.py | 2 +- src/superscript.py | 36 +-- src/validation-schema.json | 27 --- 7 files changed, 404 insertions(+), 299 deletions(-) create mode 100644 src/dep.py delete mode 100644 src/validation-schema.json diff --git a/src/config.py b/src/config.py index 332607b..46493a7 100644 --- a/src/config.py +++ b/src/config.py @@ -1,244 +1,252 @@ -import math import json -from multiprocessing import Pool -import os -from cerberus import Validator from exceptions import ConfigurationError +from cerberus import Validator from data import set_database_config, get_database_config from interface import stderr, stdout, INF, ERR -config_path = "config.json" +class Configuration: -sample_json = """ -{ - "persistent":{ - "key":{ - "database":"", - "tba":"", - "tra":{ - "CLIENT_ID":"", - "CLIENT_SECRET":"", - "url": "" - } - }, - "config-preference":"local", - "synchronize-config":false - }, - "variable":{ - "max-threads":0.5, - "team":"", - "event-delay":false, - "loop-delay":0, - "reportable":true, - "teams":[ - - ], - "modules":{ - "match":{ - "tests":{ - "balls-blocked":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-collected":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-lower-teleop":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-lower-auto":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-started":[ - "basic_stats", - "historical_analyss", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-upper-teleop":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-upper-auto":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ] - } - }, - "metric":{ - "tests":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 - }, - "ts":{ - "mu":25, - "sigma":8.33 - } - } - }, - "pit":{ - "tests":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true - } - } - } - } -} -""" + path = None + config = {} -def parse_config_persistent(send, config): - v = Validator(load_validation_schema(), allow_unknown = True) - isValidated = v.validate(config) + _sample_config = { + "persistent":{ + "key":{ + "database":"", + "tba":"", + "tra":{ + "CLIENT_ID":"", + "CLIENT_SECRET":"", + "url": "" + } + }, + "config-preference":"local", + "synchronize-config":False + }, + "variable":{ + "event-delay":False, + "loop-delay":0, + "competition": "2020ilch", + "modules":{ + "match":{ + "tests":{ + "balls-blocked":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-collected":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-started":[ + "basic_stats", + "historical_analyss", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ] + } + }, + "metric":{ + "tests":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + } + }, + "pit":{ + "tests":{ + "wheel-mechanism":True, + "low-balls":True, + "high-balls":True, + "wheel-success":True, + "strategic-focus":True, + "climb-mechanism":True, + "attitude":True + } + } + } + } + } - if not isValidated: - raise ConfigurationError(v.errors, 101) + _validation_schema = { + "persistent": { + "type": "dict", + "required": True, + "require_all": True, + "schema": { + "key": { + "type": "dict", + "require_all":True, + "schema": { + "database": {"type":"string"}, + "tba": {"type": "string"}, + "tra": { + "type": "dict", + "require_all": True, + "schema": { + "CLIENT_ID": {"type": "string"}, + "CLIENT_SECRET": {"type": "string"}, + "url": {"type": "string"} + } + } + } + }, + "config-preference": {"type": "string", "required": True}, + "synchronize-config": {"type": "boolean", "required": True} + } + } + } - apikey = config["persistent"]["key"]["database"] - tbakey = config["persistent"]["key"]["tba"] - preference = config["persistent"]["config-preference"] - sync = config["persistent"]["synchronize-config"] + def __init__(self, path): + self.path = path + self.load_config() + self.validate_config() - return apikey, tbakey, preference, sync + def load_config(self): + try: + f = open(self.path, "r") + self.config.update(json.load(f)) + f.close() + except: + self.config = self._sample_config + self.save_config() + f.close() + raise ConfigurationError("could not find config file at <" + self.path + ">, created new sample config file at that path") -def parse_config_variable(send, config): - - sys_max_threads = os.cpu_count() - try: - cfg_max_threads = config["variable"]["max-threads"] - except: - raise ConfigurationError("variable/max-threads field is invalid or missing, refer to documentation for configuration options", 109) - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - raise ConfigurationError("variable/max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, 110) - try: - exec_threads = Pool(processes = alloc_processes) - except Exception as e: - send(stderr, INF, e) - raise ConfigurationError("unable to start threads", 200) - send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") - - try: - modules = config["variable"]["modules"] - except: - raise ConfigurationError("variable/modules field is invalid or missing", 102) - - if modules == None: - raise ConfigurationError("variable/modules field is empty", 106) - - send(stdout, INF, "found and loaded competition, match, metrics, pit from config") - - return exec_threads, modules - -def resolve_config_conflicts(send, client, config, preference, sync): - - if sync: - if preference == "local" or preference == "client": - send(stdout, INF, "config-preference set to local/client, loading local config information") - remote_config = get_database_config(client) - if remote_config != config["variable"]: - set_database_config(client, config["variable"]) - send(stdout, INF, "database config was different and was updated") - return config - elif preference == "remote" or preference == "database": - send(stdout, INF, "config-preference set to remote/database, loading remote config information") - remote_config= get_database_config(client) - if remote_config != config["variable"]: - config["variable"] = remote_config - if save_config(config_path, config): - raise ConfigurationError("local config was different but could not be updated", 121) - send(stdout, INF, "local config was different and was updated") - return config - else: - raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) - else: - if preference == "local" or preference == "client": - send(stdout, INF, "config-preference set to local/client, loading local config information") - return config - elif preference == "remote" or preference == "database": - send(stdout, INF, "config-preference set to remote/database, loading database config information") - config["variable"] = get_database_config(client) - return config - else: - raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) - -def load_config(path, config_vector): - try: - f = open(path, "r") - config_vector.update(json.load(f)) + def save_config(self): + f = open(self.path, "w+") + json.dump(self.config, f, ensure_ascii=False, indent=4) f.close() - return 0 - except: - f = open(path, "w") - f.write(sample_json) - f.close() - return 1 -def load_validation_schema(): - try: - with open("validation-schema.json", "r") as f: - return json.load(f) - except: - raise FileNotFoundError("Validation schema not found at validation-schema.json") + def validate_config(self): + v = Validator(self._validation_schema, allow_unknown = True) + isValidated = v.validate(self.config) -def save_config(path, config_vector): - f = open(path, "w+") - json.dump(config_vector, f, ensure_ascii=False, indent=4) - f.close() - return 0 \ No newline at end of file + if not isValidated: + raise ConfigurationError("config validation error: " + v.errors) + + def __getattr__(self, name): # simple linear lookup method for common multikey-value paths, TYPE UNSAFE + if name == "persistent": + return self.config["persistent"] + elif name == "key": + return self.config["persistent"]["key"] + elif name == "database": + # soon to be deprecated + return self.config["persistent"]["key"]["database"] + elif name == "tba": + return self.config["persistent"]["key"]["tba"] + elif name == "tra": + return self.config["persistent"]["key"]["tra"] + elif name == "priority": + return self.config["persistent"]["config-preference"] + elif name == "sync": + return self.config["persistent"]["synchronize-config"] + elif name == "variable": + return self.config["variable"] + elif name == "event_delay": + return self.config["variable"]["event-delay"] + elif name == "loop_delay": + return self.config["variable"]["loop-delay"] + elif name == "competition": + return self.config["variable"]["competition"] + elif name == "modules": + return self.config["variable"]["modules"] + else: + return None + + def __getitem__(self, key): + return self.config[key] + + def resolve_config_conflicts(self, send, client): # needs improvement with new localization scheme + sync = self.sync + priority = self.priority + + if sync: + if priority == "local" or priority == "client": + send(stdout, INF, "config-preference set to local/client, loading local config information") + remote_config = get_database_config(client) + if remote_config != self.config["variable"]: + set_database_config(client, self.config["variable"]) + send(stdout, INF, "database config was different and was updated") + # no change to config + elif priority == "remote" or priority == "database": + send(stdout, INF, "config-preference set to remote/database, loading remote config information") + remote_config = get_database_config(client) + if remote_config != self.config["variable"]: + self.config["variable"] = remote_config + self.save_config() + # change variable to match remote + send(stdout, INF, "local config was different and was updated") + else: + raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"") + else: + if priority == "local" or priority == "client": + send(stdout, INF, "config-preference set to local/client, loading local config information") + # no change to config + elif priority == "remote" or priority == "database": + send(stdout, INF, "config-preference set to remote/database, loading database config information") + self.config["variable"] = get_database_config(client) + # change variable to match remote without updating local version + else: + raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"") \ No newline at end of file diff --git a/src/dep.py b/src/dep.py new file mode 100644 index 0000000..53abb64 --- /dev/null +++ b/src/dep.py @@ -0,0 +1,141 @@ +# contains deprecated functions, not to be used unless nessasary! + +import json + +sample_json = """ +{ + "persistent":{ + "key":{ + "database":"", + "tba":"", + "tra":{ + "CLIENT_ID":"", + "CLIENT_SECRET":"", + "url": "" + } + }, + "config-preference":"local", + "synchronize-config":false + }, + "variable":{ + "max-threads":0.5, + "team":"", + "event-delay":false, + "loop-delay":0, + "reportable":true, + "teams":[ + + ], + "modules":{ + "match":{ + "tests":{ + "balls-blocked":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-collected":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-started":[ + "basic_stats", + "historical_analyss", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ] + } + }, + "metric":{ + "tests":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + } + }, + "pit":{ + "tests":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true + } + } + } + } +} +""" + +def load_config(path, config_vector): + try: + f = open(path, "r") + config_vector.update(json.load(f)) + f.close() + return 0 + except: + f = open(path, "w") + f.write(sample_json) + f.close() + return 1 \ No newline at end of file diff --git a/src/exceptions.py b/src/exceptions.py index 13d4103..64f97dd 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -1,11 +1,7 @@ class APIError(Exception): - code = None - def __init__(self, str, endpoint): + def __init__(self, str): super().__init__(str) - self.endpoint = endpoint class ConfigurationError (Exception): - code = None - def __init__(self, str, code): - super().__init__(str) - self.code = code \ No newline at end of file + def __init__(self, str): + super().__init__(str) \ No newline at end of file diff --git a/src/module.py b/src/module.py index a059e1a..f5d5023 100644 --- a/src/module.py +++ b/src/module.py @@ -22,7 +22,7 @@ class Module(metaclass = abc.ABCMeta): def validate_config(self, *args, **kwargs): raise NotImplementedError @abc.abstractmethod - def run(self, exec_threads, *args, **kwargs): + def run(self, *args, **kwargs): raise NotImplementedError class Match (Module): @@ -46,9 +46,9 @@ class Match (Module): def validate_config(self): return True, "" - def run(self, exec_threads): + def run(self): self._load_data() - self._process_data(exec_threads) + self._process_data() self._push_results() def _load_data(self): @@ -85,7 +85,7 @@ class Match (Module): if test == "regression_sigmoidal": return an.regression(ranges, data, ['sig']) - def _process_data(self, exec_threads): + def _process_data(self): tests = self.config["tests"] data = self.data @@ -103,7 +103,6 @@ class Match (Module): input_vector.append((team, variable, test, data[team][variable])) self.data = input_vector - #self.results = list(exec_threads.map(self._simplestats, self.data)) self.results = [] for test_var_data in self.data: self.results.append(self._simplestats(test_var_data)) @@ -164,15 +163,15 @@ class Metric (Module): def validate_config(self): return True, "" - def run(self, exec_threads): + def run(self): self._load_data() - self._process_data(exec_threads) + self._process_data() self._push_results() def _load_data(self): self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp) - def _process_data(self, exec_threads): + def _process_data(self): elo_N = self.config["tests"]["elo"]["N"] elo_K = self.config["tests"]["elo"]["K"] @@ -289,15 +288,15 @@ class Pit (Module): def validate_config(self): return True, "" - def run(self, exec_threads): + def run(self): self._load_data() - self._process_data(exec_threads) + self._process_data() self._push_results() def _load_data(self): self.data = d.load_pit(self.apikey, self.competition) - def _process_data(self, exec_threads): + def _process_data(self): tests = self.config["tests"] return_vector = {} for team in self.data: diff --git a/src/pull.py b/src/pull.py index 5a95de6..09127b0 100644 --- a/src/pull.py +++ b/src/pull.py @@ -1,7 +1,7 @@ import requests import json from exceptions import APIError -from config import load_config +from dep import load_config url = "https://titanscouting.epochml.org" config_tra = {} diff --git a/src/superscript.py b/src/superscript.py index 970ab52..1cb8bff 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -23,6 +23,9 @@ __changelog__ = """changelog: - config-preference option selects between prioritizing local config and prioritizing database config - synchronize-config option selects whether to update the non prioritized config with the prioritized one - divided config options between persistent ones (keys), and variable ones (everything else) + - generalized behavior of various core components by collecting loose functions in several dependencies into classes + - module.py contains classes, each one represents a single data analysis routine + - config.py contains the Configuration class, which stores the configuration information and abstracts the getter methods 0.9.3: - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions @@ -152,16 +155,12 @@ __all__ = [ # imports: import json -from multiprocessing import freeze_support -import os -import pymongo -import sys -import time +import os, sys, time +import pymongo # soon to be deprecated import traceback import warnings import zmq -import pull -from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError +from config import Configuration, ConfigurationError from data import get_previous_time, set_current_time, check_new_database_matches from interface import splash, log, ERR, INF, stdout, stderr from module import Match, Metric, Pit @@ -171,10 +170,6 @@ config_path = "config.json" def main(send, verbose = False, profile = False, debug = False): def close_all(): - if "exec_threads" in locals(): - exec_threads.terminate() - exec_threads.join() - exec_threads.close() if "client" in locals(): client.close() if "f" in locals(): @@ -196,14 +191,11 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "current time: " + str(loop_start)) - config = {} - - if load_config(config_path, config): - raise ConfigurationError("could not find config at <" + config_path + ">, generating blank config and exiting", 110) + config = Configuration(config_path) send(stdout, INF, "found and loaded config at <" + config_path + ">") - apikey, tbakey, preference, sync = parse_config_persistent(send, config) + apikey, tbakey = config.database, config.tba send(stdout, INF, "found and loaded database and tba keys") @@ -213,13 +205,10 @@ def main(send, verbose = False, profile = False, debug = False): previous_time = get_previous_time(client) send(stdout, INF, "analysis backtimed to: " + str(previous_time)) - config = resolve_config_conflicts(send, client, config, preference, sync) + config.resolve_config_conflicts(send, client) + + config_modules, competition = config.modules, config.competition - exec_threads, config_modules = parse_config_variable(send, config) - if 'competition' in config['variable']: - competition = config['variable']['competition'] - else: - competition = pull.get_team_competition() for m in config_modules: if m in modules: start = time.time() @@ -227,7 +216,7 @@ def main(send, verbose = False, profile = False, debug = False): valid = current_module.validate_config() if not valid: continue - current_module.run(exec_threads) + current_module.run() send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds") if debug: f = open(m + ".log", "w+") @@ -360,7 +349,6 @@ def restart(pid_path): if __name__ == "__main__": if sys.platform.startswith("win"): - freeze_support() start(None, verbose = True) else: diff --git a/src/validation-schema.json b/src/validation-schema.json deleted file mode 100644 index 3b4c95c..0000000 --- a/src/validation-schema.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "persistent": { - "type": "dict", - "require_all": true, - "schema": { - "key": { - "type": "dict", - "require_all":true, - "schema": { - "database": {"type":"string"}, - "tba": {"type": "string"}, - "tra": { - "type": "dict", - "require_all": true, - "schema": { - "CLIENT_ID": {"type": "string"}, - "CLIENT_SECRET": {"type": "string"}, - "url": {"type": "string"} - } - } - } - }, - "config-preference": {"type": "string", "required": true}, - "synchronize-config": {"type": "boolean", "required": true} - } - } -} From b1eff19ea4c505692d2ecd21ba6b73b58f4eb7bc Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 19 Feb 2022 06:46:39 +0000 Subject: [PATCH 114/143] removec __all__ property from superscript Former-commit-id: 67c4684b5943467cc07b8cc638dc438ad0ee3e54 --- src/superscript.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/superscript.py b/src/superscript.py index 1cb8bff..8d9c4b9 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -147,11 +147,6 @@ __author__ = ( "Jacob Levine ", ) -__all__ = [ - "load_config", - "save_config", -] - # imports: import json From 2130182212176533e4ac418553568f60d5d1854a Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 19 Feb 2022 06:50:34 +0000 Subject: [PATCH 115/143] fixed pull APIError calls (removed endpoint arg) Former-commit-id: 5090eefdbd5b13bfda285d5480e79896dd3610a5 --- src/pull.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pull.py b/src/pull.py index 09127b0..2996267 100644 --- a/src/pull.py +++ b/src/pull.py @@ -19,7 +19,7 @@ def get_team_competition(): if json['success']: return json['competition'] else: - raise APIError(json, endpoint) + raise APIError(json) def get_team(): endpoint = '/api/fetchTeamCompetition' @@ -32,7 +32,7 @@ def get_team(): if json['success']: return json['team'] else: - raise APIError(json, endpoint) + raise APIError(json) def get_team_match_data(competition, team_num): endpoint = '/api/fetchAllTeamMatchData' @@ -47,7 +47,7 @@ def get_team_match_data(competition, team_num): if json['success']: return json['data'][team_num] else: - raise APIError(json, endpoint) + raise APIError(json) def get_teams_at_competition(competition): endpoint = '/api/fetchAllTeamNicknamesAtCompetition' @@ -61,4 +61,4 @@ def get_teams_at_competition(competition): if json['success']: return list(json['data'].keys()) else: - raise APIError(json, endpoint) \ No newline at end of file + raise APIError(json) \ No newline at end of file From 56447603e1fbe7b4e520d828297b0eada7fbf807 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 19 Feb 2022 08:52:49 +0000 Subject: [PATCH 116/143] implemented logger, dropped zmq because of constant inconsistency, daemon process now outputs to file, deleted client.py Former-commit-id: d3251e5f6404d0dce42da49e5d577651e718e168 --- src/config.py | 15 +++--- src/interface.py | 113 ++++++++++++++++++++++++++++++-------------- src/superscript.py | 115 +++++++++++++++++++++++---------------------- test/client.py | 14 ------ 4 files changed, 144 insertions(+), 113 deletions(-) delete mode 100644 test/client.py diff --git a/src/config.py b/src/config.py index 46493a7..006039c 100644 --- a/src/config.py +++ b/src/config.py @@ -3,7 +3,6 @@ from exceptions import ConfigurationError from cerberus import Validator from data import set_database_config, get_database_config -from interface import stderr, stdout, INF, ERR class Configuration: @@ -218,34 +217,34 @@ class Configuration: def __getitem__(self, key): return self.config[key] - def resolve_config_conflicts(self, send, client): # needs improvement with new localization scheme + def resolve_config_conflicts(self, logger, client): # needs improvement with new localization scheme sync = self.sync priority = self.priority if sync: if priority == "local" or priority == "client": - send(stdout, INF, "config-preference set to local/client, loading local config information") + logger.info("config-preference set to local/client, loading local config information") remote_config = get_database_config(client) if remote_config != self.config["variable"]: set_database_config(client, self.config["variable"]) - send(stdout, INF, "database config was different and was updated") + logger.info("database config was different and was updated") # no change to config elif priority == "remote" or priority == "database": - send(stdout, INF, "config-preference set to remote/database, loading remote config information") + logger.info("config-preference set to remote/database, loading remote config information") remote_config = get_database_config(client) if remote_config != self.config["variable"]: self.config["variable"] = remote_config self.save_config() # change variable to match remote - send(stdout, INF, "local config was different and was updated") + logger.info("local config was different and was updated") else: raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"") else: if priority == "local" or priority == "client": - send(stdout, INF, "config-preference set to local/client, loading local config information") + logger.info("config-preference set to local/client, loading local config information") # no change to config elif priority == "remote" or priority == "database": - send(stdout, INF, "config-preference set to remote/database, loading database config information") + logger.info("config-preference set to remote/database, loading database config information") self.config["variable"] = get_database_config(client) # change variable to match remote without updating local version else: diff --git a/src/interface.py b/src/interface.py index 3754e77..4bd7f5a 100644 --- a/src/interface.py +++ b/src/interface.py @@ -1,44 +1,87 @@ -import sys -import time -from os import system, name +from logging import Logger as L +import datetime import platform +import json -empty_delim = " " -hard_divided_delim = "|" -soft_divided_delim = "|" -l_brack = "[" -r_brack = "]" +class Logger(L): -ERR = "[ERR]" -INF = "[INF]" + send = None + file = None + debug = False -stdout = sys.stdout -stderr = sys.stderr + levels = { + 10:"[DEBUG] ", + 20:"[INFO] ", + 30:"[WARNING] ", + 40:"[ERROR] ", + 50:"[CRITICAL]", + } -def log(target, level, message, code = 0): + def __init__(self, verbose, profile, debug, file = None): + super().__init__("tra_logger") + self.debug = debug + self.file = file + if profile: + self.send = self._send_null + elif verbose: + self.send = self._send_scli + elif debug: + self.send = self._send_scli + elif file != None: + self.send = self._send_file + else: + self.send = self._send_null - message = time.ctime() + empty_delim + str(level) + l_brack + f"{code:+05}" + r_brack + empty_delim + soft_divided_delim + empty_delim + message - print(message, file = target) + def _send_null(self, msg): + pass -def clear(): - if name == "nt": - system("cls") - else: - system("clear") + def _send_scli(self, msg): + print(msg) -def splash(version): + def _send_file(self, msg): + f = open(self.file, 'a') + f.write(msg + "\n") + f.close() - def hrule(): - print("#"+38*"-"+"#") - def box(s): - temp = "|" - temp += s - temp += (40-len(s)-2)*" " - temp += "|" - print(temp) - - hrule() - box(" superscript version: " + version) - box(" os: " + platform.system()) - box(" python: " + platform.python_version()) - hrule() \ No newline at end of file + def get_time_formatted(self): + return datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S %Z") + + def log(self, level, msg): + self.send(self.get_time_formatted() + "| " + self.levels[level] + ": " + msg) + + def debug(self, msg): + self.log(10, msg) + + def info(self, msg): + self.log(20, msg) + + def warning(self, msg): + self.log(30, msg) + + def error(self, msg): + self.log(40, msg) + + def critical(self, msg): + self.log(50, msg) + + def splash(self, version): + + def hrule(): + self.send("#"+38*"-"+"#") + def box(s): + temp = "|" + temp += s + temp += (40-len(s)-2)*" " + temp += "|" + self.send(temp) + + hrule() + box(" superscript version: " + version) + box(" os: " + platform.system()) + box(" python: " + platform.python_version()) + hrule() + + def save_module_to_file(self, module, data, results): + f = open(module + ".log", "w") + json.dump({"data": data, "results":results}, f, ensure_ascii=False, indent=4) + f.close() \ No newline at end of file diff --git a/src/superscript.py b/src/superscript.py index 8d9c4b9..e67692d 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -149,32 +149,27 @@ __author__ = ( # imports: -import json import os, sys, time import pymongo # soon to be deprecated import traceback import warnings -import zmq from config import Configuration, ConfigurationError from data import get_previous_time, set_current_time, check_new_database_matches -from interface import splash, log, ERR, INF, stdout, stderr +from interface import Logger from module import Match, Metric, Pit config_path = "config.json" -def main(send, verbose = False, profile = False, debug = False): +def main(logger, verbose, profile, debug, socket_send = None): def close_all(): if "client" in locals(): client.close() - if "f" in locals(): - f.close() warnings.filterwarnings("ignore") exit_code = 0 - if verbose: - splash(__version__) + logger.splash(__version__) modules = {"match": Match, "metric": Metric, "pit": Pit} @@ -184,23 +179,25 @@ def main(send, verbose = False, profile = False, debug = False): loop_start = time.time() - send(stdout, INF, "current time: " + str(loop_start)) + logger.info("current time: " + str(loop_start)) config = Configuration(config_path) - send(stdout, INF, "found and loaded config at <" + config_path + ">") + logger.info("found and loaded config at <" + config_path + ">") apikey, tbakey = config.database, config.tba - send(stdout, INF, "found and loaded database and tba keys") + logger.info("found and loaded database and tba keys") client = pymongo.MongoClient(apikey) - send(stdout, INF, "established connection to database") - previous_time = get_previous_time(client) - send(stdout, INF, "analysis backtimed to: " + str(previous_time)) + logger.info("established connection to database") - config.resolve_config_conflicts(send, client) + previous_time = get_previous_time(client) + + logger.info("analysis backtimed to: " + str(previous_time)) + + config.resolve_config_conflicts(logger, client) config_modules, competition = config.modules, config.competition @@ -212,70 +209,73 @@ def main(send, verbose = False, profile = False, debug = False): if not valid: continue current_module.run() - send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds") + logger.info(m + " module finished in " + str(time.time() - start) + " seconds") if debug: - f = open(m + ".log", "w+") - json.dump({"data": current_module.data, "results":current_module.results}, f, ensure_ascii=False, indent=4) - f.close() + logger.save_module_to_file(m, current_module.data, current_module.results) # logging flag check done in logger set_current_time(client, loop_start) close_all() - send(stdout, INF, "closed threads and database client") - send(stdout, INF, "finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") + logger.info("closed threads and database client") + logger.info("finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") if profile: exit_code = 0 break + if debug: + exit_code = 0 + break + event_delay = config["variable"]["event-delay"] if event_delay: - send(stdout, INF, "loop delayed until database returns new matches") + logger.info("loop delayed until database returns new matches") new_match = False while not new_match: time.sleep(1) new_match = check_new_database_matches(client, competition) - send(stdout, INF, "database returned new matches") + logger.info("database returned new matches") else: loop_delay = float(config["variable"]["loop-delay"]) remaining_time = loop_delay - (time.time() - loop_start) if remaining_time > 0: - send(stdout, INF, "loop delayed by " + str(remaining_time) + " seconds") + logger.info("loop delayed by " + str(remaining_time) + " seconds") time.sleep(remaining_time) except KeyboardInterrupt: - send(stdout, INF, "detected KeyboardInterrupt, killing threads") + logger.info("detected KeyboardInterrupt, killing threads") close_all() - send(stdout, INF, "terminated threads, exiting") + logger.info("terminated threads, exiting") break except ConfigurationError as e: - send(stderr, ERR, "encountered a configuration error: " + str(e), code = e.code) - traceback.print_exc(file = stderr) + logger.error("encountered a configuration error: " + str(e)) + logger.error("".join(traceback.format_exception(e))) + #traceback.print_exc(file = stderr) exit_code = 1 close_all() break except Exception as e: - send(stderr, ERR, "encountered an exception while running", code = 1) - traceback.print_exc(file = stderr) + logger.error("encountered an exception while running") + logger.error("".join(traceback.format_exception(e))) + #traceback.print_exc(file = stderr) exit_code = 1 close_all() break return exit_code -def start(pid_path, verbose = False, profile = False, debug = False): +def start(pid_path, verbose, profile, debug): if profile: - def send(target, level, message, code = 0): - pass + logger = Logger(verbose, profile, debug) import cProfile, pstats, io profile = cProfile.Profile() profile.enable() - exit_code = main(send, profile = True) + exit_code = main(logger, verbose, profile, debug) profile.disable() f = open("profile.txt", 'w+') ps = pstats.Stats(profile, stream = f).sort_stats('cumtime') @@ -284,35 +284,38 @@ def start(pid_path, verbose = False, profile = False, debug = False): elif verbose: - exit_code = main(log, verbose = verbose) + logger = Logger(verbose, profile, debug) + + exit_code = main(logger, verbose, profile, debug) sys.exit(exit_code) elif debug: - exit_code = main(log, verbose = True, profile = True, debug = debug) + logger = Logger(verbose, profile, debug) + + exit_code = main(logger, verbose, profile, debug) sys.exit(exit_code) else: - - f = open('errorlog.log', 'w+') + + logfile = "logfile.log" + + f = open(logfile, 'w+') + f.close() + + e = open('errorlog.log', 'w+') with daemon.DaemonContext( - working_directory = os.getcwd(), - pidfile = pidfile.TimeoutPIDLockFile(pid_path), - stderr = f + working_directory = os.getcwd(), + pidfile = pidfile.TimeoutPIDLockFile(pid_path), + stderr = e ): - context = zmq.Context() - socket = context.socket(zmq.PUB) - socket.bind("tcp://*:5678") + logger = Logger(verbose, profile, debug, file = logfile) - socket.send(b'status') + exit_code = main(logger, verbose, profile, debug) - def send(target, level, message, code = 0): - socket.send(bytes("status: " + message, 'utf-8')) - - exit_code = main(send) - socket.close() f.close() + sys.exit(exit_code) def stop(pid_path): @@ -334,12 +337,12 @@ def stop(pid_path): if os.path.exists(pid_path): os.remove(pid_path) else: - traceback.print_exc(file = stderr) + traceback.print_exc(file = sys.stderr) sys.exit(1) def restart(pid_path): stop(pid_path) - start(pid_path) + start(pid_path, False, False, False) if __name__ == "__main__": @@ -353,17 +356,17 @@ if __name__ == "__main__": pid_path = "tra-daemon.pid" if len(sys.argv) == 2: if 'start' == sys.argv[1]: - start(pid_path) + start(pid_path, False, False, False) elif 'stop' == sys.argv[1]: stop(pid_path) elif 'restart' == sys.argv[1]: restart(pid_path) elif 'verbose' == sys.argv[1]: - start(None, verbose = True) + start(None, True, False, False) elif 'profile' == sys.argv[1]: - start(None, profile=True) + start(None, False, True, False) elif 'debug' == sys.argv[1]: - start(None, debug = True) + start(None, False, False, True) else: print("usage: %s start|stop|restart|verbose|profile|debug" % sys.argv[0]) sys.exit(2) diff --git a/test/client.py b/test/client.py deleted file mode 100644 index df4c0e4..0000000 --- a/test/client.py +++ /dev/null @@ -1,14 +0,0 @@ -import signal -import zmq - -signal.signal(signal.SIGINT, signal.SIG_DFL) - -context = zmq.Context() - -socket = context.socket(zmq.SUB) -socket.connect('tcp://localhost:5678') -socket.setsockopt(zmq.SUBSCRIBE, b'status') - -while True: - message = socket.recv_multipart() - print(f'Received: {message}') \ No newline at end of file From 80b63269df168229e91e4f0b4520af141f75b226 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 19 Feb 2022 22:58:58 +0000 Subject: [PATCH 117/143] readded zmq messaging Former-commit-id: e38653c4dc4ffe6aada19cac2251c3ec28bfbc2b --- src/interface.py | 28 +++++++++++++---------- src/superscript.py | 56 +++++++++++++++++++++++++++++++++++++--------- test/test_zmq.py | 14 ++++++++++++ 3 files changed, 75 insertions(+), 23 deletions(-) create mode 100644 test/test_zmq.py diff --git a/src/interface.py b/src/interface.py index 4bd7f5a..5eef26b 100644 --- a/src/interface.py +++ b/src/interface.py @@ -5,11 +5,10 @@ import json class Logger(L): - send = None file = None - debug = False levels = { + 0: "", 10:"[DEBUG] ", 20:"[INFO] ", 30:"[WARNING] ", @@ -17,20 +16,24 @@ class Logger(L): 50:"[CRITICAL]", } + targets = [] + def __init__(self, verbose, profile, debug, file = None): super().__init__("tra_logger") - self.debug = debug + self.file = file + + if file != None: + self.targets.append(self._send_file) + if profile: - self.send = self._send_null + self.targets.append(self._send_null) elif verbose: - self.send = self._send_scli + self.targets.append(self._send_scli) elif debug: - self.send = self._send_scli - elif file != None: - self.send = self._send_file + self.targets.append(self._send_scli) else: - self.send = self._send_null + self.targets.append(self._send_null) def _send_null(self, msg): pass @@ -47,7 +50,8 @@ class Logger(L): return datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S %Z") def log(self, level, msg): - self.send(self.get_time_formatted() + "| " + self.levels[level] + ": " + msg) + for t in self.targets: + t(self.get_time_formatted() + "| " + self.levels[level] + ": " + msg) def debug(self, msg): self.log(10, msg) @@ -67,13 +71,13 @@ class Logger(L): def splash(self, version): def hrule(): - self.send("#"+38*"-"+"#") + self.log(0, "#"+38*"-"+"#") def box(s): temp = "|" temp += s temp += (40-len(s)-2)*" " temp += "|" - self.send(temp) + self.log(0, temp) hrule() box(" superscript version: " + version) diff --git a/src/superscript.py b/src/superscript.py index e67692d..59e01d0 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -157,6 +157,7 @@ from config import Configuration, ConfigurationError from data import get_previous_time, set_current_time, check_new_database_matches from interface import Logger from module import Match, Metric, Pit +import zmq config_path = "config.json" @@ -180,22 +181,27 @@ def main(logger, verbose, profile, debug, socket_send = None): loop_start = time.time() logger.info("current time: " + str(loop_start)) + socket_send("current time: " + str(loop_start)) config = Configuration(config_path) logger.info("found and loaded config at <" + config_path + ">") + socket_send("found and loaded config at <" + config_path + ">") apikey, tbakey = config.database, config.tba logger.info("found and loaded database and tba keys") + socket_send("found and loaded database and tba keys") client = pymongo.MongoClient(apikey) logger.info("established connection to database") + socket_send("established connection to database") previous_time = get_previous_time(client) logger.info("analysis backtimed to: " + str(previous_time)) + socket_send("analysis backtimed to: " + str(previous_time)) config.resolve_config_conflicts(logger, client) @@ -210,6 +216,7 @@ def main(logger, verbose, profile, debug, socket_send = None): continue current_module.run() logger.info(m + " module finished in " + str(time.time() - start) + " seconds") + socket_send(m + " module finished in " + str(time.time() - start) + " seconds") if debug: logger.save_module_to_file(m, current_module.data, current_module.results) # logging flag check done in logger @@ -218,6 +225,8 @@ def main(logger, verbose, profile, debug, socket_send = None): logger.info("closed threads and database client") logger.info("finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") + socket_send("closed threads and database client") + socket_send("finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") if profile: exit_code = 0 @@ -230,36 +239,43 @@ def main(logger, verbose, profile, debug, socket_send = None): event_delay = config["variable"]["event-delay"] if event_delay: logger.info("loop delayed until database returns new matches") + socket_send("loop delayed until database returns new matches") new_match = False while not new_match: time.sleep(1) new_match = check_new_database_matches(client, competition) logger.info("database returned new matches") + socket_send("database returned new matches") else: loop_delay = float(config["variable"]["loop-delay"]) remaining_time = loop_delay - (time.time() - loop_start) if remaining_time > 0: logger.info("loop delayed by " + str(remaining_time) + " seconds") + socket_send("loop delayed by " + str(remaining_time) + " seconds") time.sleep(remaining_time) except KeyboardInterrupt: - logger.info("detected KeyboardInterrupt, killing threads") close_all() - logger.info("terminated threads, exiting") + logger.info("detected KeyboardInterrupt, exiting") + socket_send("detected KeyboardInterrupt, exiting") break except ConfigurationError as e: + str_e = "".join(traceback.format_exception(e)) logger.error("encountered a configuration error: " + str(e)) - logger.error("".join(traceback.format_exception(e))) - #traceback.print_exc(file = stderr) + logger.error(str_e) + socket_send("encountered a configuration error: " + str(e)) + socket_send(str_e) exit_code = 1 close_all() break except Exception as e: + str_e = "".join(traceback.format_exception(e)) logger.error("encountered an exception while running") - logger.error("".join(traceback.format_exception(e))) - #traceback.print_exc(file = stderr) + logger.error(str_e) + socket_send("encountered an exception while running") + socket_send(str_e) exit_code = 1 close_all() break @@ -270,12 +286,15 @@ def start(pid_path, verbose, profile, debug): if profile: + def send(msg): + pass + logger = Logger(verbose, profile, debug) import cProfile, pstats, io profile = cProfile.Profile() profile.enable() - exit_code = main(logger, verbose, profile, debug) + exit_code = main(logger, verbose, profile, debug, socket_send = send) profile.disable() f = open("profile.txt", 'w+') ps = pstats.Stats(profile, stream = f).sort_stats('cumtime') @@ -284,16 +303,22 @@ def start(pid_path, verbose, profile, debug): elif verbose: + def send(msg): + pass + logger = Logger(verbose, profile, debug) - exit_code = main(logger, verbose, profile, debug) + exit_code = main(logger, verbose, profile, debug, socket_send = send) sys.exit(exit_code) elif debug: + def send(msg): + pass + logger = Logger(verbose, profile, debug) - exit_code = main(logger, verbose, profile, debug) + exit_code = main(logger, verbose, profile, debug, socket_send = send) sys.exit(exit_code) else: @@ -310,12 +335,21 @@ def start(pid_path, verbose, profile, debug): stderr = e ): + context = zmq.Context() + socket = context.socket(zmq.PUB) + socket.bind("tcp://*:5678") + socket.send(b'status') + + def send(msg): + socket.send(bytes("status: " + msg, "utf-8")) + logger = Logger(verbose, profile, debug, file = logfile) - exit_code = main(logger, verbose, profile, debug) + exit_code = main(logger, verbose, profile, debug, socket_send = send) + socket.close() f.close() - + sys.exit(exit_code) def stop(pid_path): diff --git a/test/test_zmq.py b/test/test_zmq.py new file mode 100644 index 0000000..df4c0e4 --- /dev/null +++ b/test/test_zmq.py @@ -0,0 +1,14 @@ +import signal +import zmq + +signal.signal(signal.SIGINT, signal.SIG_DFL) + +context = zmq.Context() + +socket = context.socket(zmq.SUB) +socket.connect('tcp://localhost:5678') +socket.setsockopt(zmq.SUBSCRIBE, b'status') + +while True: + message = socket.recv_multipart() + print(f'Received: {message}') \ No newline at end of file From 0cace3cec3652f419aa781aeb14e77670bf4c77c Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 19 Feb 2022 23:29:58 +0000 Subject: [PATCH 118/143] improved exit code reporting Former-commit-id: f36f45bc6a2773b8a821d0e046ce1b47fc4ced0f --- src/superscript.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/superscript.py b/src/superscript.py index 59e01d0..fa032aa 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -168,7 +168,6 @@ def main(logger, verbose, profile, debug, socket_send = None): client.close() warnings.filterwarnings("ignore") - exit_code = 0 logger.splash(__version__) @@ -229,12 +228,10 @@ def main(logger, verbose, profile, debug, socket_send = None): socket_send("finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") if profile: - exit_code = 0 - break + return 0 if debug: - exit_code = 0 - break + return 0 event_delay = config["variable"]["event-delay"] if event_delay: @@ -258,7 +255,7 @@ def main(logger, verbose, profile, debug, socket_send = None): close_all() logger.info("detected KeyboardInterrupt, exiting") socket_send("detected KeyboardInterrupt, exiting") - break + return 0 except ConfigurationError as e: str_e = "".join(traceback.format_exception(e)) @@ -266,9 +263,8 @@ def main(logger, verbose, profile, debug, socket_send = None): logger.error(str_e) socket_send("encountered a configuration error: " + str(e)) socket_send(str_e) - exit_code = 1 close_all() - break + return 1 except Exception as e: str_e = "".join(traceback.format_exception(e)) @@ -276,11 +272,8 @@ def main(logger, verbose, profile, debug, socket_send = None): logger.error(str_e) socket_send("encountered an exception while running") socket_send(str_e) - exit_code = 1 close_all() - break - - return exit_code + return 1 def start(pid_path, verbose, profile, debug): From 21d92e65b21de16af3aafe653a9d28c455482bab Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sun, 20 Feb 2022 22:53:11 +0000 Subject: [PATCH 119/143] add binary paths to devcontainer.json Former-commit-id: a323f3161d6ea326b0215d5ed5ebbadae0f38597 --- .devcontainer/devcontainer.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f62f520..e8c7a57 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,11 +5,11 @@ }, "settings": { "terminal.integrated.shell.linux": "/bin/bash", - "python.pythonPath": "", + "python.pythonPath": "/usr/local/bin/python", "python.linting.enabled": true, "python.linting.pylintEnabled": true, - "python.linting.pylintPath": "", - "python.testing.pytestPath": "", + "python.linting.pylintPath": "/usr/local/bin/pylint", + "python.testing.pytestPath": "/usr/local/bin/pytest", "editor.tabSize": 4, "editor.insertSpaces": false }, From d7ed695ad11c551695237205fc38e9c2835b455f Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sun, 20 Feb 2022 23:16:02 +0000 Subject: [PATCH 120/143] fixed build script, added binutils to docker container for build requirements, added required scipy modules to superscript.spec Former-commit-id: 60a6fc1106dc90514bd1270138ab9166efa29290 --- .devcontainer/Dockerfile | 2 +- build/build-CLI.bat | 2 +- build/build-CLI.sh | 2 +- src/superscript.spec | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f762d03..9992aaa 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,6 @@ FROM python:slim WORKDIR / RUN apt-get -y update; apt-get -y upgrade -RUN apt-get -y install git +RUN apt-get -y install git binutils COPY requirements.txt . RUN pip install -r requirements.txt \ No newline at end of file diff --git a/build/build-CLI.bat b/build/build-CLI.bat index 29199e2..d5be7f8 100644 --- a/build/build-CLI.bat +++ b/build/build-CLI.bat @@ -1,4 +1,4 @@ -set pathtospec="../src/cli/superscript.spec" +set pathtospec="../src/superscript.spec" set pathtodist="../dist/" set pathtowork="temp/" diff --git a/build/build-CLI.sh b/build/build-CLI.sh index e328718..82f275e 100644 --- a/build/build-CLI.sh +++ b/build/build-CLI.sh @@ -1,4 +1,4 @@ -pathtospec="../src/cli/superscript.spec" +pathtospec="../src/superscript.spec" pathtodist="../dist/" pathtowork="temp/" diff --git a/src/superscript.spec b/src/superscript.spec index 5d09620..fdde061 100644 --- a/src/superscript.spec +++ b/src/superscript.spec @@ -9,13 +9,15 @@ a = Analysis(['superscript.py'], hiddenimports=[ "dnspython", "sklearn.utils._weight_vector", + "sklearn.utils._typedefs", + "sklearn.neighbors._partition_nodes", "requests", ], hookspath=[], runtime_hooks=[], excludes=[ "matplotlib", - "pandas" + #"pandas" ], win_no_prefer_redirects=False, win_private_assemblies=False, From 9a1a45f1c9573aa2e08493321350f8ba79845e4d Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sun, 20 Feb 2022 23:38:18 +0000 Subject: [PATCH 121/143] removed cert verification for requests library Former-commit-id: 4914b98a21478a10be7d2f737dd6d5669b4c5681 --- src/data.py | 2 +- src/pull.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/data.py b/src/data.py index f8d80fa..dac7f9a 100644 --- a/src/data.py +++ b/src/data.py @@ -51,7 +51,7 @@ def get_metrics_data_formatted(client, competition): return out def get_pit_data_formatted(client, competition): - x=requests.get("https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) + x=requests.get("https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition, verify=False) x = x.json() x = x['data'] x = x.keys() diff --git a/src/pull.py b/src/pull.py index 2996267..ee65571 100644 --- a/src/pull.py +++ b/src/pull.py @@ -1,5 +1,4 @@ import requests -import json from exceptions import APIError from dep import load_config @@ -14,7 +13,7 @@ def get_team_competition(): "CLIENT_ID": trakey['CLIENT_ID'], "CLIENT_SECRET": trakey['CLIENT_SECRET'] } - response = requests.request("GET", url + endpoint, params=params) + response = requests.request("GET", url + endpoint, verify=False, params=params) json = response.json() if json['success']: return json['competition'] @@ -27,7 +26,7 @@ def get_team(): "CLIENT_ID": trakey['CLIENT_ID'], "CLIENT_SECRET": trakey['CLIENT_SECRET'] } - response = requests.request("GET", url + endpoint, params=params) + response = requests.request("GET", url + endpoint, verify=False, params=params) json = response.json() if json['success']: return json['team'] @@ -42,7 +41,7 @@ def get_team_match_data(competition, team_num): "CLIENT_ID": trakey['CLIENT_ID'], "CLIENT_SECRET": trakey['CLIENT_SECRET'] } - response = requests.request("GET", url + endpoint, params=params) + response = requests.request("GET", url + endpoint, verify=False, params=params) json = response.json() if json['success']: return json['data'][team_num] @@ -56,7 +55,7 @@ def get_teams_at_competition(competition): "CLIENT_ID": trakey['CLIENT_ID'], "CLIENT_SECRET": trakey['CLIENT_SECRET'] } - response = requests.request("GET", url + endpoint, params=params) + response = requests.request("GET", url + endpoint, verify=False, params=params) json = response.json() if json['success']: return list(json['data'].keys()) From 4836f48a34af6aec25907709a19bf7df54608192 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 24 Feb 2022 02:56:08 +0000 Subject: [PATCH 122/143] pyinstaller onedir workaround for requests issue, moved spec file to build folder Former-commit-id: 4427eba5c993222cd2ebfdfc76a4685e25e18b0c --- .gitignore | 3 +-- build/build-CLI.sh | 2 +- build/superscript.spec | 50 ++++++++++++++++++++++++++++++++++++++++++ src/data.py | 4 ++-- src/pull.py | 8 +++---- src/superscript.spec | 41 ---------------------------------- 6 files changed, 58 insertions(+), 50 deletions(-) create mode 100644 build/superscript.spec delete mode 100644 src/superscript.spec diff --git a/.gitignore b/.gitignore index 0c62b1f..d4dcdea 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,4 @@ **/*.log **/errorlog.txt -/dist/superscript.* -/dist/superscript \ No newline at end of file +/dist/* \ No newline at end of file diff --git a/build/build-CLI.sh b/build/build-CLI.sh index 82f275e..41b2c8b 100644 --- a/build/build-CLI.sh +++ b/build/build-CLI.sh @@ -1,4 +1,4 @@ -pathtospec="../src/superscript.spec" +pathtospec="superscript.spec" pathtodist="../dist/" pathtowork="temp/" diff --git a/build/superscript.spec b/build/superscript.spec new file mode 100644 index 0000000..53c3d37 --- /dev/null +++ b/build/superscript.spec @@ -0,0 +1,50 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + +a = Analysis( + ['../src/superscript.py'], + pathex=[], + binaries=[], + datas=[], + hiddenimports=['dnspython', 'sklearn.utils._weight_vector', 'sklearn.utils._typedefs', 'sklearn.neighbors._partition_nodes', 'requests'], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=['matplotlib'], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False +) +pyz = PYZ( + a.pure, + a.zipped_data, + cipher=block_cipher +) +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='superscript', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=True, + disable_windowed_traceback=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None +) +coll = COLLECT( + exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='superscript' +) \ No newline at end of file diff --git a/src/data.py b/src/data.py index dac7f9a..1eb6447 100644 --- a/src/data.py +++ b/src/data.py @@ -4,7 +4,7 @@ import pandas as pd def pull_new_tba_matches(apikey, competition, cutoff): api_key= apikey - x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth-Key":api_key}, verify=False) + x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth-Key":api_key}) out = [] for i in x.json(): if i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm": @@ -51,7 +51,7 @@ def get_metrics_data_formatted(client, competition): return out def get_pit_data_formatted(client, competition): - x=requests.get("https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition, verify=False) + x=requests.get("https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) x = x.json() x = x['data'] x = x.keys() diff --git a/src/pull.py b/src/pull.py index ee65571..9f5af43 100644 --- a/src/pull.py +++ b/src/pull.py @@ -13,7 +13,7 @@ def get_team_competition(): "CLIENT_ID": trakey['CLIENT_ID'], "CLIENT_SECRET": trakey['CLIENT_SECRET'] } - response = requests.request("GET", url + endpoint, verify=False, params=params) + response = requests.request("GET", url + endpoint, params=params) json = response.json() if json['success']: return json['competition'] @@ -26,7 +26,7 @@ def get_team(): "CLIENT_ID": trakey['CLIENT_ID'], "CLIENT_SECRET": trakey['CLIENT_SECRET'] } - response = requests.request("GET", url + endpoint, verify=False, params=params) + response = requests.request("GET", url + endpoint, params=params) json = response.json() if json['success']: return json['team'] @@ -41,7 +41,7 @@ def get_team_match_data(competition, team_num): "CLIENT_ID": trakey['CLIENT_ID'], "CLIENT_SECRET": trakey['CLIENT_SECRET'] } - response = requests.request("GET", url + endpoint, verify=False, params=params) + response = requests.request("GET", url + endpoint, params=params) json = response.json() if json['success']: return json['data'][team_num] @@ -55,7 +55,7 @@ def get_teams_at_competition(competition): "CLIENT_ID": trakey['CLIENT_ID'], "CLIENT_SECRET": trakey['CLIENT_SECRET'] } - response = requests.request("GET", url + endpoint, verify=False, params=params) + response = requests.request("GET", url + endpoint, params=params) json = response.json() if json['success']: return list(json['data'].keys()) diff --git a/src/superscript.spec b/src/superscript.spec deleted file mode 100644 index fdde061..0000000 --- a/src/superscript.spec +++ /dev/null @@ -1,41 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- - -block_cipher = None - -a = Analysis(['superscript.py'], - pathex=['/workspaces/tra-data-analysis/src'], - binaries=[], - datas=[], - hiddenimports=[ - "dnspython", - "sklearn.utils._weight_vector", - "sklearn.utils._typedefs", - "sklearn.neighbors._partition_nodes", - "requests", - ], - hookspath=[], - runtime_hooks=[], - excludes=[ - "matplotlib", - #"pandas" - ], - win_no_prefer_redirects=False, - win_private_assemblies=False, - cipher=block_cipher, - noarchive=False) -pyz = PYZ(a.pure, a.zipped_data, - cipher=block_cipher) -exe = EXE(pyz, - a.scripts, - a.binaries, - a.zipfiles, - a.datas, - [('W ignore', None, 'OPTION')], - name='superscript', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - upx_exclude=[], - runtime_tmpdir=None, - console=True ) From 8c28c24d60ee00688afdcbb6bb817d4ed213bd0c Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 14 Mar 2022 01:33:24 +0000 Subject: [PATCH 123/143] removed all unessasary files, moved important files to folder "competition" Former-commit-id: 59becb22abc3305a36e2876351e6c7306e3f551e --- .devcontainer/Dockerfile | 6 --- .devcontainer/devcontainer.json | 22 -------- .github/ISSUE_TEMPLATE/bug_report.md | 38 -------------- .github/ISSUE_TEMPLATE/feature_request.md | 20 -------- .github/workflows/build-cli.yml | 35 ------------- .github/workflows/superscript-unit.yml | 34 ------------- build/build-CLI.bat | 5 -- build/build-CLI.sh | 5 -- build/superscript.spec | 50 ------------------- {src => competition}/config.py | 0 {src => competition}/data.py | 0 {src => competition}/dep.py | 0 {src => competition}/exceptions.py | 0 {src => competition}/interface.py | 0 {src => competition}/module.py | 0 {src => competition}/pull.py | 0 .../requirements.txt | 1 - {src => competition}/superscript.py | 0 test/test_superscript.py | 2 - test/test_zmq.py | 14 ------ 20 files changed, 232 deletions(-) delete mode 100644 .devcontainer/Dockerfile delete mode 100644 .devcontainer/devcontainer.json delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 .github/workflows/build-cli.yml delete mode 100644 .github/workflows/superscript-unit.yml delete mode 100644 build/build-CLI.bat delete mode 100644 build/build-CLI.sh delete mode 100644 build/superscript.spec rename {src => competition}/config.py (100%) rename {src => competition}/data.py (100%) rename {src => competition}/dep.py (100%) rename {src => competition}/exceptions.py (100%) rename {src => competition}/interface.py (100%) rename {src => competition}/module.py (100%) rename {src => competition}/pull.py (100%) rename {.devcontainer => competition}/requirements.txt (85%) rename {src => competition}/superscript.py (100%) delete mode 100644 test/test_superscript.py delete mode 100644 test/test_zmq.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 9992aaa..0000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM python:slim -WORKDIR / -RUN apt-get -y update; apt-get -y upgrade -RUN apt-get -y install git binutils -COPY requirements.txt . -RUN pip install -r requirements.txt \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index e8c7a57..0000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "TRA Analysis Development Environment", - "build": { - "dockerfile": "Dockerfile", - }, - "settings": { - "terminal.integrated.shell.linux": "/bin/bash", - "python.pythonPath": "/usr/local/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.linting.pylintPath": "/usr/local/bin/pylint", - "python.testing.pytestPath": "/usr/local/bin/pytest", - "editor.tabSize": 4, - "editor.insertSpaces": false - }, - "extensions": [ - "mhutchie.git-graph", - "ms-python.python", - "waderyan.gitblame" - ], - "postCreateCommand": "" - } \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index dd84ea7..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - -**Additional context** -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index bbcbbe7..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/build-cli.yml b/.github/workflows/build-cli.yml deleted file mode 100644 index 7b9f8d7..0000000 --- a/.github/workflows/build-cli.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Build Superscript Linux - -on: - release: - types: [published, edited] - -jobs: - generate: - name: Build Linux - runs-on: ubuntu-latest - steps: - - name: Checkout master - uses: actions/checkout@master - - name: Install Dependencies - run: pip install -r requirements.txt - working-directory: src/ - - name: Give Execute Permission - run: chmod +x build-CLI.sh - working-directory: build/ - - name: Build Binary - run: ./build-CLI.sh - working-directory: build/ - - name: Copy Binary to Root Dir - run: cp superscript .. - working-directory: dist/ - - name: Upload Release Asset - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: superscript - asset_name: superscript - tag: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/superscript-unit.yml b/.github/workflows/superscript-unit.yml deleted file mode 100644 index 3a4e4c5..0000000 --- a/.github/workflows/superscript-unit.yml +++ /dev/null @@ -1,34 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Superscript Unit Tests - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.7, 3.8] - - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest - if [ -f src/requirements.txt ]; then pip install -r src/requirements.txt; fi - - name: Test with pytest - run: | - pytest test/ diff --git a/build/build-CLI.bat b/build/build-CLI.bat deleted file mode 100644 index d5be7f8..0000000 --- a/build/build-CLI.bat +++ /dev/null @@ -1,5 +0,0 @@ -set pathtospec="../src/superscript.spec" -set pathtodist="../dist/" -set pathtowork="temp/" - -pyinstaller --clean --distpath %pathtodist% --workpath %pathtowork% %pathtospec% \ No newline at end of file diff --git a/build/build-CLI.sh b/build/build-CLI.sh deleted file mode 100644 index 41b2c8b..0000000 --- a/build/build-CLI.sh +++ /dev/null @@ -1,5 +0,0 @@ -pathtospec="superscript.spec" -pathtodist="../dist/" -pathtowork="temp/" - -pyinstaller --clean --distpath ${pathtodist} --workpath ${pathtowork} ${pathtospec} \ No newline at end of file diff --git a/build/superscript.spec b/build/superscript.spec deleted file mode 100644 index 53c3d37..0000000 --- a/build/superscript.spec +++ /dev/null @@ -1,50 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- - -block_cipher = None - -a = Analysis( - ['../src/superscript.py'], - pathex=[], - binaries=[], - datas=[], - hiddenimports=['dnspython', 'sklearn.utils._weight_vector', 'sklearn.utils._typedefs', 'sklearn.neighbors._partition_nodes', 'requests'], - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=['matplotlib'], - win_no_prefer_redirects=False, - win_private_assemblies=False, - cipher=block_cipher, - noarchive=False -) -pyz = PYZ( - a.pure, - a.zipped_data, - cipher=block_cipher -) -exe = EXE( - pyz, - a.scripts, - [], - exclude_binaries=True, - name='superscript', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - console=True, - disable_windowed_traceback=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None -) -coll = COLLECT( - exe, - a.binaries, - a.zipfiles, - a.datas, - strip=False, - upx=True, - upx_exclude=[], - name='superscript' -) \ No newline at end of file diff --git a/src/config.py b/competition/config.py similarity index 100% rename from src/config.py rename to competition/config.py diff --git a/src/data.py b/competition/data.py similarity index 100% rename from src/data.py rename to competition/data.py diff --git a/src/dep.py b/competition/dep.py similarity index 100% rename from src/dep.py rename to competition/dep.py diff --git a/src/exceptions.py b/competition/exceptions.py similarity index 100% rename from src/exceptions.py rename to competition/exceptions.py diff --git a/src/interface.py b/competition/interface.py similarity index 100% rename from src/interface.py rename to competition/interface.py diff --git a/src/module.py b/competition/module.py similarity index 100% rename from src/module.py rename to competition/module.py diff --git a/src/pull.py b/competition/pull.py similarity index 100% rename from src/pull.py rename to competition/pull.py diff --git a/.devcontainer/requirements.txt b/competition/requirements.txt similarity index 85% rename from .devcontainer/requirements.txt rename to competition/requirements.txt index 56ec5dc..4f9a9c7 100644 --- a/.devcontainer/requirements.txt +++ b/competition/requirements.txt @@ -5,7 +5,6 @@ pyinstaller pylint pymongo pyparsing -pytest python-daemon pyzmq requests diff --git a/src/superscript.py b/competition/superscript.py similarity index 100% rename from src/superscript.py rename to competition/superscript.py diff --git a/test/test_superscript.py b/test/test_superscript.py deleted file mode 100644 index f670919..0000000 --- a/test/test_superscript.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_(): - assert 1 == 1 \ No newline at end of file diff --git a/test/test_zmq.py b/test/test_zmq.py deleted file mode 100644 index df4c0e4..0000000 --- a/test/test_zmq.py +++ /dev/null @@ -1,14 +0,0 @@ -import signal -import zmq - -signal.signal(signal.SIGINT, signal.SIG_DFL) - -context = zmq.Context() - -socket = context.socket(zmq.SUB) -socket.connect('tcp://localhost:5678') -socket.setsockopt(zmq.SUBSCRIBE, b'status') - -while True: - message = socket.recv_multipart() - print(f'Received: {message}') \ No newline at end of file From 3c6e3ac58e1b4d1fd219822aceed4dd584a90682 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 14 Mar 2022 01:53:50 +0000 Subject: [PATCH 124/143] added pandas to requirements, readded dev docker files Former-commit-id: 3fa4ef57e5e9542ce245ab0ef9b8320e21c9507c --- .devcontainer/.devcontainer | 22 ++++++++++++++++++++++ .devcontainer/Dockerfile | 6 ++++++ .devcontainer/requirements.txt | 16 ++++++++++++++++ competition/requirements.txt | 1 + 4 files changed, 45 insertions(+) create mode 100644 .devcontainer/.devcontainer create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/requirements.txt diff --git a/.devcontainer/.devcontainer b/.devcontainer/.devcontainer new file mode 100644 index 0000000..06e29f8 --- /dev/null +++ b/.devcontainer/.devcontainer @@ -0,0 +1,22 @@ +{ + "name": "TRA Analysis Development Environment", + "build": { + "dockerfile": "Dockerfile" + }, + "settings": { + "terminal.integrated.shell.linux": "/bin/bash", + "python.pythonPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.linting.pylintPath": "/usr/local/bin/pylint", + "python.testing.pytestPath": "/usr/local/bin/pytest", + "editor.tabSize": 4, + "editor.insertSpaces": false + }, + "extensions": [ + "mhutchie.git-graph", + "ms-python.python", + "waderyan.gitblame" + ], + "postCreateCommand": "" +} \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..9992aaa --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,6 @@ +FROM python:slim +WORKDIR / +RUN apt-get -y update; apt-get -y upgrade +RUN apt-get -y install git binutils +COPY requirements.txt . +RUN pip install -r requirements.txt \ No newline at end of file diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt new file mode 100644 index 0000000..3225fc7 --- /dev/null +++ b/.devcontainer/requirements.txt @@ -0,0 +1,16 @@ +cerberus +dnspython +numpy +pandas +pyinstaller +pylint +pymongo +pyparsing +pytest +python-daemon +pyzmq +requests +scikit-learn +scipy +six +tra-analysis \ No newline at end of file diff --git a/competition/requirements.txt b/competition/requirements.txt index 4f9a9c7..503e263 100644 --- a/competition/requirements.txt +++ b/competition/requirements.txt @@ -1,6 +1,7 @@ cerberus dnspython numpy +pandas pyinstaller pylint pymongo From c9dd09f5e9fb8942126c7005414475eb0da01379 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sun, 13 Mar 2022 18:54:50 -0700 Subject: [PATCH 125/143] fixed file name Former-commit-id: 7d113b378854316c3af5e5c58f589c5c062040b4 --- .devcontainer/{.devcontainer => devcontainer.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .devcontainer/{.devcontainer => devcontainer.json} (100%) diff --git a/.devcontainer/.devcontainer b/.devcontainer/devcontainer.json similarity index 100% rename from .devcontainer/.devcontainer rename to .devcontainer/devcontainer.json From e8a5bb75f80c66f049fcfaa7755855b0ef055d22 Mon Sep 17 00:00:00 2001 From: Dev Date: Sun, 13 Mar 2022 21:56:34 -0500 Subject: [PATCH 126/143] add sbatch script Former-commit-id: f521f7b3f69df71171cd046a40bcbcb6637967a6 --- submit.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 submit.sh diff --git a/submit.sh b/submit.sh new file mode 100644 index 0000000..ab42d9d --- /dev/null +++ b/submit.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +#SBATCH --job-name=tra-superscript +#SBATCH --output=superscript.out +#SBATCH --ntasks=8 +#SBATCH --time=24:00:00 +#SBATCH --mem-per-cpu=256 +#SBATCH --mail-user=dsingh@imsa.edu +#SBATCH -p cpu-long + +unset XDG_RUNTIME_DIR +/home/tools/anaconda3/bin/conda init +source ~/.bashrc +source activate base +cd competition +python superscript.py debug From def2fc9b73505f837f94af45acb7c84442aed597 Mon Sep 17 00:00:00 2001 From: Dev Date: Mon, 14 Mar 2022 20:55:00 -0500 Subject: [PATCH 127/143] change gitignore, add up submit Former-commit-id: 927f0a1a4c3dd0aff6fb4fca5f99ea62bc61584f --- .gitignore | 5 ++++- submit.sh | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d4dcdea..963fb80 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,7 @@ **/*.log **/errorlog.txt -/dist/* \ No newline at end of file +/dist/* + +slurm-tra-superscript.out +config*.json \ No newline at end of file diff --git a/submit.sh b/submit.sh index ab42d9d..fca42ad 100644 --- a/submit.sh +++ b/submit.sh @@ -1,7 +1,7 @@ #!/bin/bash # #SBATCH --job-name=tra-superscript -#SBATCH --output=superscript.out +#SBATCH --output=slurm-tra-superscript.out #SBATCH --ntasks=8 #SBATCH --time=24:00:00 #SBATCH --mem-per-cpu=256 From 143218dda36b1acf0a24b804afd7f90ca79445bc Mon Sep 17 00:00:00 2001 From: Dev Date: Wed, 16 Mar 2022 18:39:30 -0500 Subject: [PATCH 128/143] split sbatch commands --- submit.sh => submit-debug.sh | 3 --- submit-prod.sh | 13 +++++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) rename submit.sh => submit-debug.sh (79%) create mode 100644 submit-prod.sh diff --git a/submit.sh b/submit-debug.sh similarity index 79% rename from submit.sh rename to submit-debug.sh index fca42ad..c924563 100644 --- a/submit.sh +++ b/submit-debug.sh @@ -9,8 +9,5 @@ #SBATCH -p cpu-long unset XDG_RUNTIME_DIR -/home/tools/anaconda3/bin/conda init -source ~/.bashrc -source activate base cd competition python superscript.py debug diff --git a/submit-prod.sh b/submit-prod.sh new file mode 100644 index 0000000..85b54ea --- /dev/null +++ b/submit-prod.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# +#SBATCH --job-name=tra-superscript +#SBATCH --output=slurm-tra-superscript.out +#SBATCH --ntasks=8 +#SBATCH --time=24:00:00 +#SBATCH --mem-per-cpu=256 +#SBATCH --mail-user=dsingh@imsa.edu +#SBATCH -p cpu-long + +unset XDG_RUNTIME_DIR +cd competition +python superscript.py verbose From 8908f05cbe1798f2bc6e08554b9d84fad1cae0c0 Mon Sep 17 00:00:00 2001 From: Dev Date: Wed, 16 Mar 2022 18:51:10 -0500 Subject: [PATCH 129/143] fix prod issues --- submit-debug.sh | 1 - submit-prod.sh | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/submit-debug.sh b/submit-debug.sh index c924563..861af0b 100644 --- a/submit-debug.sh +++ b/submit-debug.sh @@ -8,6 +8,5 @@ #SBATCH --mail-user=dsingh@imsa.edu #SBATCH -p cpu-long -unset XDG_RUNTIME_DIR cd competition python superscript.py debug diff --git a/submit-prod.sh b/submit-prod.sh index 85b54ea..1b58edf 100644 --- a/submit-prod.sh +++ b/submit-prod.sh @@ -1,13 +1,12 @@ #!/bin/bash # #SBATCH --job-name=tra-superscript -#SBATCH --output=slurm-tra-superscript.out +#SBATCH --output=PROD_slurm-tra-superscript.out #SBATCH --ntasks=8 #SBATCH --time=24:00:00 #SBATCH --mem-per-cpu=256 #SBATCH --mail-user=dsingh@imsa.edu #SBATCH -p cpu-long -unset XDG_RUNTIME_DIR cd competition python superscript.py verbose From ef63c1de7e2a148b6b00c4c423b44bbf15e0aedf Mon Sep 17 00:00:00 2001 From: Dev Date: Thu, 24 Mar 2022 16:19:47 -0500 Subject: [PATCH 130/143] add ability to load manual JSON for metrics --- competition/data.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/competition/data.py b/competition/data.py index 1eb6447..da8117a 100644 --- a/competition/data.py +++ b/competition/data.py @@ -1,6 +1,7 @@ import requests import pull import pandas as pd +import json def pull_new_tba_matches(apikey, competition, cutoff): api_key= apikey @@ -11,6 +12,12 @@ def pull_new_tba_matches(apikey, competition, cutoff): out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) return out +def pull_new_tba_matches_manual(apikey, competition, cutoff): + filename = competition+"-wins.json" + with open(filename, 'r') as f: + data = json.load(f) + return data + def get_team_match_data(client, competition, team_num): db = client.data_scouting mdata = db.matchdata From 9752fd323b872107fbe058fc703e70dd33ad2544 Mon Sep 17 00:00:00 2001 From: Dev Date: Fri, 25 Mar 2022 13:39:24 -0500 Subject: [PATCH 131/143] remove time check --- competition/data.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/competition/data.py b/competition/data.py index da8117a..bd833ee 100644 --- a/competition/data.py +++ b/competition/data.py @@ -4,11 +4,12 @@ import pandas as pd import json def pull_new_tba_matches(apikey, competition, cutoff): - api_key= apikey + api_key= apikey x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth-Key":api_key}) + json = x.json() out = [] - for i in x.json(): - if i["actual_time"] != None and i["actual_time"]-cutoff >= 0 and i["comp_level"] == "qm": + for i in json: + if i["actual_time"] != None and i["comp_level"] == "qm": out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) return out @@ -203,4 +204,4 @@ def push_pit(client, competition, pit): def check_new_database_matches(client, competition): - return True \ No newline at end of file + return True From 3fe2922e976dcb11c96e4b2e849097fd086a9400 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 28 Mar 2022 10:13:42 -0500 Subject: [PATCH 132/143] experimental trueskill support --- competition/data.py | 82 +++++++++++++++++++++++++----------- competition/module.py | 40 ++++++++++++++++-- competition/requirements.txt | 3 +- competition/ts_predict.py | 13 ++++++ 4 files changed, 110 insertions(+), 28 deletions(-) create mode 100644 competition/ts_predict.py diff --git a/competition/data.py b/competition/data.py index bd833ee..364867d 100644 --- a/competition/data.py +++ b/competition/data.py @@ -3,27 +3,32 @@ import pull import pandas as pd import json + def pull_new_tba_matches(apikey, competition, cutoff): - api_key= apikey - x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth-Key":api_key}) + api_key = apikey + x = requests.get("https://www.thebluealliance.com/api/v3/event/" + + competition+"/matches/simple", headers={"X-TBA-Auth-Key": api_key}) json = x.json() out = [] for i in json: if i["actual_time"] != None and i["comp_level"] == "qm": - out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) + out.append({"match": i['match_number'], "blue": list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red": list( + map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) return out + def pull_new_tba_matches_manual(apikey, competition, cutoff): filename = competition+"-wins.json" with open(filename, 'r') as f: - data = json.load(f) + data = json.load(f) return data + def get_team_match_data(client, competition, team_num): db = client.data_scouting mdata = db.matchdata out = {} - for i in mdata.find({"competition" : competition, "team_scouted": str(team_num)}): + for i in mdata.find({"competition": competition, "team_scouted": str(team_num)}): out[i['match']] = i['data'] return pd.DataFrame(out) @@ -31,23 +36,27 @@ def get_team_pit_data(client, competition, team_num): db = client.data_scouting mdata = db.pitdata out = {} - return mdata.find_one({"competition" : competition, "team_scouted": str(team_num)})["data"] + return mdata.find_one({"competition": competition, "team_scouted": str(team_num)})["data"] + def get_team_metrics_data(client, competition, team_num): db = client.data_processing mdata = db.team_metrics - return mdata.find_one({"competition" : competition, "team": team_num}) + return mdata.find_one({"competition": competition, "team": team_num}) + def get_match_data_formatted(client, competition): teams_at_comp = pull.get_teams_at_competition(competition) out = {} for team in teams_at_comp: try: - out[int(team)] = unkeyify_2l(get_team_match_data(client, competition, team).transpose().to_dict()) + out[int(team)] = unkeyify_2l(get_team_match_data( + client, competition, team).transpose().to_dict()) except: pass return out + def get_metrics_data_formatted(client, competition): teams_at_comp = pull.get_teams_at_competition(competition) out = {} @@ -58,8 +67,10 @@ def get_metrics_data_formatted(client, competition): pass return out + def get_pit_data_formatted(client, competition): - x=requests.get("https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) + x = requests.get( + "https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) x = x.json() x = x['data'] x = x.keys() @@ -71,12 +82,14 @@ def get_pit_data_formatted(client, competition): pass return out + def get_pit_variable_data(client, competition): db = client.data_processing mdata = db.team_pit out = {} return mdata.find() + def get_pit_variable_formatted(client, competition): temp = get_pit_variable_data(client, competition) out = {} @@ -84,30 +97,39 @@ def get_pit_variable_formatted(client, competition): out[i["variable"]] = i["data"] return out -def push_team_tests_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_tests"): - db = client[dbname] - mdata = db[colname] - mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "data" : data}, True) -def push_team_metrics_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"): +def push_team_tests_data(client, competition, team_num, data, dbname="data_processing", colname="team_tests"): db = client[dbname] mdata = db[colname] - mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True) + mdata.replace_one({"competition": competition, "team": team_num}, {"_id": competition + + str(team_num)+"am", "competition": competition, "team": team_num, "data": data}, True) -def push_team_pit_data(client, competition, variable, data, dbname = "data_processing", colname = "team_pit"): + +def push_team_metrics_data(client, competition, team_num, data, dbname="data_processing", colname="team_metrics"): db = client[dbname] mdata = db[colname] - mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True) + mdata.replace_one({"competition": competition, "team": team_num}, {"_id": competition + + str(team_num)+"am", "competition": competition, "team": team_num, "metrics": data}, True) + + +def push_team_pit_data(client, competition, variable, data, dbname="data_processing", colname="team_pit"): + db = client[dbname] + mdata = db[colname] + mdata.replace_one({"competition": competition, "variable": variable}, { + "competition": competition, "variable": variable, "data": data}, True) + def get_analysis_flags(client, flag): db = client.data_processing mdata = db.flags - return mdata.find_one({flag:{"$exists":True}}) + return mdata.find_one({flag: {"$exists": True}}) + def set_analysis_flags(client, flag, data): db = client.data_processing mdata = db.flags - return mdata.replace_one({flag:{"$exists":True}}, data, True) + return mdata.replace_one({flag: {"$exists": True}}, data, True) + def unkeyify_2l(layered_dict): out = {} @@ -115,11 +137,12 @@ def unkeyify_2l(layered_dict): add = [] sortkey = [] for j in layered_dict[i].keys(): - add.append([j,layered_dict[i][j]]) - add.sort(key = lambda x: x[0]) + add.append([j, layered_dict[i][j]]) + add.sort(key=lambda x: x[0]) out[i] = list(map(lambda x: x[1], add)) return out + def get_previous_time(client): previous_time = get_analysis_flags(client, "latest_update") @@ -135,23 +158,28 @@ def get_previous_time(client): return previous_time + def set_current_time(client, current_time): - set_analysis_flags(client, "latest_update", {"latest_update":current_time}) + set_analysis_flags(client, "latest_update", {"latest_update": current_time}) + def get_database_config(client): remote_config = get_analysis_flags(client, "config") return remote_config["config"] if remote_config != None else None + def set_database_config(client, config): set_analysis_flags(client, "config", {"config": config}) + def load_match(client, competition): return get_match_data_formatted(client, competition) + def load_metric(client, competition, match, group_name, metrics): group = {} @@ -163,7 +191,8 @@ def load_metric(client, competition, match, group_name, metrics): if db_data == None: elo = {"score": metrics["elo"]["score"]} - gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} + gl2 = {"score": metrics["gl2"]["score"], + "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} ts = {"mu": metrics["ts"]["mu"], "sigma": metrics["ts"]["sigma"]} group[team] = {"elo": elo, "gl2": gl2, "ts": ts} @@ -180,28 +209,33 @@ def load_metric(client, competition, match, group_name, metrics): return group + def load_pit(client, competition): return get_pit_data_formatted(client, competition) + def push_match(client, competition, results): for team in results: push_team_tests_data(client, competition, team, results[team]) + def push_metric(client, competition, metric): for team in metric: push_team_metrics_data(client, competition, team, metric[team]) + def push_pit(client, competition, pit): for variable in pit: - + push_team_pit_data(client, competition, variable, pit[variable]) + def check_new_database_matches(client, competition): return True diff --git a/competition/module.py b/competition/module.py index f5d5023..d51d56b 100644 --- a/competition/module.py +++ b/competition/module.py @@ -1,8 +1,10 @@ import abc import data as d import signal +import trueskill import numpy as np from tra_analysis import Analysis as an +from ts_predict import win_probability class Module(metaclass = abc.ABCMeta): @@ -180,9 +182,9 @@ class Metric (Module): red = {} blu = {} - + for match in matches: - + red = d.load_metric(self.apikey, self.competition, match, "red", self.config["tests"]) blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config["tests"]) @@ -198,6 +200,9 @@ class Metric (Module): gl2_red_vol_total = 0 gl2_blu_vol_total = 0 + ts_red_team = {} + ts_blu_team = {} + for team in red: elo_red_total += red[team]["elo"]["score"] @@ -206,6 +211,10 @@ class Metric (Module): gl2_red_rd_total += red[team]["gl2"]["rd"] gl2_red_vol_total += red[team]["gl2"]["vol"] + tmu = red[team]["ts"]["mu"] or 25 + tsigma = red[team]["ts"]["sigma"] or 8.333 + ts_red_team[team] = trueskill.Rating(mu=tmu, sigma=tsigma) + for team in blu: elo_blu_total += blu[team]["elo"]["score"] @@ -214,25 +223,43 @@ class Metric (Module): gl2_blu_rd_total += blu[team]["gl2"]["rd"] gl2_blu_vol_total += blu[team]["gl2"]["vol"] + tmu = blu[team]["ts"]["mu"] or 25 + tsigma = blu[team]["ts"]["sigma"] or 8.333 + ts_blu_team[team] = trueskill.Rating(mu=tmu, sigma=tsigma) + red_elo = {"score": elo_red_total / len(red)} blu_elo = {"score": elo_blu_total / len(blu)} red_gl2 = {"score": gl2_red_score_total / len(red), "rd": gl2_red_rd_total / len(red), "vol": gl2_red_vol_total / len(red)} blu_gl2 = {"score": gl2_blu_score_total / len(blu), "rd": gl2_blu_rd_total / len(blu), "vol": gl2_blu_vol_total / len(blu)} - + print('here') if match["winner"] == "red": observations = {"red": 1, "blu": 0} + ts_obs = [1,0] elif match["winner"] == "blue": observations = {"red": 0, "blu": 1} + ts_obs = [0,1] else: observations = {"red": 0.5, "blu": 0.5} + ts_obs = [0,0] + ts_red = list(ts_red_team.values()) + ts_blu = list(ts_blu_team.values()) + new_red, new_blu = trueskill.rate([ts_red, ts_blu], ranks=ts_obs) + new_red_ts = {} + new_blu_ts = {} + for key, value in zip(ts_red_team.keys(), new_red): + new_red_ts[key] = value + for key, value in zip(ts_blu_team.keys(), new_blu): + new_blu_ts[key] = value + print("red" if win_probability(new_red, new_blu) > 0.5 else "blue", match["winner"]) + # now there are new trueskll ratings for each team based on the win/loss from DB red_elo_delta = an.Metric().elo(red_elo["score"], blu_elo["score"], observations["red"], elo_N, elo_K) - red_elo["score"] blu_elo_delta = an.Metric().elo(blu_elo["score"], red_elo["score"], observations["blu"], elo_N, elo_K) - blu_elo["score"] @@ -250,6 +277,9 @@ class Metric (Module): red[team]["gl2"]["rd"] = red[team]["gl2"]["rd"] + red_gl2_delta["rd"] red[team]["gl2"]["vol"] = red[team]["gl2"]["vol"] + red_gl2_delta["vol"] + red[team]["ts"]["mu"] = new_red_ts[team].mu + red[team]["ts"]["sigma"] = new_red_ts[team].sigma + for team in blu: blu[team]["elo"]["score"] = blu[team]["elo"]["score"] + blu_elo_delta @@ -258,6 +288,10 @@ class Metric (Module): blu[team]["gl2"]["rd"] = blu[team]["gl2"]["rd"] + blu_gl2_delta["rd"] blu[team]["gl2"]["vol"] = blu[team]["gl2"]["vol"] + blu_gl2_delta["vol"] + + blu[team]["ts"]["mu"] = new_blu_ts[team].mu + blu[team]["ts"]["sigma"] = new_blu_ts[team].sigma + temp_vector = {} temp_vector.update(red) temp_vector.update(blu) diff --git a/competition/requirements.txt b/competition/requirements.txt index 503e263..5ea81ce 100644 --- a/competition/requirements.txt +++ b/competition/requirements.txt @@ -12,4 +12,5 @@ requests scikit-learn scipy six -tra-analysis \ No newline at end of file +tra-analysis +trueskill \ No newline at end of file diff --git a/competition/ts_predict.py b/competition/ts_predict.py new file mode 100644 index 0000000..5faae96 --- /dev/null +++ b/competition/ts_predict.py @@ -0,0 +1,13 @@ +from trueskill import Rating +import trueskill +from trueskill import TrueSkill +import math + +BETA = 8.333/2 +cdf = TrueSkill().cdf +def win_probability(a, b): + deltaMu = sum([x.mu for x in a]) - sum([x.mu for x in b]) + sumSigma = sum([x.sigma ** 2 for x in a]) + sum([x.sigma ** 2 for x in b]) + playerCount = len(a) + len(b) + denominator = math.sqrt(playerCount * (BETA * BETA) + sumSigma) + return cdf(deltaMu / denominator) \ No newline at end of file From 25e4babd71c587e2298547eb2cc1316d6be5c8e4 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 28 Mar 2022 14:21:23 -0500 Subject: [PATCH 133/143] Revert "experimental trueskill support" This reverts commit 3fe2922e976dcb11c96e4b2e849097fd086a9400. --- competition/data.py | 78 ++++++++++-------------------------- competition/module.py | 40 ++---------------- competition/requirements.txt | 3 +- competition/ts_predict.py | 13 ------ 4 files changed, 26 insertions(+), 108 deletions(-) delete mode 100644 competition/ts_predict.py diff --git a/competition/data.py b/competition/data.py index 364867d..bd833ee 100644 --- a/competition/data.py +++ b/competition/data.py @@ -3,32 +3,27 @@ import pull import pandas as pd import json - def pull_new_tba_matches(apikey, competition, cutoff): - api_key = apikey - x = requests.get("https://www.thebluealliance.com/api/v3/event/" + - competition+"/matches/simple", headers={"X-TBA-Auth-Key": api_key}) + api_key= apikey + x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth-Key":api_key}) json = x.json() out = [] for i in json: if i["actual_time"] != None and i["comp_level"] == "qm": - out.append({"match": i['match_number'], "blue": list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red": list( - map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) + out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) return out - def pull_new_tba_matches_manual(apikey, competition, cutoff): filename = competition+"-wins.json" with open(filename, 'r') as f: - data = json.load(f) + data = json.load(f) return data - def get_team_match_data(client, competition, team_num): db = client.data_scouting mdata = db.matchdata out = {} - for i in mdata.find({"competition": competition, "team_scouted": str(team_num)}): + for i in mdata.find({"competition" : competition, "team_scouted": str(team_num)}): out[i['match']] = i['data'] return pd.DataFrame(out) @@ -36,27 +31,23 @@ def get_team_pit_data(client, competition, team_num): db = client.data_scouting mdata = db.pitdata out = {} - return mdata.find_one({"competition": competition, "team_scouted": str(team_num)})["data"] - + return mdata.find_one({"competition" : competition, "team_scouted": str(team_num)})["data"] def get_team_metrics_data(client, competition, team_num): db = client.data_processing mdata = db.team_metrics - return mdata.find_one({"competition": competition, "team": team_num}) - + return mdata.find_one({"competition" : competition, "team": team_num}) def get_match_data_formatted(client, competition): teams_at_comp = pull.get_teams_at_competition(competition) out = {} for team in teams_at_comp: try: - out[int(team)] = unkeyify_2l(get_team_match_data( - client, competition, team).transpose().to_dict()) + out[int(team)] = unkeyify_2l(get_team_match_data(client, competition, team).transpose().to_dict()) except: pass return out - def get_metrics_data_formatted(client, competition): teams_at_comp = pull.get_teams_at_competition(competition) out = {} @@ -67,10 +58,8 @@ def get_metrics_data_formatted(client, competition): pass return out - def get_pit_data_formatted(client, competition): - x = requests.get( - "https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) + x=requests.get("https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) x = x.json() x = x['data'] x = x.keys() @@ -82,14 +71,12 @@ def get_pit_data_formatted(client, competition): pass return out - def get_pit_variable_data(client, competition): db = client.data_processing mdata = db.team_pit out = {} return mdata.find() - def get_pit_variable_formatted(client, competition): temp = get_pit_variable_data(client, competition) out = {} @@ -97,39 +84,30 @@ def get_pit_variable_formatted(client, competition): out[i["variable"]] = i["data"] return out - -def push_team_tests_data(client, competition, team_num, data, dbname="data_processing", colname="team_tests"): +def push_team_tests_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_tests"): db = client[dbname] mdata = db[colname] - mdata.replace_one({"competition": competition, "team": team_num}, {"_id": competition + - str(team_num)+"am", "competition": competition, "team": team_num, "data": data}, True) + mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "data" : data}, True) - -def push_team_metrics_data(client, competition, team_num, data, dbname="data_processing", colname="team_metrics"): +def push_team_metrics_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"): db = client[dbname] mdata = db[colname] - mdata.replace_one({"competition": competition, "team": team_num}, {"_id": competition + - str(team_num)+"am", "competition": competition, "team": team_num, "metrics": data}, True) + mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True) - -def push_team_pit_data(client, competition, variable, data, dbname="data_processing", colname="team_pit"): +def push_team_pit_data(client, competition, variable, data, dbname = "data_processing", colname = "team_pit"): db = client[dbname] mdata = db[colname] - mdata.replace_one({"competition": competition, "variable": variable}, { - "competition": competition, "variable": variable, "data": data}, True) - + mdata.replace_one({"competition" : competition, "variable": variable}, {"competition" : competition, "variable" : variable, "data" : data}, True) def get_analysis_flags(client, flag): db = client.data_processing mdata = db.flags - return mdata.find_one({flag: {"$exists": True}}) - + return mdata.find_one({flag:{"$exists":True}}) def set_analysis_flags(client, flag, data): db = client.data_processing mdata = db.flags - return mdata.replace_one({flag: {"$exists": True}}, data, True) - + return mdata.replace_one({flag:{"$exists":True}}, data, True) def unkeyify_2l(layered_dict): out = {} @@ -137,12 +115,11 @@ def unkeyify_2l(layered_dict): add = [] sortkey = [] for j in layered_dict[i].keys(): - add.append([j, layered_dict[i][j]]) - add.sort(key=lambda x: x[0]) + add.append([j,layered_dict[i][j]]) + add.sort(key = lambda x: x[0]) out[i] = list(map(lambda x: x[1], add)) return out - def get_previous_time(client): previous_time = get_analysis_flags(client, "latest_update") @@ -158,28 +135,23 @@ def get_previous_time(client): return previous_time - def set_current_time(client, current_time): - set_analysis_flags(client, "latest_update", {"latest_update": current_time}) - + set_analysis_flags(client, "latest_update", {"latest_update":current_time}) def get_database_config(client): remote_config = get_analysis_flags(client, "config") return remote_config["config"] if remote_config != None else None - def set_database_config(client, config): set_analysis_flags(client, "config", {"config": config}) - def load_match(client, competition): return get_match_data_formatted(client, competition) - def load_metric(client, competition, match, group_name, metrics): group = {} @@ -191,8 +163,7 @@ def load_metric(client, competition, match, group_name, metrics): if db_data == None: elo = {"score": metrics["elo"]["score"]} - gl2 = {"score": metrics["gl2"]["score"], - "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} + gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} ts = {"mu": metrics["ts"]["mu"], "sigma": metrics["ts"]["sigma"]} group[team] = {"elo": elo, "gl2": gl2, "ts": ts} @@ -209,33 +180,28 @@ def load_metric(client, competition, match, group_name, metrics): return group - def load_pit(client, competition): return get_pit_data_formatted(client, competition) - def push_match(client, competition, results): for team in results: push_team_tests_data(client, competition, team, results[team]) - def push_metric(client, competition, metric): for team in metric: push_team_metrics_data(client, competition, team, metric[team]) - def push_pit(client, competition, pit): for variable in pit: - + push_team_pit_data(client, competition, variable, pit[variable]) - def check_new_database_matches(client, competition): return True diff --git a/competition/module.py b/competition/module.py index d51d56b..f5d5023 100644 --- a/competition/module.py +++ b/competition/module.py @@ -1,10 +1,8 @@ import abc import data as d import signal -import trueskill import numpy as np from tra_analysis import Analysis as an -from ts_predict import win_probability class Module(metaclass = abc.ABCMeta): @@ -182,9 +180,9 @@ class Metric (Module): red = {} blu = {} - + for match in matches: - + red = d.load_metric(self.apikey, self.competition, match, "red", self.config["tests"]) blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config["tests"]) @@ -200,9 +198,6 @@ class Metric (Module): gl2_red_vol_total = 0 gl2_blu_vol_total = 0 - ts_red_team = {} - ts_blu_team = {} - for team in red: elo_red_total += red[team]["elo"]["score"] @@ -211,10 +206,6 @@ class Metric (Module): gl2_red_rd_total += red[team]["gl2"]["rd"] gl2_red_vol_total += red[team]["gl2"]["vol"] - tmu = red[team]["ts"]["mu"] or 25 - tsigma = red[team]["ts"]["sigma"] or 8.333 - ts_red_team[team] = trueskill.Rating(mu=tmu, sigma=tsigma) - for team in blu: elo_blu_total += blu[team]["elo"]["score"] @@ -223,43 +214,25 @@ class Metric (Module): gl2_blu_rd_total += blu[team]["gl2"]["rd"] gl2_blu_vol_total += blu[team]["gl2"]["vol"] - tmu = blu[team]["ts"]["mu"] or 25 - tsigma = blu[team]["ts"]["sigma"] or 8.333 - ts_blu_team[team] = trueskill.Rating(mu=tmu, sigma=tsigma) - red_elo = {"score": elo_red_total / len(red)} blu_elo = {"score": elo_blu_total / len(blu)} red_gl2 = {"score": gl2_red_score_total / len(red), "rd": gl2_red_rd_total / len(red), "vol": gl2_red_vol_total / len(red)} blu_gl2 = {"score": gl2_blu_score_total / len(blu), "rd": gl2_blu_rd_total / len(blu), "vol": gl2_blu_vol_total / len(blu)} - print('here') + if match["winner"] == "red": observations = {"red": 1, "blu": 0} - ts_obs = [1,0] elif match["winner"] == "blue": observations = {"red": 0, "blu": 1} - ts_obs = [0,1] else: observations = {"red": 0.5, "blu": 0.5} - ts_obs = [0,0] - ts_red = list(ts_red_team.values()) - ts_blu = list(ts_blu_team.values()) - new_red, new_blu = trueskill.rate([ts_red, ts_blu], ranks=ts_obs) - new_red_ts = {} - new_blu_ts = {} - for key, value in zip(ts_red_team.keys(), new_red): - new_red_ts[key] = value - for key, value in zip(ts_blu_team.keys(), new_blu): - new_blu_ts[key] = value - print("red" if win_probability(new_red, new_blu) > 0.5 else "blue", match["winner"]) - # now there are new trueskll ratings for each team based on the win/loss from DB red_elo_delta = an.Metric().elo(red_elo["score"], blu_elo["score"], observations["red"], elo_N, elo_K) - red_elo["score"] blu_elo_delta = an.Metric().elo(blu_elo["score"], red_elo["score"], observations["blu"], elo_N, elo_K) - blu_elo["score"] @@ -277,9 +250,6 @@ class Metric (Module): red[team]["gl2"]["rd"] = red[team]["gl2"]["rd"] + red_gl2_delta["rd"] red[team]["gl2"]["vol"] = red[team]["gl2"]["vol"] + red_gl2_delta["vol"] - red[team]["ts"]["mu"] = new_red_ts[team].mu - red[team]["ts"]["sigma"] = new_red_ts[team].sigma - for team in blu: blu[team]["elo"]["score"] = blu[team]["elo"]["score"] + blu_elo_delta @@ -288,10 +258,6 @@ class Metric (Module): blu[team]["gl2"]["rd"] = blu[team]["gl2"]["rd"] + blu_gl2_delta["rd"] blu[team]["gl2"]["vol"] = blu[team]["gl2"]["vol"] + blu_gl2_delta["vol"] - - blu[team]["ts"]["mu"] = new_blu_ts[team].mu - blu[team]["ts"]["sigma"] = new_blu_ts[team].sigma - temp_vector = {} temp_vector.update(red) temp_vector.update(blu) diff --git a/competition/requirements.txt b/competition/requirements.txt index 5ea81ce..503e263 100644 --- a/competition/requirements.txt +++ b/competition/requirements.txt @@ -12,5 +12,4 @@ requests scikit-learn scipy six -tra-analysis -trueskill \ No newline at end of file +tra-analysis \ No newline at end of file diff --git a/competition/ts_predict.py b/competition/ts_predict.py deleted file mode 100644 index 5faae96..0000000 --- a/competition/ts_predict.py +++ /dev/null @@ -1,13 +0,0 @@ -from trueskill import Rating -import trueskill -from trueskill import TrueSkill -import math - -BETA = 8.333/2 -cdf = TrueSkill().cdf -def win_probability(a, b): - deltaMu = sum([x.mu for x in a]) - sum([x.mu for x in b]) - sumSigma = sum([x.sigma ** 2 for x in a]) + sum([x.sigma ** 2 for x in b]) - playerCount = len(a) + len(b) - denominator = math.sqrt(playerCount * (BETA * BETA) + sumSigma) - return cdf(deltaMu / denominator) \ No newline at end of file From ac8002aaf8bc1cb50b4593194ac9ae92a6602029 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 28 Mar 2022 14:43:24 -0500 Subject: [PATCH 134/143] delete on start --- competition/data.py | 7 +++++++ competition/module.py | 7 ++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/competition/data.py b/competition/data.py index bd833ee..1cf0131 100644 --- a/competition/data.py +++ b/competition/data.py @@ -11,6 +11,7 @@ def pull_new_tba_matches(apikey, competition, cutoff): for i in json: if i["actual_time"] != None and i["comp_level"] == "qm": out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) + out.sort(key=lambda x: x['match']) return out def pull_new_tba_matches_manual(apikey, competition, cutoff): @@ -27,6 +28,12 @@ def get_team_match_data(client, competition, team_num): out[i['match']] = i['data'] return pd.DataFrame(out) +def clear_metrics(client, competition): + db = client.data_processing + data = db.team_metrics + data.delete_many({competition: competition}) + return True + def get_team_pit_data(client, competition, team_num): db = client.data_scouting mdata = db.pitdata diff --git a/competition/module.py b/competition/module.py index f5d5023..eaeed4c 100644 --- a/competition/module.py +++ b/competition/module.py @@ -3,6 +3,7 @@ import data as d import signal import numpy as np from tra_analysis import Analysis as an +from tqdm import tqdm class Module(metaclass = abc.ABCMeta): @@ -173,6 +174,8 @@ class Metric (Module): def _process_data(self): + d.clear_metrics(self.apikey, self.competition) + elo_N = self.config["tests"]["elo"]["N"] elo_K = self.config["tests"]["elo"]["K"] @@ -180,9 +183,7 @@ class Metric (Module): red = {} blu = {} - - for match in matches: - + for match in tqdm(matches, desc="Metrics"): red = d.load_metric(self.apikey, self.competition, match, "red", self.config["tests"]) blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config["tests"]) From 82ec2d85ccd2e4a02a292bba8705eb5efeeabdbb Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 28 Mar 2022 15:47:46 -0500 Subject: [PATCH 135/143] clear metrics only on script start --- competition/module.py | 2 -- competition/superscript.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/competition/module.py b/competition/module.py index eaeed4c..a2eec3a 100644 --- a/competition/module.py +++ b/competition/module.py @@ -174,8 +174,6 @@ class Metric (Module): def _process_data(self): - d.clear_metrics(self.apikey, self.competition) - elo_N = self.config["tests"]["elo"]["N"] elo_K = self.config["tests"]["elo"]["K"] diff --git a/competition/superscript.py b/competition/superscript.py index fa032aa..48aab1a 100644 --- a/competition/superscript.py +++ b/competition/superscript.py @@ -154,7 +154,7 @@ import pymongo # soon to be deprecated import traceback import warnings from config import Configuration, ConfigurationError -from data import get_previous_time, set_current_time, check_new_database_matches +from data import get_previous_time, set_current_time, check_new_database_matches, clear_metrics from interface import Logger from module import Match, Metric, Pit import zmq @@ -205,7 +205,7 @@ def main(logger, verbose, profile, debug, socket_send = None): config.resolve_config_conflicts(logger, client) config_modules, competition = config.modules, config.competition - + clear_metrics(client, config.competition) for m in config_modules: if m in modules: start = time.time() From cdd81295fc4469fb01b8cec8f7db8d6fe4bd4180 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Mon, 28 Mar 2022 22:02:39 +0000 Subject: [PATCH 136/143] commented metrics in module.py --- competition/module.py | 44 +++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/competition/module.py b/competition/module.py index a2eec3a..5d7b897 100644 --- a/competition/module.py +++ b/competition/module.py @@ -181,9 +181,9 @@ class Metric (Module): red = {} blu = {} - for match in tqdm(matches, desc="Metrics"): - red = d.load_metric(self.apikey, self.competition, match, "red", self.config["tests"]) - blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config["tests"]) + for match in tqdm(matches, desc="Metrics"): # grab matches and loop through each one + red = d.load_metric(self.apikey, self.competition, match, "red", self.config["tests"]) # get the current ratings for red + blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config["tests"]) # get the current ratings for blue elo_red_total = 0 elo_blu_total = 0 @@ -197,7 +197,7 @@ class Metric (Module): gl2_red_vol_total = 0 gl2_blu_vol_total = 0 - for team in red: + for team in red: # for each team in red, add up the elo score and gl2 score components elo_red_total += red[team]["elo"]["score"] @@ -205,7 +205,7 @@ class Metric (Module): gl2_red_rd_total += red[team]["gl2"]["rd"] gl2_red_vol_total += red[team]["gl2"]["vol"] - for team in blu: + for team in blu: # for each team in blue, add up the elo score and gl2 score components elo_blu_total += blu[team]["elo"]["score"] @@ -213,35 +213,35 @@ class Metric (Module): gl2_blu_rd_total += blu[team]["gl2"]["rd"] gl2_blu_vol_total += blu[team]["gl2"]["vol"] - red_elo = {"score": elo_red_total / len(red)} - blu_elo = {"score": elo_blu_total / len(blu)} + red_elo = {"score": elo_red_total / len(red)} # average the scores by dividing by 3 + blu_elo = {"score": elo_blu_total / len(blu)} # average the scores by dividing by 3 - red_gl2 = {"score": gl2_red_score_total / len(red), "rd": gl2_red_rd_total / len(red), "vol": gl2_red_vol_total / len(red)} - blu_gl2 = {"score": gl2_blu_score_total / len(blu), "rd": gl2_blu_rd_total / len(blu), "vol": gl2_blu_vol_total / len(blu)} + red_gl2 = {"score": gl2_red_score_total / len(red), "rd": gl2_red_rd_total / len(red), "vol": gl2_red_vol_total / len(red)} # average the scores by dividing by 3 + blu_gl2 = {"score": gl2_blu_score_total / len(blu), "rd": gl2_blu_rd_total / len(blu), "vol": gl2_blu_vol_total / len(blu)} # average the scores by dividing by 3 - if match["winner"] == "red": + if match["winner"] == "red": # if red won, set observations to {"red": 1, "blu": 0} observations = {"red": 1, "blu": 0} - elif match["winner"] == "blue": + elif match["winner"] == "blue": # if blue won, set observations to {"red": 0, "blu": 1} observations = {"red": 0, "blu": 1} - else: + else: # otherwise it was a tie and observations is {"red": 0.5, "blu": 0.5} observations = {"red": 0.5, "blu": 0.5} - red_elo_delta = an.Metric().elo(red_elo["score"], blu_elo["score"], observations["red"], elo_N, elo_K) - red_elo["score"] - blu_elo_delta = an.Metric().elo(blu_elo["score"], red_elo["score"], observations["blu"], elo_N, elo_K) - blu_elo["score"] + red_elo_delta = an.Metric().elo(red_elo["score"], blu_elo["score"], observations["red"], elo_N, elo_K) - red_elo["score"] # calculate new elo for red using analysis, this is a delta + blu_elo_delta = an.Metric().elo(blu_elo["score"], red_elo["score"], observations["blu"], elo_N, elo_K) - blu_elo["score"] # calculate new elo for blue using analysis, this is a delta - new_red_gl2_score, new_red_gl2_rd, new_red_gl2_vol = an.Metric().glicko2(red_gl2["score"], red_gl2["rd"], red_gl2["vol"], [blu_gl2["score"]], [blu_gl2["rd"]], [observations["red"], observations["blu"]]) - new_blu_gl2_score, new_blu_gl2_rd, new_blu_gl2_vol = an.Metric().glicko2(blu_gl2["score"], blu_gl2["rd"], blu_gl2["vol"], [red_gl2["score"]], [red_gl2["rd"]], [observations["blu"], observations["red"]]) + new_red_gl2_score, new_red_gl2_rd, new_red_gl2_vol = an.Metric().glicko2(red_gl2["score"], red_gl2["rd"], red_gl2["vol"], [blu_gl2["score"]], [blu_gl2["rd"]], [observations["red"], observations["blu"]]) # calculate new scores for gl2 for red + new_blu_gl2_score, new_blu_gl2_rd, new_blu_gl2_vol = an.Metric().glicko2(blu_gl2["score"], blu_gl2["rd"], blu_gl2["vol"], [red_gl2["score"]], [red_gl2["rd"]], [observations["blu"], observations["red"]]) # calculate new scores for gl2 for blue - red_gl2_delta = {"score": new_red_gl2_score - red_gl2["score"], "rd": new_red_gl2_rd - red_gl2["rd"], "vol": new_red_gl2_vol - red_gl2["vol"]} - blu_gl2_delta = {"score": new_blu_gl2_score - blu_gl2["score"], "rd": new_blu_gl2_rd - blu_gl2["rd"], "vol": new_blu_gl2_vol - blu_gl2["vol"]} + red_gl2_delta = {"score": new_red_gl2_score - red_gl2["score"], "rd": new_red_gl2_rd - red_gl2["rd"], "vol": new_red_gl2_vol - red_gl2["vol"]} # calculate gl2 deltas for red + blu_gl2_delta = {"score": new_blu_gl2_score - blu_gl2["score"], "rd": new_blu_gl2_rd - blu_gl2["rd"], "vol": new_blu_gl2_vol - blu_gl2["vol"]} # calculate gl2 deltas for blue - for team in red: + for team in red: # for each team on red, add the previous score with the delta to find the new score red[team]["elo"]["score"] = red[team]["elo"]["score"] + red_elo_delta @@ -249,7 +249,7 @@ class Metric (Module): red[team]["gl2"]["rd"] = red[team]["gl2"]["rd"] + red_gl2_delta["rd"] red[team]["gl2"]["vol"] = red[team]["gl2"]["vol"] + red_gl2_delta["vol"] - for team in blu: + for team in blu: # for each team on blue, add the previous score with the delta to find the new score blu[team]["elo"]["score"] = blu[team]["elo"]["score"] + blu_elo_delta @@ -258,10 +258,10 @@ class Metric (Module): blu[team]["gl2"]["vol"] = blu[team]["gl2"]["vol"] + blu_gl2_delta["vol"] temp_vector = {} - temp_vector.update(red) + temp_vector.update(red) # update the team's score with the temporay vector temp_vector.update(blu) - d.push_metric(self.apikey, self.competition, temp_vector) + d.push_metric(self.apikey, self.competition, temp_vector) # push new scores to db def _push_results(self): pass From fdcdadb8b2bdb3af9ecc026328fcd3826b723733 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Mon, 28 Mar 2022 23:17:25 -0500 Subject: [PATCH 137/143] only use gl2 --- competition/data.py | 9 ++------- competition/dep.py | 9 --------- competition/module.py | 22 ++-------------------- 3 files changed, 4 insertions(+), 36 deletions(-) diff --git a/competition/data.py b/competition/data.py index 1cf0131..93ba419 100644 --- a/competition/data.py +++ b/competition/data.py @@ -168,22 +168,17 @@ def load_metric(client, competition, match, group_name, metrics): db_data = get_team_metrics_data(client, competition, team) if db_data == None: - - elo = {"score": metrics["elo"]["score"]} gl2 = {"score": metrics["gl2"]["score"], "rd": metrics["gl2"]["rd"], "vol": metrics["gl2"]["vol"]} - ts = {"mu": metrics["ts"]["mu"], "sigma": metrics["ts"]["sigma"]} - group[team] = {"elo": elo, "gl2": gl2, "ts": ts} + group[team] = {"gl2": gl2} else: metrics = db_data["metrics"] - elo = metrics["elo"] gl2 = metrics["gl2"] - ts = metrics["ts"] - group[team] = {"elo": elo, "gl2": gl2, "ts": ts} + group[team] = {"gl2": gl2} return group diff --git a/competition/dep.py b/competition/dep.py index 53abb64..891dfb3 100644 --- a/competition/dep.py +++ b/competition/dep.py @@ -96,20 +96,11 @@ sample_json = """ }, "metric":{ "tests":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, "gl2":{ "score":1500, "rd":250, "vol":0.06 }, - "ts":{ - "mu":25, - "sigma":8.33 - } } }, "pit":{ diff --git a/competition/module.py b/competition/module.py index 5d7b897..705da9c 100644 --- a/competition/module.py +++ b/competition/module.py @@ -174,9 +174,6 @@ class Metric (Module): def _process_data(self): - elo_N = self.config["tests"]["elo"]["N"] - elo_K = self.config["tests"]["elo"]["K"] - matches = self.data red = {} @@ -184,9 +181,6 @@ class Metric (Module): for match in tqdm(matches, desc="Metrics"): # grab matches and loop through each one red = d.load_metric(self.apikey, self.competition, match, "red", self.config["tests"]) # get the current ratings for red blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config["tests"]) # get the current ratings for blue - - elo_red_total = 0 - elo_blu_total = 0 gl2_red_score_total = 0 gl2_blu_score_total = 0 @@ -197,24 +191,18 @@ class Metric (Module): gl2_red_vol_total = 0 gl2_blu_vol_total = 0 - for team in red: # for each team in red, add up the elo score and gl2 score components - - elo_red_total += red[team]["elo"]["score"] + for team in red: # for each team in red, add up gl2 score components gl2_red_score_total += red[team]["gl2"]["score"] gl2_red_rd_total += red[team]["gl2"]["rd"] gl2_red_vol_total += red[team]["gl2"]["vol"] - for team in blu: # for each team in blue, add up the elo score and gl2 score components - - elo_blu_total += blu[team]["elo"]["score"] + for team in blu: # for each team in blue, add up gl2 score components gl2_blu_score_total += blu[team]["gl2"]["score"] gl2_blu_rd_total += blu[team]["gl2"]["rd"] gl2_blu_vol_total += blu[team]["gl2"]["vol"] - red_elo = {"score": elo_red_total / len(red)} # average the scores by dividing by 3 - blu_elo = {"score": elo_blu_total / len(blu)} # average the scores by dividing by 3 red_gl2 = {"score": gl2_red_score_total / len(red), "rd": gl2_red_rd_total / len(red), "vol": gl2_red_vol_total / len(red)} # average the scores by dividing by 3 blu_gl2 = {"score": gl2_blu_score_total / len(blu), "rd": gl2_blu_rd_total / len(blu), "vol": gl2_blu_vol_total / len(blu)} # average the scores by dividing by 3 @@ -232,8 +220,6 @@ class Metric (Module): observations = {"red": 0.5, "blu": 0.5} - red_elo_delta = an.Metric().elo(red_elo["score"], blu_elo["score"], observations["red"], elo_N, elo_K) - red_elo["score"] # calculate new elo for red using analysis, this is a delta - blu_elo_delta = an.Metric().elo(blu_elo["score"], red_elo["score"], observations["blu"], elo_N, elo_K) - blu_elo["score"] # calculate new elo for blue using analysis, this is a delta new_red_gl2_score, new_red_gl2_rd, new_red_gl2_vol = an.Metric().glicko2(red_gl2["score"], red_gl2["rd"], red_gl2["vol"], [blu_gl2["score"]], [blu_gl2["rd"]], [observations["red"], observations["blu"]]) # calculate new scores for gl2 for red new_blu_gl2_score, new_blu_gl2_rd, new_blu_gl2_vol = an.Metric().glicko2(blu_gl2["score"], blu_gl2["rd"], blu_gl2["vol"], [red_gl2["score"]], [red_gl2["rd"]], [observations["blu"], observations["red"]]) # calculate new scores for gl2 for blue @@ -243,16 +229,12 @@ class Metric (Module): for team in red: # for each team on red, add the previous score with the delta to find the new score - red[team]["elo"]["score"] = red[team]["elo"]["score"] + red_elo_delta - red[team]["gl2"]["score"] = red[team]["gl2"]["score"] + red_gl2_delta["score"] red[team]["gl2"]["rd"] = red[team]["gl2"]["rd"] + red_gl2_delta["rd"] red[team]["gl2"]["vol"] = red[team]["gl2"]["vol"] + red_gl2_delta["vol"] for team in blu: # for each team on blue, add the previous score with the delta to find the new score - blu[team]["elo"]["score"] = blu[team]["elo"]["score"] + blu_elo_delta - blu[team]["gl2"]["score"] = blu[team]["gl2"]["score"] + blu_gl2_delta["score"] blu[team]["gl2"]["rd"] = blu[team]["gl2"]["rd"] + blu_gl2_delta["rd"] blu[team]["gl2"]["vol"] = blu[team]["gl2"]["vol"] + blu_gl2_delta["vol"] From 69c6059ff888f5cc7b0e2f2af92d9464ca32b0a2 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Tue, 29 Mar 2022 21:17:58 +0000 Subject: [PATCH 138/143] added result logging for metric module --- competition/module.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/competition/module.py b/competition/module.py index 705da9c..69aa5b0 100644 --- a/competition/module.py +++ b/competition/module.py @@ -174,6 +174,8 @@ class Metric (Module): def _process_data(self): + self.results = {} + matches = self.data red = {} @@ -243,6 +245,8 @@ class Metric (Module): temp_vector.update(red) # update the team's score with the temporay vector temp_vector.update(blu) + self.results[match['match']] = temp_vector + d.push_metric(self.apikey, self.competition, temp_vector) # push new scores to db def _push_results(self): From 8e5fa7eace292562fc7b85eb5ea81816cd866e67 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Tue, 29 Mar 2022 18:43:28 -0500 Subject: [PATCH 139/143] reintroduce cutoff usage --- competition/data.py | 6 +++--- competition/superscript.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/competition/data.py b/competition/data.py index 93ba419..a2eb477 100644 --- a/competition/data.py +++ b/competition/data.py @@ -9,7 +9,7 @@ def pull_new_tba_matches(apikey, competition, cutoff): json = x.json() out = [] for i in json: - if i["actual_time"] != None and i["comp_level"] == "qm": + if i["actual_time"] != None and i["comp_level"] == "qm" and i["actual_time"]-cutoff >= 0: out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) out.sort(key=lambda x: x['match']) return out @@ -17,8 +17,8 @@ def pull_new_tba_matches(apikey, competition, cutoff): def pull_new_tba_matches_manual(apikey, competition, cutoff): filename = competition+"-wins.json" with open(filename, 'r') as f: - data = json.load(f) - return data + data = json.load(f) + return data def get_team_match_data(client, competition, team_num): db = client.data_scouting diff --git a/competition/superscript.py b/competition/superscript.py index 48aab1a..e695874 100644 --- a/competition/superscript.py +++ b/competition/superscript.py @@ -205,7 +205,6 @@ def main(logger, verbose, profile, debug, socket_send = None): config.resolve_config_conflicts(logger, client) config_modules, competition = config.modules, config.competition - clear_metrics(client, config.competition) for m in config_modules: if m in modules: start = time.time() From b5c8a91fada93cf848536ecdd04e9b4f2f190568 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Wed, 30 Mar 2022 01:30:55 -0500 Subject: [PATCH 140/143] update data.py for new metrics structure --- competition/data.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/competition/data.py b/competition/data.py index a2eb477..4dd53c5 100644 --- a/competition/data.py +++ b/competition/data.py @@ -1,3 +1,4 @@ +from calendar import c import requests import pull import pandas as pd @@ -43,7 +44,15 @@ def get_team_pit_data(client, competition, team_num): def get_team_metrics_data(client, competition, team_num): db = client.data_processing mdata = db.team_metrics - return mdata.find_one({"competition" : competition, "team": team_num}) + temp = mdata.find_one({"team": team_num}) + if temp != None: + if competition in temp['metrics'].keys(): + temp = temp['metrics'][competition] + else : + temp = None + else: + temp = None + return temp def get_match_data_formatted(client, competition): teams_at_comp = pull.get_teams_at_competition(competition) @@ -99,7 +108,7 @@ def push_team_tests_data(client, competition, team_num, data, dbname = "data_pro def push_team_metrics_data(client, competition, team_num, data, dbname = "data_processing", colname = "team_metrics"): db = client[dbname] mdata = db[colname] - mdata.replace_one({"competition" : competition, "team": team_num}, {"_id": competition+str(team_num)+"am", "competition" : competition, "team" : team_num, "metrics" : data}, True) + mdata.update_one({"team": team_num}, {"$set": {"metrics.{}".format(competition): data}}, upsert=True) def push_team_pit_data(client, competition, variable, data, dbname = "data_processing", colname = "team_pit"): db = client[dbname] @@ -174,7 +183,7 @@ def load_metric(client, competition, match, group_name, metrics): else: - metrics = db_data["metrics"] + metrics = db_data gl2 = metrics["gl2"] From d847f6d6a7ce623258e4c931a3de6a0ae62bc9ea Mon Sep 17 00:00:00 2001 From: Dev Date: Fri, 8 Apr 2022 15:20:41 -0500 Subject: [PATCH 141/143] hi --- DEBUG_slurm-tra-superscript.out | 17 ++++ PROD_slurm-tra-superscript.out | 170 ++++++++++++++++++++++++++++++++ competition/data.py | 4 +- competition/module.py | 11 ++- 4 files changed, 196 insertions(+), 6 deletions(-) create mode 100644 DEBUG_slurm-tra-superscript.out create mode 100644 PROD_slurm-tra-superscript.out diff --git a/DEBUG_slurm-tra-superscript.out b/DEBUG_slurm-tra-superscript.out new file mode 100644 index 0000000..2384b35 --- /dev/null +++ b/DEBUG_slurm-tra-superscript.out @@ -0,0 +1,17 @@ +2022/04/08 14:45:12 | : #--------------------------------------# +2022/04/08 14:45:12 | : | superscript version: 1.0.0 | +2022/04/08 14:45:12 | : | os: Linux | +2022/04/08 14:45:12 | : | python: 3.9.7 | +2022/04/08 14:45:12 | : #--------------------------------------# +2022/04/08 14:45:12 | [INFO] : current time: 1649447112.4795144 +2022/04/08 14:45:12 | [INFO] : found and loaded config at +2022/04/08 14:45:12 | [INFO] : found and loaded database and tba keys +2022/04/08 14:45:12 | [INFO] : established connection to database +2022/04/08 14:45:13 | [INFO] : analysis backtimed to: 0.0 +2022/04/08 14:45:13 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:45:24 | [INFO] : match module finished in 11.742639541625977 seconds + Metrics: 0%| | 0/34 [00:00 +2022/04/08 13:39:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:39:18 | [INFO] : established connection to database +2022/04/08 13:39:19 | [INFO] : analysis backtimed to: 0.0 +2022/04/08 13:39:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:39:25 | [INFO] : match module finished in 6.360178709030151 seconds + Metrics: 0%| | 0/25 [00:00 +2022/04/08 13:44:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:44:18 | [INFO] : established connection to database +2022/04/08 13:44:19 | [INFO] : analysis backtimed to: 1649443158.5109549 +2022/04/08 13:44:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:44:26 | [INFO] : match module finished in 6.962671756744385 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 13:44:26 | [INFO] : metric module finished in 0.18820714950561523 seconds +2022/04/08 13:44:28 | [INFO] : pit module finished in 1.7953612804412842 seconds +2022/04/08 13:44:28 | [INFO] : closed threads and database client +2022/04/08 13:44:28 | [INFO] : finished all tasks in 9.482671737670898 seconds, looping +2022/04/08 13:44:28 | [INFO] : loop delayed by 290.5172882080078 seconds +2022/04/08 13:49:18 | [INFO] : current time: 1649443758.7116833 +2022/04/08 13:49:18 | [INFO] : found and loaded config at +2022/04/08 13:49:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:49:18 | [INFO] : established connection to database +2022/04/08 13:49:19 | [INFO] : analysis backtimed to: 1649443458.6113253 +2022/04/08 13:49:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:49:26 | [INFO] : match module finished in 7.532702207565308 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 13:49:26 | [INFO] : metric module finished in 0.06511259078979492 seconds +2022/04/08 13:49:28 | [INFO] : pit module finished in 1.819601058959961 seconds +2022/04/08 13:49:28 | [INFO] : closed threads and database client +2022/04/08 13:49:28 | [INFO] : finished all tasks in 9.962053775787354 seconds, looping +2022/04/08 13:49:28 | [INFO] : loop delayed by 290.03790378570557 seconds +2022/04/08 13:54:18 | [INFO] : current time: 1649444058.811996 +2022/04/08 13:54:18 | [INFO] : found and loaded config at +2022/04/08 13:54:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:54:18 | [INFO] : established connection to database +2022/04/08 13:54:19 | [INFO] : analysis backtimed to: 1649443758.7116833 +2022/04/08 13:54:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:54:26 | [INFO] : match module finished in 7.571904420852661 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 13:54:26 | [INFO] : metric module finished in 0.15475034713745117 seconds +2022/04/08 13:54:28 | [INFO] : pit module finished in 1.6621544361114502 seconds +2022/04/08 13:54:28 | [INFO] : closed threads and database client +2022/04/08 13:54:28 | [INFO] : finished all tasks in 9.827200174331665 seconds, looping +2022/04/08 13:54:28 | [INFO] : loop delayed by 290.17272448539734 seconds +2022/04/08 13:59:18 | [INFO] : current time: 1649444358.912293 +2022/04/08 13:59:18 | [INFO] : found and loaded config at +2022/04/08 13:59:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:59:19 | [INFO] : established connection to database +2022/04/08 13:59:19 | [INFO] : analysis backtimed to: 1649444058.811996 +2022/04/08 13:59:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:59:30 | [INFO] : match module finished in 10.847434997558594 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 13:59:30 | [INFO] : metric module finished in 0.10496973991394043 seconds +2022/04/08 13:59:32 | [INFO] : pit module finished in 1.7203879356384277 seconds +2022/04/08 13:59:32 | [INFO] : closed threads and database client +2022/04/08 13:59:32 | [INFO] : finished all tasks in 13.211304664611816 seconds, looping +2022/04/08 13:59:32 | [INFO] : loop delayed by 286.7886555194855 seconds +2022/04/08 14:04:19 | [INFO] : current time: 1649444659.012761 +2022/04/08 14:04:19 | [INFO] : found and loaded config at +2022/04/08 14:04:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:04:19 | [INFO] : established connection to database +2022/04/08 14:04:19 | [INFO] : analysis backtimed to: 1649444358.912293 +2022/04/08 14:04:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:04:28 | [INFO] : match module finished in 9.347296714782715 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:04:28 | [INFO] : metric module finished in 0.10346341133117676 seconds +2022/04/08 14:04:31 | [INFO] : pit module finished in 2.109464168548584 seconds +2022/04/08 14:04:31 | [INFO] : closed threads and database client +2022/04/08 14:04:31 | [INFO] : finished all tasks in 12.072788000106812 seconds, looping +2022/04/08 14:04:31 | [INFO] : loop delayed by 287.92717266082764 seconds +2022/04/08 14:09:19 | [INFO] : current time: 1649444959.0845735 +2022/04/08 14:09:19 | [INFO] : found and loaded config at +2022/04/08 14:09:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:09:19 | [INFO] : established connection to database +2022/04/08 14:09:19 | [INFO] : analysis backtimed to: 1649444659.012761 +2022/04/08 14:09:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:09:30 | [INFO] : match module finished in 10.792919158935547 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:09:30 | [INFO] : metric module finished in 0.08684110641479492 seconds +2022/04/08 14:09:32 | [INFO] : pit module finished in 1.9306814670562744 seconds +2022/04/08 14:09:32 | [INFO] : closed threads and database client +2022/04/08 14:09:32 | [INFO] : finished all tasks in 13.267388105392456 seconds, looping +2022/04/08 14:09:32 | [INFO] : loop delayed by 286.73254227638245 seconds +2022/04/08 14:14:19 | [INFO] : current time: 1649445259.0884073 +2022/04/08 14:14:19 | [INFO] : found and loaded config at +2022/04/08 14:14:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:14:19 | [INFO] : established connection to database +2022/04/08 14:14:19 | [INFO] : analysis backtimed to: 1649444959.0845735 +2022/04/08 14:14:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:14:29 | [INFO] : match module finished in 9.690043449401855 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:14:29 | [INFO] : metric module finished in 0.1428515911102295 seconds +2022/04/08 14:14:31 | [INFO] : pit module finished in 1.9340870380401611 seconds +2022/04/08 14:14:31 | [INFO] : closed threads and database client +2022/04/08 14:14:31 | [INFO] : finished all tasks in 12.373121976852417 seconds, looping +2022/04/08 14:14:31 | [INFO] : loop delayed by 287.6267948150635 seconds +2022/04/08 14:19:19 | [INFO] : current time: 1649445559.188727 +2022/04/08 14:19:19 | [INFO] : found and loaded config at +2022/04/08 14:19:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:19:19 | [INFO] : established connection to database +2022/04/08 14:19:19 | [INFO] : analysis backtimed to: 1649445259.0884073 +2022/04/08 14:19:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:19:30 | [INFO] : match module finished in 10.267187118530273 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:19:30 | [INFO] : metric module finished in 0.5178976058959961 seconds +2022/04/08 14:19:32 | [INFO] : pit module finished in 1.655653476715088 seconds +2022/04/08 14:19:32 | [INFO] : closed threads and database client +2022/04/08 14:19:32 | [INFO] : finished all tasks in 13.050731897354126 seconds, looping +2022/04/08 14:19:32 | [INFO] : loop delayed by 286.9491786956787 seconds +2022/04/08 14:24:19 | [INFO] : current time: 1649445859.2890737 +2022/04/08 14:24:19 | [INFO] : found and loaded config at +2022/04/08 14:24:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:24:19 | [INFO] : established connection to database +2022/04/08 14:24:19 | [INFO] : analysis backtimed to: 1649445559.188727 +2022/04/08 14:24:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:24:30 | [INFO] : match module finished in 11.110676765441895 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:24:31 | [INFO] : metric module finished in 0.09472298622131348 seconds +2022/04/08 14:24:32 | [INFO] : pit module finished in 1.9062983989715576 seconds +2022/04/08 14:24:33 | [INFO] : closed threads and database client +2022/04/08 14:24:33 | [INFO] : finished all tasks in 13.732285976409912 seconds, looping +2022/04/08 14:24:33 | [INFO] : loop delayed by 286.2676737308502 seconds +2022/04/08 14:29:19 | [INFO] : current time: 1649446159.3893542 +2022/04/08 14:29:19 | [INFO] : found and loaded config at +2022/04/08 14:29:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:29:19 | [INFO] : established connection to database +2022/04/08 14:29:19 | [INFO] : analysis backtimed to: 1649445859.2890737 +2022/04/08 14:29:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:29:31 | [INFO] : match module finished in 11.64138388633728 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:29:31 | [INFO] : metric module finished in 0.08331465721130371 seconds +2022/04/08 14:29:33 | [INFO] : pit module finished in 1.7486073970794678 seconds +2022/04/08 14:29:33 | [INFO] : closed threads and database client +2022/04/08 14:29:33 | [INFO] : finished all tasks in 14.08924150466919 seconds, looping +2022/04/08 14:29:33 | [INFO] : loop delayed by 285.9107177257538 seconds +2022/04/08 14:34:19 | [INFO] : current time: 1649446459.4896593 +2022/04/08 14:34:19 | [INFO] : found and loaded config at +2022/04/08 14:34:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:34:19 | [INFO] : established connection to database +2022/04/08 14:34:20 | [INFO] : analysis backtimed to: 1649446159.3893542 +2022/04/08 14:34:20 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:34:34 | [INFO] : match module finished in 14.8920156955719 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:34:35 | [INFO] : metric module finished in 0.11336660385131836 seconds +2022/04/08 14:34:36 | [INFO] : pit module finished in 1.7640228271484375 seconds +2022/04/08 14:34:36 | [INFO] : closed threads and database client +2022/04/08 14:34:36 | [INFO] : finished all tasks in 17.381346464157104 seconds, looping +2022/04/08 14:34:36 | [INFO] : loop delayed by 282.6186122894287 seconds +2022/04/08 14:39:19 | [INFO] : current time: 1649446759.5899608 +2022/04/08 14:39:19 | [INFO] : found and loaded config at +2022/04/08 14:39:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:39:19 | [INFO] : established connection to database +2022/04/08 14:39:20 | [INFO] : analysis backtimed to: 1649446459.4896593 +2022/04/08 14:39:20 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:39:32 | [INFO] : match module finished in 12.305787801742554 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +slurmstepd-cpu-dl580-0: error: *** JOB 242 ON cpu-dl580-0 CANCELLED AT 2022-04-08T14:41:13 *** diff --git a/competition/data.py b/competition/data.py index 4dd53c5..3b798f9 100644 --- a/competition/data.py +++ b/competition/data.py @@ -4,13 +4,13 @@ import pull import pandas as pd import json -def pull_new_tba_matches(apikey, competition, cutoff): +def pull_new_tba_matches(apikey, competition, last_match): api_key= apikey x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth-Key":api_key}) json = x.json() out = [] for i in json: - if i["actual_time"] != None and i["comp_level"] == "qm" and i["actual_time"]-cutoff >= 0: + if i["actual_time"] != None and i["comp_level"] == "qm" and i["match_number"] > last_match : out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) out.sort(key=lambda x: x['match']) return out diff --git a/competition/module.py b/competition/module.py index 69aa5b0..c3635aa 100644 --- a/competition/module.py +++ b/competition/module.py @@ -170,17 +170,19 @@ class Metric (Module): self._push_results() def _load_data(self): - self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp) + self.last_match = d.get_analysis_flags(self.apikey, 'metrics_last_match')['metrics_last_match'] + print("Previous last match", self.last_match) + self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.last_match) def _process_data(self): self.results = {} - + self.match = 0 matches = self.data - red = {} blu = {} for match in tqdm(matches, desc="Metrics"): # grab matches and loop through each one + self.match = max(self.match, int(match['match'])) red = d.load_metric(self.apikey, self.competition, match, "red", self.config["tests"]) # get the current ratings for red blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config["tests"]) # get the current ratings for blue @@ -248,7 +250,8 @@ class Metric (Module): self.results[match['match']] = temp_vector d.push_metric(self.apikey, self.competition, temp_vector) # push new scores to db - + print("New last match", self.match) + d.set_analysis_flags(self.apikey, 'metrics_last_match', {'metrics_last_match': self.match}) def _push_results(self): pass From 11398290eb4e4243c25d6b8dafa068ee9e135ef3 Mon Sep 17 00:00:00 2001 From: Dev Date: Fri, 8 Apr 2022 15:40:33 -0500 Subject: [PATCH 142/143] fix --- DEBUG_slurm-tra-superscript.out | 17 ---- PROD_slurm-tra-superscript.out | 170 -------------------------------- competition/data.py | 4 +- competition/module.py | 2 +- 4 files changed, 3 insertions(+), 190 deletions(-) delete mode 100644 DEBUG_slurm-tra-superscript.out delete mode 100644 PROD_slurm-tra-superscript.out diff --git a/DEBUG_slurm-tra-superscript.out b/DEBUG_slurm-tra-superscript.out deleted file mode 100644 index 2384b35..0000000 --- a/DEBUG_slurm-tra-superscript.out +++ /dev/null @@ -1,17 +0,0 @@ -2022/04/08 14:45:12 | : #--------------------------------------# -2022/04/08 14:45:12 | : | superscript version: 1.0.0 | -2022/04/08 14:45:12 | : | os: Linux | -2022/04/08 14:45:12 | : | python: 3.9.7 | -2022/04/08 14:45:12 | : #--------------------------------------# -2022/04/08 14:45:12 | [INFO] : current time: 1649447112.4795144 -2022/04/08 14:45:12 | [INFO] : found and loaded config at -2022/04/08 14:45:12 | [INFO] : found and loaded database and tba keys -2022/04/08 14:45:12 | [INFO] : established connection to database -2022/04/08 14:45:13 | [INFO] : analysis backtimed to: 0.0 -2022/04/08 14:45:13 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 14:45:24 | [INFO] : match module finished in 11.742639541625977 seconds - Metrics: 0%| | 0/34 [00:00 -2022/04/08 13:39:18 | [INFO] : found and loaded database and tba keys -2022/04/08 13:39:18 | [INFO] : established connection to database -2022/04/08 13:39:19 | [INFO] : analysis backtimed to: 0.0 -2022/04/08 13:39:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 13:39:25 | [INFO] : match module finished in 6.360178709030151 seconds - Metrics: 0%| | 0/25 [00:00 -2022/04/08 13:44:18 | [INFO] : found and loaded database and tba keys -2022/04/08 13:44:18 | [INFO] : established connection to database -2022/04/08 13:44:19 | [INFO] : analysis backtimed to: 1649443158.5109549 -2022/04/08 13:44:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 13:44:26 | [INFO] : match module finished in 6.962671756744385 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 13:44:26 | [INFO] : metric module finished in 0.18820714950561523 seconds -2022/04/08 13:44:28 | [INFO] : pit module finished in 1.7953612804412842 seconds -2022/04/08 13:44:28 | [INFO] : closed threads and database client -2022/04/08 13:44:28 | [INFO] : finished all tasks in 9.482671737670898 seconds, looping -2022/04/08 13:44:28 | [INFO] : loop delayed by 290.5172882080078 seconds -2022/04/08 13:49:18 | [INFO] : current time: 1649443758.7116833 -2022/04/08 13:49:18 | [INFO] : found and loaded config at -2022/04/08 13:49:18 | [INFO] : found and loaded database and tba keys -2022/04/08 13:49:18 | [INFO] : established connection to database -2022/04/08 13:49:19 | [INFO] : analysis backtimed to: 1649443458.6113253 -2022/04/08 13:49:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 13:49:26 | [INFO] : match module finished in 7.532702207565308 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 13:49:26 | [INFO] : metric module finished in 0.06511259078979492 seconds -2022/04/08 13:49:28 | [INFO] : pit module finished in 1.819601058959961 seconds -2022/04/08 13:49:28 | [INFO] : closed threads and database client -2022/04/08 13:49:28 | [INFO] : finished all tasks in 9.962053775787354 seconds, looping -2022/04/08 13:49:28 | [INFO] : loop delayed by 290.03790378570557 seconds -2022/04/08 13:54:18 | [INFO] : current time: 1649444058.811996 -2022/04/08 13:54:18 | [INFO] : found and loaded config at -2022/04/08 13:54:18 | [INFO] : found and loaded database and tba keys -2022/04/08 13:54:18 | [INFO] : established connection to database -2022/04/08 13:54:19 | [INFO] : analysis backtimed to: 1649443758.7116833 -2022/04/08 13:54:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 13:54:26 | [INFO] : match module finished in 7.571904420852661 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 13:54:26 | [INFO] : metric module finished in 0.15475034713745117 seconds -2022/04/08 13:54:28 | [INFO] : pit module finished in 1.6621544361114502 seconds -2022/04/08 13:54:28 | [INFO] : closed threads and database client -2022/04/08 13:54:28 | [INFO] : finished all tasks in 9.827200174331665 seconds, looping -2022/04/08 13:54:28 | [INFO] : loop delayed by 290.17272448539734 seconds -2022/04/08 13:59:18 | [INFO] : current time: 1649444358.912293 -2022/04/08 13:59:18 | [INFO] : found and loaded config at -2022/04/08 13:59:18 | [INFO] : found and loaded database and tba keys -2022/04/08 13:59:19 | [INFO] : established connection to database -2022/04/08 13:59:19 | [INFO] : analysis backtimed to: 1649444058.811996 -2022/04/08 13:59:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 13:59:30 | [INFO] : match module finished in 10.847434997558594 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 13:59:30 | [INFO] : metric module finished in 0.10496973991394043 seconds -2022/04/08 13:59:32 | [INFO] : pit module finished in 1.7203879356384277 seconds -2022/04/08 13:59:32 | [INFO] : closed threads and database client -2022/04/08 13:59:32 | [INFO] : finished all tasks in 13.211304664611816 seconds, looping -2022/04/08 13:59:32 | [INFO] : loop delayed by 286.7886555194855 seconds -2022/04/08 14:04:19 | [INFO] : current time: 1649444659.012761 -2022/04/08 14:04:19 | [INFO] : found and loaded config at -2022/04/08 14:04:19 | [INFO] : found and loaded database and tba keys -2022/04/08 14:04:19 | [INFO] : established connection to database -2022/04/08 14:04:19 | [INFO] : analysis backtimed to: 1649444358.912293 -2022/04/08 14:04:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 14:04:28 | [INFO] : match module finished in 9.347296714782715 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 14:04:28 | [INFO] : metric module finished in 0.10346341133117676 seconds -2022/04/08 14:04:31 | [INFO] : pit module finished in 2.109464168548584 seconds -2022/04/08 14:04:31 | [INFO] : closed threads and database client -2022/04/08 14:04:31 | [INFO] : finished all tasks in 12.072788000106812 seconds, looping -2022/04/08 14:04:31 | [INFO] : loop delayed by 287.92717266082764 seconds -2022/04/08 14:09:19 | [INFO] : current time: 1649444959.0845735 -2022/04/08 14:09:19 | [INFO] : found and loaded config at -2022/04/08 14:09:19 | [INFO] : found and loaded database and tba keys -2022/04/08 14:09:19 | [INFO] : established connection to database -2022/04/08 14:09:19 | [INFO] : analysis backtimed to: 1649444659.012761 -2022/04/08 14:09:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 14:09:30 | [INFO] : match module finished in 10.792919158935547 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 14:09:30 | [INFO] : metric module finished in 0.08684110641479492 seconds -2022/04/08 14:09:32 | [INFO] : pit module finished in 1.9306814670562744 seconds -2022/04/08 14:09:32 | [INFO] : closed threads and database client -2022/04/08 14:09:32 | [INFO] : finished all tasks in 13.267388105392456 seconds, looping -2022/04/08 14:09:32 | [INFO] : loop delayed by 286.73254227638245 seconds -2022/04/08 14:14:19 | [INFO] : current time: 1649445259.0884073 -2022/04/08 14:14:19 | [INFO] : found and loaded config at -2022/04/08 14:14:19 | [INFO] : found and loaded database and tba keys -2022/04/08 14:14:19 | [INFO] : established connection to database -2022/04/08 14:14:19 | [INFO] : analysis backtimed to: 1649444959.0845735 -2022/04/08 14:14:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 14:14:29 | [INFO] : match module finished in 9.690043449401855 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 14:14:29 | [INFO] : metric module finished in 0.1428515911102295 seconds -2022/04/08 14:14:31 | [INFO] : pit module finished in 1.9340870380401611 seconds -2022/04/08 14:14:31 | [INFO] : closed threads and database client -2022/04/08 14:14:31 | [INFO] : finished all tasks in 12.373121976852417 seconds, looping -2022/04/08 14:14:31 | [INFO] : loop delayed by 287.6267948150635 seconds -2022/04/08 14:19:19 | [INFO] : current time: 1649445559.188727 -2022/04/08 14:19:19 | [INFO] : found and loaded config at -2022/04/08 14:19:19 | [INFO] : found and loaded database and tba keys -2022/04/08 14:19:19 | [INFO] : established connection to database -2022/04/08 14:19:19 | [INFO] : analysis backtimed to: 1649445259.0884073 -2022/04/08 14:19:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 14:19:30 | [INFO] : match module finished in 10.267187118530273 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 14:19:30 | [INFO] : metric module finished in 0.5178976058959961 seconds -2022/04/08 14:19:32 | [INFO] : pit module finished in 1.655653476715088 seconds -2022/04/08 14:19:32 | [INFO] : closed threads and database client -2022/04/08 14:19:32 | [INFO] : finished all tasks in 13.050731897354126 seconds, looping -2022/04/08 14:19:32 | [INFO] : loop delayed by 286.9491786956787 seconds -2022/04/08 14:24:19 | [INFO] : current time: 1649445859.2890737 -2022/04/08 14:24:19 | [INFO] : found and loaded config at -2022/04/08 14:24:19 | [INFO] : found and loaded database and tba keys -2022/04/08 14:24:19 | [INFO] : established connection to database -2022/04/08 14:24:19 | [INFO] : analysis backtimed to: 1649445559.188727 -2022/04/08 14:24:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 14:24:30 | [INFO] : match module finished in 11.110676765441895 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 14:24:31 | [INFO] : metric module finished in 0.09472298622131348 seconds -2022/04/08 14:24:32 | [INFO] : pit module finished in 1.9062983989715576 seconds -2022/04/08 14:24:33 | [INFO] : closed threads and database client -2022/04/08 14:24:33 | [INFO] : finished all tasks in 13.732285976409912 seconds, looping -2022/04/08 14:24:33 | [INFO] : loop delayed by 286.2676737308502 seconds -2022/04/08 14:29:19 | [INFO] : current time: 1649446159.3893542 -2022/04/08 14:29:19 | [INFO] : found and loaded config at -2022/04/08 14:29:19 | [INFO] : found and loaded database and tba keys -2022/04/08 14:29:19 | [INFO] : established connection to database -2022/04/08 14:29:19 | [INFO] : analysis backtimed to: 1649445859.2890737 -2022/04/08 14:29:19 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 14:29:31 | [INFO] : match module finished in 11.64138388633728 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 14:29:31 | [INFO] : metric module finished in 0.08331465721130371 seconds -2022/04/08 14:29:33 | [INFO] : pit module finished in 1.7486073970794678 seconds -2022/04/08 14:29:33 | [INFO] : closed threads and database client -2022/04/08 14:29:33 | [INFO] : finished all tasks in 14.08924150466919 seconds, looping -2022/04/08 14:29:33 | [INFO] : loop delayed by 285.9107177257538 seconds -2022/04/08 14:34:19 | [INFO] : current time: 1649446459.4896593 -2022/04/08 14:34:19 | [INFO] : found and loaded config at -2022/04/08 14:34:19 | [INFO] : found and loaded database and tba keys -2022/04/08 14:34:19 | [INFO] : established connection to database -2022/04/08 14:34:20 | [INFO] : analysis backtimed to: 1649446159.3893542 -2022/04/08 14:34:20 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 14:34:34 | [INFO] : match module finished in 14.8920156955719 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -2022/04/08 14:34:35 | [INFO] : metric module finished in 0.11336660385131836 seconds -2022/04/08 14:34:36 | [INFO] : pit module finished in 1.7640228271484375 seconds -2022/04/08 14:34:36 | [INFO] : closed threads and database client -2022/04/08 14:34:36 | [INFO] : finished all tasks in 17.381346464157104 seconds, looping -2022/04/08 14:34:36 | [INFO] : loop delayed by 282.6186122894287 seconds -2022/04/08 14:39:19 | [INFO] : current time: 1649446759.5899608 -2022/04/08 14:39:19 | [INFO] : found and loaded config at -2022/04/08 14:39:19 | [INFO] : found and loaded database and tba keys -2022/04/08 14:39:19 | [INFO] : established connection to database -2022/04/08 14:39:20 | [INFO] : analysis backtimed to: 1649446459.4896593 -2022/04/08 14:39:20 | [INFO] : config-preference set to local/client, loading local config information -2022/04/08 14:39:32 | [INFO] : match module finished in 12.305787801742554 seconds - Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] -slurmstepd-cpu-dl580-0: error: *** JOB 242 ON cpu-dl580-0 CANCELLED AT 2022-04-08T14:41:13 *** diff --git a/competition/data.py b/competition/data.py index 3b798f9..b3cf8cf 100644 --- a/competition/data.py +++ b/competition/data.py @@ -118,12 +118,12 @@ def push_team_pit_data(client, competition, variable, data, dbname = "data_proce def get_analysis_flags(client, flag): db = client.data_processing mdata = db.flags - return mdata.find_one({flag:{"$exists":True}}) + return mdata.find_one({"_id": "2022"}) def set_analysis_flags(client, flag, data): db = client.data_processing mdata = db.flags - return mdata.replace_one({flag:{"$exists":True}}, data, True) + return mdata.update_one({"_id": "2022"}, {"$set": data}) def unkeyify_2l(layered_dict): out = {} diff --git a/competition/module.py b/competition/module.py index c3635aa..1bac30c 100644 --- a/competition/module.py +++ b/competition/module.py @@ -177,7 +177,7 @@ class Metric (Module): def _process_data(self): self.results = {} - self.match = 0 + self.match = self.last_match matches = self.data red = {} blu = {} From 8e6c44db65cd3a429b0a9e7a33284a0b83fa753a Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Fri, 17 Mar 2023 10:09:48 -0500 Subject: [PATCH 143/143] updates for 2023ilpe --- competition/data.py | 2 +- competition/pull.py | 4 ++-- competition/requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/competition/data.py b/competition/data.py index b3cf8cf..2512f22 100644 --- a/competition/data.py +++ b/competition/data.py @@ -75,7 +75,7 @@ def get_metrics_data_formatted(client, competition): return out def get_pit_data_formatted(client, competition): - x=requests.get("https://titanscouting.epochml.org/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) + x=requests.get("https://scouting.titanrobotics2022.com/api/fetchAllTeamNicknamesAtCompetition?competition="+competition) x = x.json() x = x['data'] x = x.keys() diff --git a/competition/pull.py b/competition/pull.py index 9f5af43..67d3455 100644 --- a/competition/pull.py +++ b/competition/pull.py @@ -2,7 +2,7 @@ import requests from exceptions import APIError from dep import load_config -url = "https://titanscouting.epochml.org" +url = "https://scouting.titanrobotics2022.com" config_tra = {} load_config("config.json", config_tra) trakey = config_tra['persistent']['key']['tra'] @@ -60,4 +60,4 @@ def get_teams_at_competition(competition): if json['success']: return list(json['data'].keys()) else: - raise APIError(json) \ No newline at end of file + raise APIError(json) diff --git a/competition/requirements.txt b/competition/requirements.txt index 503e263..420b618 100644 --- a/competition/requirements.txt +++ b/competition/requirements.txt @@ -12,4 +12,4 @@ requests scikit-learn scipy six -tra-analysis \ No newline at end of file +tra-analysis