diff --git a/DEBUG_slurm-tra-superscript.out b/DEBUG_slurm-tra-superscript.out new file mode 100644 index 0000000..2384b35 --- /dev/null +++ b/DEBUG_slurm-tra-superscript.out @@ -0,0 +1,17 @@ +2022/04/08 14:45:12 | : #--------------------------------------# +2022/04/08 14:45:12 | : | superscript version: 1.0.0 | +2022/04/08 14:45:12 | : | os: Linux | +2022/04/08 14:45:12 | : | python: 3.9.7 | +2022/04/08 14:45:12 | : #--------------------------------------# +2022/04/08 14:45:12 | [INFO] : current time: 1649447112.4795144 +2022/04/08 14:45:12 | [INFO] : found and loaded config at +2022/04/08 14:45:12 | [INFO] : found and loaded database and tba keys +2022/04/08 14:45:12 | [INFO] : established connection to database +2022/04/08 14:45:13 | [INFO] : analysis backtimed to: 0.0 +2022/04/08 14:45:13 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:45:24 | [INFO] : match module finished in 11.742639541625977 seconds + Metrics: 0%| | 0/34 [00:00 +2022/04/08 13:39:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:39:18 | [INFO] : established connection to database +2022/04/08 13:39:19 | [INFO] : analysis backtimed to: 0.0 +2022/04/08 13:39:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:39:25 | [INFO] : match module finished in 6.360178709030151 seconds + Metrics: 0%| | 0/25 [00:00 +2022/04/08 13:44:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:44:18 | [INFO] : established connection to database +2022/04/08 13:44:19 | [INFO] : analysis backtimed to: 1649443158.5109549 +2022/04/08 13:44:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:44:26 | [INFO] : match module finished in 6.962671756744385 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 13:44:26 | [INFO] : metric module finished in 0.18820714950561523 seconds +2022/04/08 13:44:28 | [INFO] : pit module finished in 1.7953612804412842 seconds +2022/04/08 13:44:28 | [INFO] : closed threads and database client +2022/04/08 13:44:28 | [INFO] : finished all tasks in 9.482671737670898 seconds, looping +2022/04/08 13:44:28 | [INFO] : loop delayed by 290.5172882080078 seconds +2022/04/08 13:49:18 | [INFO] : current time: 1649443758.7116833 +2022/04/08 13:49:18 | [INFO] : found and loaded config at +2022/04/08 13:49:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:49:18 | [INFO] : established connection to database +2022/04/08 13:49:19 | [INFO] : analysis backtimed to: 1649443458.6113253 +2022/04/08 13:49:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:49:26 | [INFO] : match module finished in 7.532702207565308 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 13:49:26 | [INFO] : metric module finished in 0.06511259078979492 seconds +2022/04/08 13:49:28 | [INFO] : pit module finished in 1.819601058959961 seconds +2022/04/08 13:49:28 | [INFO] : closed threads and database client +2022/04/08 13:49:28 | [INFO] : finished all tasks in 9.962053775787354 seconds, looping +2022/04/08 13:49:28 | [INFO] : loop delayed by 290.03790378570557 seconds +2022/04/08 13:54:18 | [INFO] : current time: 1649444058.811996 +2022/04/08 13:54:18 | [INFO] : found and loaded config at +2022/04/08 13:54:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:54:18 | [INFO] : established connection to database +2022/04/08 13:54:19 | [INFO] : analysis backtimed to: 1649443758.7116833 +2022/04/08 13:54:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:54:26 | [INFO] : match module finished in 7.571904420852661 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 13:54:26 | [INFO] : metric module finished in 0.15475034713745117 seconds +2022/04/08 13:54:28 | [INFO] : pit module finished in 1.6621544361114502 seconds +2022/04/08 13:54:28 | [INFO] : closed threads and database client +2022/04/08 13:54:28 | [INFO] : finished all tasks in 9.827200174331665 seconds, looping +2022/04/08 13:54:28 | [INFO] : loop delayed by 290.17272448539734 seconds +2022/04/08 13:59:18 | [INFO] : current time: 1649444358.912293 +2022/04/08 13:59:18 | [INFO] : found and loaded config at +2022/04/08 13:59:18 | [INFO] : found and loaded database and tba keys +2022/04/08 13:59:19 | [INFO] : established connection to database +2022/04/08 13:59:19 | [INFO] : analysis backtimed to: 1649444058.811996 +2022/04/08 13:59:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 13:59:30 | [INFO] : match module finished in 10.847434997558594 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 13:59:30 | [INFO] : metric module finished in 0.10496973991394043 seconds +2022/04/08 13:59:32 | [INFO] : pit module finished in 1.7203879356384277 seconds +2022/04/08 13:59:32 | [INFO] : closed threads and database client +2022/04/08 13:59:32 | [INFO] : finished all tasks in 13.211304664611816 seconds, looping +2022/04/08 13:59:32 | [INFO] : loop delayed by 286.7886555194855 seconds +2022/04/08 14:04:19 | [INFO] : current time: 1649444659.012761 +2022/04/08 14:04:19 | [INFO] : found and loaded config at +2022/04/08 14:04:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:04:19 | [INFO] : established connection to database +2022/04/08 14:04:19 | [INFO] : analysis backtimed to: 1649444358.912293 +2022/04/08 14:04:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:04:28 | [INFO] : match module finished in 9.347296714782715 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:04:28 | [INFO] : metric module finished in 0.10346341133117676 seconds +2022/04/08 14:04:31 | [INFO] : pit module finished in 2.109464168548584 seconds +2022/04/08 14:04:31 | [INFO] : closed threads and database client +2022/04/08 14:04:31 | [INFO] : finished all tasks in 12.072788000106812 seconds, looping +2022/04/08 14:04:31 | [INFO] : loop delayed by 287.92717266082764 seconds +2022/04/08 14:09:19 | [INFO] : current time: 1649444959.0845735 +2022/04/08 14:09:19 | [INFO] : found and loaded config at +2022/04/08 14:09:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:09:19 | [INFO] : established connection to database +2022/04/08 14:09:19 | [INFO] : analysis backtimed to: 1649444659.012761 +2022/04/08 14:09:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:09:30 | [INFO] : match module finished in 10.792919158935547 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:09:30 | [INFO] : metric module finished in 0.08684110641479492 seconds +2022/04/08 14:09:32 | [INFO] : pit module finished in 1.9306814670562744 seconds +2022/04/08 14:09:32 | [INFO] : closed threads and database client +2022/04/08 14:09:32 | [INFO] : finished all tasks in 13.267388105392456 seconds, looping +2022/04/08 14:09:32 | [INFO] : loop delayed by 286.73254227638245 seconds +2022/04/08 14:14:19 | [INFO] : current time: 1649445259.0884073 +2022/04/08 14:14:19 | [INFO] : found and loaded config at +2022/04/08 14:14:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:14:19 | [INFO] : established connection to database +2022/04/08 14:14:19 | [INFO] : analysis backtimed to: 1649444959.0845735 +2022/04/08 14:14:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:14:29 | [INFO] : match module finished in 9.690043449401855 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:14:29 | [INFO] : metric module finished in 0.1428515911102295 seconds +2022/04/08 14:14:31 | [INFO] : pit module finished in 1.9340870380401611 seconds +2022/04/08 14:14:31 | [INFO] : closed threads and database client +2022/04/08 14:14:31 | [INFO] : finished all tasks in 12.373121976852417 seconds, looping +2022/04/08 14:14:31 | [INFO] : loop delayed by 287.6267948150635 seconds +2022/04/08 14:19:19 | [INFO] : current time: 1649445559.188727 +2022/04/08 14:19:19 | [INFO] : found and loaded config at +2022/04/08 14:19:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:19:19 | [INFO] : established connection to database +2022/04/08 14:19:19 | [INFO] : analysis backtimed to: 1649445259.0884073 +2022/04/08 14:19:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:19:30 | [INFO] : match module finished in 10.267187118530273 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:19:30 | [INFO] : metric module finished in 0.5178976058959961 seconds +2022/04/08 14:19:32 | [INFO] : pit module finished in 1.655653476715088 seconds +2022/04/08 14:19:32 | [INFO] : closed threads and database client +2022/04/08 14:19:32 | [INFO] : finished all tasks in 13.050731897354126 seconds, looping +2022/04/08 14:19:32 | [INFO] : loop delayed by 286.9491786956787 seconds +2022/04/08 14:24:19 | [INFO] : current time: 1649445859.2890737 +2022/04/08 14:24:19 | [INFO] : found and loaded config at +2022/04/08 14:24:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:24:19 | [INFO] : established connection to database +2022/04/08 14:24:19 | [INFO] : analysis backtimed to: 1649445559.188727 +2022/04/08 14:24:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:24:30 | [INFO] : match module finished in 11.110676765441895 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:24:31 | [INFO] : metric module finished in 0.09472298622131348 seconds +2022/04/08 14:24:32 | [INFO] : pit module finished in 1.9062983989715576 seconds +2022/04/08 14:24:33 | [INFO] : closed threads and database client +2022/04/08 14:24:33 | [INFO] : finished all tasks in 13.732285976409912 seconds, looping +2022/04/08 14:24:33 | [INFO] : loop delayed by 286.2676737308502 seconds +2022/04/08 14:29:19 | [INFO] : current time: 1649446159.3893542 +2022/04/08 14:29:19 | [INFO] : found and loaded config at +2022/04/08 14:29:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:29:19 | [INFO] : established connection to database +2022/04/08 14:29:19 | [INFO] : analysis backtimed to: 1649445859.2890737 +2022/04/08 14:29:19 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:29:31 | [INFO] : match module finished in 11.64138388633728 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:29:31 | [INFO] : metric module finished in 0.08331465721130371 seconds +2022/04/08 14:29:33 | [INFO] : pit module finished in 1.7486073970794678 seconds +2022/04/08 14:29:33 | [INFO] : closed threads and database client +2022/04/08 14:29:33 | [INFO] : finished all tasks in 14.08924150466919 seconds, looping +2022/04/08 14:29:33 | [INFO] : loop delayed by 285.9107177257538 seconds +2022/04/08 14:34:19 | [INFO] : current time: 1649446459.4896593 +2022/04/08 14:34:19 | [INFO] : found and loaded config at +2022/04/08 14:34:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:34:19 | [INFO] : established connection to database +2022/04/08 14:34:20 | [INFO] : analysis backtimed to: 1649446159.3893542 +2022/04/08 14:34:20 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:34:34 | [INFO] : match module finished in 14.8920156955719 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +2022/04/08 14:34:35 | [INFO] : metric module finished in 0.11336660385131836 seconds +2022/04/08 14:34:36 | [INFO] : pit module finished in 1.7640228271484375 seconds +2022/04/08 14:34:36 | [INFO] : closed threads and database client +2022/04/08 14:34:36 | [INFO] : finished all tasks in 17.381346464157104 seconds, looping +2022/04/08 14:34:36 | [INFO] : loop delayed by 282.6186122894287 seconds +2022/04/08 14:39:19 | [INFO] : current time: 1649446759.5899608 +2022/04/08 14:39:19 | [INFO] : found and loaded config at +2022/04/08 14:39:19 | [INFO] : found and loaded database and tba keys +2022/04/08 14:39:19 | [INFO] : established connection to database +2022/04/08 14:39:20 | [INFO] : analysis backtimed to: 1649446459.4896593 +2022/04/08 14:39:20 | [INFO] : config-preference set to local/client, loading local config information +2022/04/08 14:39:32 | [INFO] : match module finished in 12.305787801742554 seconds + Metrics: 0it [00:00, ?it/s] Metrics: 0it [00:00, ?it/s] +slurmstepd-cpu-dl580-0: error: *** JOB 242 ON cpu-dl580-0 CANCELLED AT 2022-04-08T14:41:13 *** diff --git a/competition/data.py b/competition/data.py index 4dd53c5..3b798f9 100644 --- a/competition/data.py +++ b/competition/data.py @@ -4,13 +4,13 @@ import pull import pandas as pd import json -def pull_new_tba_matches(apikey, competition, cutoff): +def pull_new_tba_matches(apikey, competition, last_match): api_key= apikey x=requests.get("https://www.thebluealliance.com/api/v3/event/"+competition+"/matches/simple", headers={"X-TBA-Auth-Key":api_key}) json = x.json() out = [] for i in json: - if i["actual_time"] != None and i["comp_level"] == "qm" and i["actual_time"]-cutoff >= 0: + if i["actual_time"] != None and i["comp_level"] == "qm" and i["match_number"] > last_match : out.append({"match" : i['match_number'], "blue" : list(map(lambda x: int(x[3:]), i['alliances']['blue']['team_keys'])), "red" : list(map(lambda x: int(x[3:]), i['alliances']['red']['team_keys'])), "winner": i["winning_alliance"]}) out.sort(key=lambda x: x['match']) return out diff --git a/competition/module.py b/competition/module.py index 69aa5b0..c3635aa 100644 --- a/competition/module.py +++ b/competition/module.py @@ -170,17 +170,19 @@ class Metric (Module): self._push_results() def _load_data(self): - self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp) + self.last_match = d.get_analysis_flags(self.apikey, 'metrics_last_match')['metrics_last_match'] + print("Previous last match", self.last_match) + self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.last_match) def _process_data(self): self.results = {} - + self.match = 0 matches = self.data - red = {} blu = {} for match in tqdm(matches, desc="Metrics"): # grab matches and loop through each one + self.match = max(self.match, int(match['match'])) red = d.load_metric(self.apikey, self.competition, match, "red", self.config["tests"]) # get the current ratings for red blu = d.load_metric(self.apikey, self.competition, match, "blue", self.config["tests"]) # get the current ratings for blue @@ -248,7 +250,8 @@ class Metric (Module): self.results[match['match']] = temp_vector d.push_metric(self.apikey, self.competition, temp_vector) # push new scores to db - + print("New last match", self.match) + d.set_analysis_flags(self.apikey, 'metrics_last_match', {'metrics_last_match': self.match}) def _push_results(self): pass