From 531c8f80f36d2170bef582fbb6625377b58a7b2c Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 14 Oct 2021 22:52:15 +0000 Subject: [PATCH 1/6] possible fix: UDP broadcast, not connecing from outside LAN Signed-off-by: Arthur Lu --- src/cli/client.py | 19 +++++++++++++++++ src/cli/superscript.py | 46 ++++++++++++------------------------------ 2 files changed, 32 insertions(+), 33 deletions(-) create mode 100644 src/cli/client.py diff --git a/src/cli/client.py b/src/cli/client.py new file mode 100644 index 0000000..e3e7241 --- /dev/null +++ b/src/cli/client.py @@ -0,0 +1,19 @@ +import socket + +client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) # UDP + +# Enable port reusage so we will be able to run multiple clients and servers on single (host, port). +# Do not use socket.SO_REUSEADDR except you using linux(kernel<3.9): goto https://stackoverflow.com/questions/14388706/how-do-so-reuseaddr-and-so-reuseport-differ for more information. +# For linux hosts all sockets that want to share the same address and port combination must belong to processes that share the same effective user ID! +# So, on linux(kernel>=3.9) you have to run multiple servers and clients under one user to share the same (host, port). +# Thanks to @stevenreddie +client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + +# Enable broadcasting mode +client.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + +client.connect(("", 5678)) +while True: + # Thanks @seym45 for a fix + data, addr = client.recvfrom(1024) + print("received message: %s"%data) \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 3dd42cc..36432cb 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -151,18 +151,16 @@ __all__ = [ # imports: -import asyncio import json import math from multiprocessing import Pool, freeze_support import os import pymongo +import socket import sys -import threading import time import traceback import warnings -import websockets from interface import splash, log, ERR, INF, stdout, stderr from data import get_previous_time, pull_new_tba_matches, set_current_time, load_match, push_match, load_pit, push_pit, get_database_config, set_database_config, check_new_database_matches @@ -541,44 +539,26 @@ def start(pid_path, verbose = False, profile = False, debug = False): sys.exit(exit_code) else: - + f = open('errorlog.log', 'w+') with daemon.DaemonContext( - working_directory=os.getcwd(), - pidfile=pidfile.TimeoutPIDLockFile(pid_path), - stderr=f + working_directory = os.getcwd(), + pidfile = pidfile.TimeoutPIDLockFile(pid_path), + stderr = f ): - async def handler(client, path): - clients.append(client) - while True: - try: - pong_waiter = await client.ping() - await pong_waiter - time.sleep(3) - except Exception as e: - clients.remove(client) - break + server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + server.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + server.settimeout(0.2) - async def send_one(client, data): - await client.send(data) - def send(target, level, message, code = 0): - message_clients = clients.copy() - for client in message_clients: - try: - asyncio.run(send_one(client, message)) - except: - pass - - clients = [] - start_server = websockets.serve(handler, "0.0.0.0", 5678) - - asyncio.get_event_loop().run_until_complete(start_server) - threading.Thread(target = asyncio.get_event_loop().run_forever).start() + server.sendto(bytes(message, 'utf-8'), ('', 5678)) exit_code = main(send) - sys.exit(exit_code) + server.close() + f.close() + sys.exit() def stop(pid_path): try: From 7b05707e3bf81f083805c13aed93e814f0373d30 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Thu, 14 Oct 2021 23:30:15 +0000 Subject: [PATCH 2/6] works locally again --- src/cli/client.py | 2 +- src/cli/data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/client.py b/src/cli/client.py index e3e7241..e96620a 100644 --- a/src/cli/client.py +++ b/src/cli/client.py @@ -12,7 +12,7 @@ client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) # Enable broadcasting mode client.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) -client.connect(("", 5678)) +client.bind(("", 5678)) while True: # Thanks @seym45 for a fix data, addr = client.recvfrom(1024) diff --git a/src/cli/data.py b/src/cli/data.py index c223a79..c46e726 100644 --- a/src/cli/data.py +++ b/src/cli/data.py @@ -193,7 +193,7 @@ def push_metric(client, competition, metric): def push_pit(client, competition, pit): for variable in pit: - + push_team_pit_data(client, competition, variable, pit[variable]) def check_new_database_matches(client, competition): From c0c46a1b9deb5e43d6f3714a40440092ceb06e36 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 15 Oct 2021 06:12:50 +0000 Subject: [PATCH 3/6] solution using zmq PUB/SUB, working locally --- src/cli/client.py | 24 ++++++++++-------------- src/cli/superscript.py | 18 +++++++++++------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/cli/client.py b/src/cli/client.py index e96620a..bc4a15b 100644 --- a/src/cli/client.py +++ b/src/cli/client.py @@ -1,19 +1,15 @@ -import socket +import signal +import zmq -client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) # UDP -# Enable port reusage so we will be able to run multiple clients and servers on single (host, port). -# Do not use socket.SO_REUSEADDR except you using linux(kernel<3.9): goto https://stackoverflow.com/questions/14388706/how-do-so-reuseaddr-and-so-reuseport-differ for more information. -# For linux hosts all sockets that want to share the same address and port combination must belong to processes that share the same effective user ID! -# So, on linux(kernel>=3.9) you have to run multiple servers and clients under one user to share the same (host, port). -# Thanks to @stevenreddie -client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) +signal.signal(signal.SIGINT, signal.SIG_DFL) -# Enable broadcasting mode -client.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) +context = zmq.Context() + +socket = context.socket(zmq.SUB) +socket.connect('tcp://localhost:5678') +socket.setsockopt(zmq.SUBSCRIBE, b'status') -client.bind(("", 5678)) while True: - # Thanks @seym45 for a fix - data, addr = client.recvfrom(1024) - print("received message: %s"%data) \ No newline at end of file + message = socket.recv_multipart() + print(f'Received: {message}') \ No newline at end of file diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 36432cb..371d18f 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -156,7 +156,6 @@ import math from multiprocessing import Pool, freeze_support import os import pymongo -import socket import sys import time import traceback @@ -314,6 +313,8 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "closed threads and database client") send(stdout, INF, "finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") + raise Exception("boop") + if profile: return 0 # return instead of break to avoid sys.exit @@ -547,16 +548,19 @@ def start(pid_path, verbose = False, profile = False, debug = False): stderr = f ): - server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - server.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - server.settimeout(0.2) + import zmq + + context = zmq.Context() + socket = context.socket(zmq.PUB) + socket.bind("tcp://*:5678") + + socket.send(b'status') def send(target, level, message, code = 0): - server.sendto(bytes(message, 'utf-8'), ('', 5678)) + socket.send(bytes("status: " + message, 'utf-8')) exit_code = main(send) - server.close() + socket.close() f.close() sys.exit() From 41fb81cfac711a7af9f64dbbd1bc408158e058fd Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Fri, 15 Oct 2021 06:20:05 +0000 Subject: [PATCH 4/6] verification of zmq solution working remotely Signed-off-by: Arthur Lu --- src/cli/superscript.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 371d18f..13ebc1b 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -160,6 +160,7 @@ import sys import time import traceback import warnings +import zmq from interface import splash, log, ERR, INF, stdout, stderr from data import get_previous_time, pull_new_tba_matches, set_current_time, load_match, push_match, load_pit, push_pit, get_database_config, set_database_config, check_new_database_matches @@ -313,8 +314,6 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "closed threads and database client") send(stdout, INF, "finished all tasks in " + str(time.time() - loop_start) + " seconds, looping") - raise Exception("boop") - if profile: return 0 # return instead of break to avoid sys.exit @@ -548,8 +547,6 @@ def start(pid_path, verbose = False, profile = False, debug = False): stderr = f ): - import zmq - context = zmq.Context() socket = context.socket(zmq.PUB) socket.bind("tcp://*:5678") From ab13e10271d605dbbee9305d65526aaf7d0d1c07 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 16 Oct 2021 20:21:12 +0000 Subject: [PATCH 5/6] moved sample zmq SUB client to test/ --- {src/cli => test}/client.py | 1 - 1 file changed, 1 deletion(-) rename {src/cli => test}/client.py (88%) diff --git a/src/cli/client.py b/test/client.py similarity index 88% rename from src/cli/client.py rename to test/client.py index bc4a15b..df4c0e4 100644 --- a/src/cli/client.py +++ b/test/client.py @@ -1,7 +1,6 @@ import signal import zmq - signal.signal(signal.SIGINT, signal.SIG_DFL) context = zmq.Context() From 828e45844eaeeb70ed90b4cc85a2c114b20b6d5e Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 16 Oct 2021 20:28:57 +0000 Subject: [PATCH 6/6] removed websockets from requirements.txr, fixed return code of daemon, removed websockets from hidden imports of superscript.spec --- src/cli/superscript.py | 2 +- src/cli/superscript.spec | 2 -- src/requirements.txt | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/cli/superscript.py b/src/cli/superscript.py index 13ebc1b..70f497f 100644 --- a/src/cli/superscript.py +++ b/src/cli/superscript.py @@ -559,7 +559,7 @@ def start(pid_path, verbose = False, profile = False, debug = False): exit_code = main(send) socket.close() f.close() - sys.exit() + sys.exit(exit_code) def stop(pid_path): try: diff --git a/src/cli/superscript.spec b/src/cli/superscript.spec index 7664ae0..04cafa7 100644 --- a/src/cli/superscript.spec +++ b/src/cli/superscript.spec @@ -10,8 +10,6 @@ a = Analysis(['superscript.py'], "dnspython", "sklearn.utils._weight_vector", "requests", - "websockets.legacy", - "websockets.legacy.server", ], hookspath=[], runtime_hooks=[], diff --git a/src/requirements.txt b/src/requirements.txt index 644ff65..88cc58b 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -15,5 +15,5 @@ pandas kivy==2.0.0rc2 -websockets +pyzmq python-daemon \ No newline at end of file