From 6e5484f5f1356edb50d61b4e7e93f4e77018165e Mon Sep 17 00:00:00 2001 From: Kevin Donahue Date: Sat, 27 Jul 2024 19:49:56 -0400 Subject: [PATCH] use different port for websocket server --- README.md | 50 ++++++++++++++++++++++++++++--------- docker-compose.yml | 2 +- pc/pc/config.py | 2 +- pc/pc/main.py | 2 +- pc/pc/websockets_handler.py | 2 +- 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 337a005..cf7fbee 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,35 @@ # ctts This project was inspired by wanting **a way of seeing what the -[sky brightness](https://en.wikipedia.org/wiki/Sky_brightness) is at different points around the globe, and how that is -changing over time**.. +[sky brightness](https://en.wikipedia.org/wiki/Sky_brightness) +is over the entire earth surface, and how it changes over time**.. -Needless to say, we cannot have sensors for this sort of thing -everwhere we would want a measurement, therefore we need a way -of performing this measurement -in a less direct way.. +Given that it would be infeasible to have [sensors](http://unihedron.com/projects/darksky/TSL237-E32.pdf) +everywhere that we want a brightness measurement, it would make +sense to have a way of performing this measurement indirectly. --- The approach this project takes is to model the relationship between a set of independent variables and the dependent variable -(sky brightness) available in a [public dataset](http://www.unihedron.com/projects/darksky/database/?csv=true). +(sky brightness) available in a [public dataset](http://www.unihedron.com/projects/darksky/database/?csv=true) using +pytorch. H3 is then used to discretize cells over the earth that form the basis of the requests made to the api server serving up the sky brightness. -Another component pushes brightness values from the model onto -rabbitmq while saving them to postgres, to enable historical lookup. +This component pushes brightness values from the model onto +rabbitmq while another component consumes them - inserting to +postgres to enable historical lookup. ## running locally -this will spin up the process of the prediction producer container repeatedly asking the api server for sky brightness -measurements across all [resolution 0 h3 cells](https://h3geo.org/docs/core-library/restable/) and publishing to -rabbitmq, which the consumer container reads from and stores in postgres. +This will spin up the process of the prediction producer container +repeatedly asking the api container for sky brightness measurements +_at the current time_ across all [resolution 0 h3 cells](https://h3geo.org/docs/core-library/restable/); +publishing them to rabbitmq, which the consumer container reads from +and stores messages (in postgres container). ```shell # create the volume for weather data @@ -38,3 +41,26 @@ docker volume create --name open-meteo-data # run the containers docker-compose up --build ``` + +Rabbitmq will take time to start up, at which time `producer` and +`consumer` containers will attempt restart to form connection. +Once rabbitmq does start, there should be output like this: + +```shell +producer-1 | 2024-07-27 23:48:09,301 [INFO] publishing brightness message {'uuid': '86967a4d-d6a5-4421-8db1-73d9d0d45ea5', 'lat': 11.509775527199592, 'lon': -55.499062349013, 'h3_id': '805ffffffffffff', 'utc_iso': '2024-07-27T23:48:09.301613', 'utc_ns': 1722124089301613056, 'mpsas': 8.6264, 'model_version': '0.1.0'} +producer-1 | 2024-07-27 23:48:09,304 [INFO] 805ffffffffffff has had 2 predictions published +consumer-1 | 2024-07-27 23:48:09,321 [INFO] inserting brightness message for 805ffffffffffff +consumer-1 | 2024-07-27 23:48:09,327 [INFO] broadcasting to 0 websocket clients on consumer:8090 +api-1 | 2024-07-27 23:48:09,922 [INFO] 172.19.0.7:43300 - "GET /api/v1/predict?lat=16.702868303031234&lon=-13.374845104752373 HTTP/1.1" 200 +producer-1 | 2024-07-27 23:48:09,926 [INFO] HTTP Request: GET http://api:8000/api/v1/predict?lat=16.702868303031234&lon=-13.374845104752373 "HTTP/1.1 200 OK" +producer-1 | 2024-07-27 23:48:09,927 [INFO] publishing brightness message {'uuid': '01d5912d-752b-4611-8de9-4f092dc21c5c', 'lat': 16.702868303031234, 'lon': -13.374845104752373, 'h3_id': '8055fffffffffff', 'utc_iso': '2024-07-27T23:48:09.927243', 'utc_ns': 1722124089927243008, 'mpsas': 4.5332, 'model_version': '0.1.0'} +producer-1 | 2024-07-27 23:48:09,930 [INFO] 8055fffffffffff has had 3 predictions published +consumer-1 | 2024-07-27 23:48:09,945 [INFO] inserting brightness message for 8055fffffffffff +consumer-1 | 2024-07-27 23:48:09,950 [INFO] broadcasting to 0 websocket clients on consumer:8090 +api-1 | 2024-07-27 23:48:10,540 [INFO] 172.19.0.7:43300 - "GET /api/v1/predict?lat=-7.460529604384309&lon=84.45314174117765 HTTP/1.1" 200 +producer-1 | 2024-07-27 23:48:10,544 [INFO] HTTP Request: GET http://api:8000/api/v1/predict?lat=-7.460529604384309&lon=84.45314174117765 "HTTP/1.1 200 OK" +producer-1 | 2024-07-27 23:48:10,547 [INFO] publishing brightness message {'uuid': 'd015ee51-d60e-4722-9fb3-f442fd3c52e3', 'lat': -7.460529604384309, 'lon': 84.45314174117765, 'h3_id': '8087fffffffffff', 'utc_iso': '2024-07-27T23:48:10.547142', 'utc_ns': 1722124090547142144, 'mpsas': 9.0234, 'model_version': '0.1.0'} +producer-1 | 2024-07-27 23:48:10,549 [INFO] 8087fffffffffff has had 4 predictions published +consumer-1 | 2024-07-27 23:48:10,565 [INFO] inserting brightness message for 8087fffffffffff +consumer-1 | 2024-07-27 23:48:10,572 [INFO] broadcasting to 0 websocket clients on consumer:8090 +``` \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 624ac8b..a924648 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,7 +73,7 @@ services: consumer: build: ./pc ports: - - "8080:8080" + - "8090:8090" environment: AMQP_HOST: "rabbitmq" PG_DATABASE: "postgres" diff --git a/pc/pc/config.py b/pc/pc/config.py index 0ab1fbf..f45ddf3 100644 --- a/pc/pc/config.py +++ b/pc/pc/config.py @@ -14,4 +14,4 @@ AMQP_PREDICTION_QUEUE = os.getenv("AMQP_PREDICTION_QUEUE", "prediction") WS_HOST = os.getenv("WS_HOST", "consumer") -WS_PORT = int(os.getenv("WS_PORT", 8080)) +WS_PORT = int(os.getenv("WS_PORT", 8090)) diff --git a/pc/pc/main.py b/pc/pc/main.py index eebf8fb..e2b584c 100644 --- a/pc/pc/main.py +++ b/pc/pc/main.py @@ -9,7 +9,7 @@ from pc.model import BrightnessMessage from pc.websockets_handler import WebSocketsHandler -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") log = logging.getLogger(__name__) websockets_handler = WebSocketsHandler() diff --git a/pc/pc/websockets_handler.py b/pc/pc/websockets_handler.py index 9202be0..2792a91 100644 --- a/pc/pc/websockets_handler.py +++ b/pc/pc/websockets_handler.py @@ -28,6 +28,6 @@ async def register_client(websocket): async def broadcast(self, message: BrightnessMessage): """send the message to all websockets""" - log.info(f"broadcasting to {len(self.clients)} clients") + log.info(f"broadcasting to {len(self.clients)} websocket clients on {WS_HOST}:{WS_PORT}") message_json = json.dumps(asdict(message)) broadcast(self.clients, message_json)