From b255475ac8aa126d87dfd0364ddf7b7b80bfee6d Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Tue, 17 Oct 2023 10:03:00 +0300 Subject: [PATCH 1/2] Update development configuration - Merged relevant parts of docker-compose.yml from mittaridatapumppu-dev - Moved mittaridatapumppu-dev:README.md contents to CONTRIBUTING.md - Added other mittaridatapumppu services as submodules - Cleaned up and updated docker-compose.yml --- .gitmodules | 12 ++ CONTRIBUTING.md | 122 ++++++++++++++++++ docker-compose.yml | 92 ++++++++----- docs/deviceregistry.md | 8 +- fvhiot-python | 1 + .../deviceregistry/settings.py | 2 +- mittaridatapumppu-endpoint | 1 + mittaridatapumppu-parser | 1 + mittaridatapumppu-persister | 1 + tests/README.md | 12 +- 10 files changed, 208 insertions(+), 44 deletions(-) create mode 100644 .gitmodules create mode 100644 CONTRIBUTING.md create mode 160000 fvhiot-python create mode 160000 mittaridatapumppu-endpoint create mode 160000 mittaridatapumppu-parser create mode 160000 mittaridatapumppu-persister diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ed96663 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,12 @@ +[submodule "mittaridatapumppu-endpoint"] + path = mittaridatapumppu-endpoint + url = git@github.com:City-of-Helsinki/mittaridatapumppu-endpoint.git +[submodule "mittaridatapumppu-parser"] + path = mittaridatapumppu-parser + url = git@github.com:City-of-Helsinki/mittaridatapumppu-parser.git +[submodule "mittaridatapumppu-persister"] + path = mittaridatapumppu-persister + url = git@github.com:City-of-Helsinki/mittaridatapumppu-persister.git +[submodule "fvhiot-python"] + path = fvhiot-python + url = git@github.com:ForumViriumHelsinki/FVHIoT-python.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2d5a26e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,122 @@ +## Introduction + +This repository contains a development environment for the Mittaridatapumppu project. + +It is based on the microservices architecture and consists of the following services: +- [deviceregistry](https://github.com/City-of-Helsinki/mittaridatapumppu) +- [endpoint](https://github.com/City-of-Helsinki/mittaridatapumppu-endpoint) +- [parser](https://github.com/City-of-Helsinki/mittaridatapumppu-parser) +- [persister](https://github.com/City-of-Helsinki/mittaridatapumppu-persister) + +Local Kafka is used as a message broker between the services. +For demo purposes there is also an InfluxDB V2 database for storing the parsed data. + +Included [docker-compose.yml](docker-compose.yml) file is for running +the whole pipeline locally. + +There are also some scripts for populating the database with real data. + +## Developer's guide + +### Clone the repo +```shell +git clone --recurse-submodules git@github.com:City-of-Helsinki/mittaridatapumppu.git +``` + +### Branches + +Create applicable branches in sub-repositories where needed while developing, +e.g. `feature/new-feature` or `bugfix/bugfix-description`. + +### Developing with Docker + +Set up the environment by building the images and loading initial data for development. + +```shell +docker compose up --build +docker compose exec deviceregistry python manage.py loaddata devices/fixtures/auth.json devices/fixtures/authtoken.json devices/fixtures/endpoints.json +``` + +Later on you can start the services with + +```shell +docker compose up +``` + +#### Developing without Docker + + +Do not use this for now. It is not tested with the latest code and probably does not work. + +```shell +dropdb deviceregistry +createdb deviceregistry +psql deviceregistry -c "create extension postgis;commit;" +export DEBUG=1 +export DATABASE_LOCAL=1 +python manage.py makemigrations +python manage.py migrate +python manage.py loaddata devices/fixtures/auth.json devices/fixtures/authtoken.json devices/fixtures/endpoints.json +``` + + +### Log in to Django admin + +Go to http://localhost:8000/admin/ and login with `root` and `ruutruut`. Check +that there are some +[endpoints](http://127.0.0.1:8000/admin/endpoints/endpoint/) defined. + +Verify that the [hosts endpoint](http://127.0.0.1:8000/api/v1/hosts/localhost/) +returns localhost and at least 2 endpoints. + +### Populate database with real data + +You will need to have the device registry service (deviceregistry) running and +an Excel file containing data of devices and locations and uirasmeta.py, +which contains additional metadata of UiRaS sensors. + +```shell +python mittaridatapumppu/devices/scripts/populate_db_via_api.py --api-url http://127.0.0.1:8000/api/v1/ --api-token abcdef1234567890abcdef1234567890abcdef12 --excel-file Mittaridatapumppu-initial-data.xlsx +python mittaridatapumppu/devices/scripts/populate_db_via_api.py --api-url http://127.0.0.1:8000/api/v1/ --api-token abcdef1234567890abcdef1234567890abcdef12 --uirasmeta + +# Try it out with httpie (https://httpie.org/): +http GET http://127.0.0.1:8000/api/v1/devices/70B3D57050011422/ "Authorization:Token abcdef1234567890abcdef1234567890abcdef12" +# The same with curl: +curl http://127.0.0.1:8000/api/v1/devices/70B3D57050011422/ -H "Authorization:Token abcdef1234567890abcdef1234567890abcdef12" | jq +``` + +### Send data to the pipeline + +```shell +python mittaridatapumppu-endpoint/tests/test_api2.py +``` + +Check docker logs what happens. + +### Check the data in InfluxDB + +Go to http://localhost:18086/ and login with `root` and `ruutruut`. + +### Check the data in Django admin + +Go to http://localhost:8000/admin/ and login with `root` and `ruutruut`. + +### Running the tests + +Currently, there are both Django tests and pytest tests. + +Run tests: + +```shell +docker compose exec deviceregistry python manage.py test devices/tests +docker compose exec deviceregistry pytest devices/tests/test_api_devicetype.py +docker compose exec deviceregistry pytest devices/tests/test_api_device.py +``` + +Later only pytest tests will be used. + +## Code Formatting + +## Linting + +## Pre-commit hook diff --git a/docker-compose.yml b/docker-compose.yml index 5e75958..7126f8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,79 +4,76 @@ version: '3.8' services: - endpoint-digita: + endpoint: image: ghcr.io/city-of-helsinki/mittaridatapumppu-endpoint + build: ./mittaridatapumppu-endpoint ports: - "8001:8000" depends_on: kafka: condition: service_healthy + restart: unless-stopped + volumes: + - ./mittaridatapumppu-endpoint:/home/app + - ./fvhiot-python/fvhiot:/home/app/fvhiot environment: - # Endpoint config - ALLOWED_IP_ADDRESSES: "127.0.0.1" - AUTH_TOKEN: "abcd1234" - DATA_SOURCE_NAME: "digita.thingpark.http" - ENDPOINT_PATH: "/digita/v2" - HTTP_REQUESTHANDLER: "endpoints.digita.aiothingpark" - # Kafka client config - KAFKA_HOST: "kafka" - KAFKA_PORT: 9092 + DEVREG_ENDPOINTS_URL: "http://deviceregistry:8000/api/v1/hosts/localhost/" + DEVREG_API_TOKEN: "abcdef1234567890abcdef1234567890abcdef12" KAFKA_BOOTSTRAP_SERVERS: "kafka:9092" - KAFKA_GROUP_ID: "digita_dev" - KAFKA_PARSED_DATA_TOPIC_NAME: "digita.parseddata" - KAFKA_RAW_DATA_TOPIC_NAME: "digita.rawdata" - # Debug config - LOG_LEVEL: "DEBUG" + UVICORN_LOG_LEVEL: "debug" + UVICORN_RELOAD: "true" DEBUG: 1 parser-digita: image: ghcr.io/city-of-helsinki/mittaridatapumppu-parser + build: ./mittaridatapumppu-parser + container_name: parser-digita depends_on: kafka: condition: service_healthy restart: unless-stopped + volumes: + - ./fvhiot-python/fvhiot:/home/app/fvhiot environment: # Kafka client config - KAFKA_HOST: "kafka" - KAFKA_PORT: 9092 KAFKA_BOOTSTRAP_SERVERS: "kafka:9092" KAFKA_GROUP_ID: "digita_dev" KAFKA_PARSED_DATA_TOPIC_NAME: "digita.parseddata" KAFKA_RAW_DATA_TOPIC_NAME: "digita.rawdata" # Parser config - DEVICE_REGISTRY_URL: "http://devreg:8000/api/v1" - DEVICE_REGISTRY_TOKEN: b48455759b691baf3b811ba437ce9e581fc0a37e + DEVICE_REGISTRY_URL: "http://deviceregistry:8000/api/v1" + DEVICE_REGISTRY_TOKEN: abcdef1234567890abcdef1234567890abcdef12 # Debug config - LOG_LEVEL: "DEBUG" DEBUG: 1 - DEV_SERVER: 1 persister-influxdb: image: ghcr.io/city-of-helsinki/mittaridatapumppu-persister + build: ./mittaridatapumppu-persister + container_name: persister-influxdb depends_on: kafka: condition: service_healthy +# influxdb2: +# condition: service_healthy restart: unless-stopped + volumes: + - ./fvhiot-python/fvhiot:/home/app/fvhiot environment: # Kafka client config - KAFKA_HOST: "kafka" - KAFKA_PORT: 9092 KAFKA_BOOTSTRAP_SERVERS: "kafka:9092" KAFKA_GROUP_ID: "digita_dev" KAFKA_PARSED_DATA_TOPIC_NAME: "digita.parseddata" KAFKA_RAW_DATA_TOPIC_NAME: "digita.rawdata" # Persister config - DEVICE_REGISTRY_URL: "http://devreg:8000/api/v1" - DEVICE_REGISTRY_TOKEN: b48455759b691baf3b811ba437ce9e581fc0a37e + DEVICE_REGISTRY_URL: "http://deviceregistry:8000/api/v1" + DEVICE_REGISTRY_TOKEN: abcdef1234567890abcdef1234567890abcdef12 # InfluxDB config - INFLUXDB_URL: "https://influx.fvh.io" - INFLUXDB_ORG: "Testing" - INFLUXDB_BUCKET: "DigitaTest" - INFLUXDB_TOKEN: "-DWlK7UjluBsFRgQ6ZTU82zfir89lAdY00hSvm-hhM-z2cmA-TwRekLj4C_KxU4jAU23H3vTkgGIQjqs68xHWg==" + INFLUXDB_URL: "http://influxdb2:8086" + INFLUXDB_ORG: "Development" + INFLUXDB_BUCKET: "devbucket" + INFLUXDB_TOKEN: "insercure-token-for-development-and-testing-purposes-only-KxU4jAU23H3vTk" # Debug config - LOG_LEVEL: "DEBUG" DEBUG: 1 - DEV_SERVER: 1 kafka: image: bitnami/kafka:3.4 @@ -102,10 +99,12 @@ services: test: kafka-topics.sh --bootstrap-server kafka:9092 --topic hc --create --if-not-exists && kafka-topics.sh --bootstrap-server kafka:9092 --topic hc --describe timeout: 5s - devreg: + deviceregistry: image: ghcr.io/city-of-helsinki/mittaridatapumppu-deviceregistry build: . + container_name: deviceregistry volumes: + - ./mittaridatapumppu-deviceregistry:/home/app - media:/media ports: - "8000:8000" @@ -116,15 +115,19 @@ services: SECRET_KEY: # PostgreSQL config DJANGO_DB_HOST: + DJANGO_DB_NAME: DJANGO_DB_USER: DJANGO_DB_PASSWORD: DJANGO_DB_PORT: # Debug config + UVICORN_LOG_LEVEL: "debug" + UVICORN_RELOAD: "true" DEBUG: 1 # UVICORN_RELOAD: true # UVICORN_LOG_LEVEL: true depends_on: - - db + db: + condition: service_healthy db: image: postgis/postgis:16-3.4 @@ -134,6 +137,29 @@ services: POSTGRES_PASSWORD: postgres volumes: - postgres_data:/var/lib/postgresql/data/ + healthcheck: + test: "pg_isready -U postgres -d postgres" + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + + influxdb2: + image: influxdb:2.7 + ports: + - "18086:8086" + environment: + DOCKER_INFLUXDB_INIT_MODE: setup + DOCKER_INFLUXDB_INIT_USERNAME: root + DOCKER_INFLUXDB_INIT_PASSWORD: ruutruut + DOCKER_INFLUXDB_INIT_ORG: Development + DOCKER_INFLUXDB_INIT_BUCKET: devbucket + DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: "insercure-token-for-development-and-testing-purposes-only-KxU4jAU23H3vTk" + healthcheck: + test: ["CMD", "curl", "influxdb2:18086/api/v2/ping"] + interval: 1m + timeout: 10s + retries: 5 volumes: postgres_data: diff --git a/docs/deviceregistry.md b/docs/deviceregistry.md index 3bda7ae..5ef97f2 100644 --- a/docs/deviceregistry.md +++ b/docs/deviceregistry.md @@ -34,8 +34,8 @@ pre-commit install ``` docker-compose up --build -d -docker-compose exec devreg python manage.py migrate -docker-compose exec devreg python manage.py createsuperuser +docker-compose exec deviceregistry python manage.py migrate +docker-compose exec deviceregistry python manage.py createsuperuser ``` @@ -44,13 +44,13 @@ docker-compose exec devreg python manage.py createsuperuser for api tests: ``` -docker-compose exec devreg pytest devices/tests/test_api.py +docker-compose exec deviceregistry pytest devices/tests/test_api.py ``` for model and admin page tests ``` -docker-compose exec devreg python manage.py test devices/tests +docker-compose exec deviceregistry python manage.py test devices/tests ``` # Misc stuff diff --git a/fvhiot-python b/fvhiot-python new file mode 160000 index 0000000..677797f --- /dev/null +++ b/fvhiot-python @@ -0,0 +1 @@ +Subproject commit 677797fa6e86a3d37b43cab53edb659418b7f256 diff --git a/mittaridatapumppu-deviceregistry/deviceregistry/settings.py b/mittaridatapumppu-deviceregistry/deviceregistry/settings.py index eca1962..ccbbf4a 100644 --- a/mittaridatapumppu-deviceregistry/deviceregistry/settings.py +++ b/mittaridatapumppu-deviceregistry/deviceregistry/settings.py @@ -20,7 +20,7 @@ SECRET_KEY=(str, "django-insecure-vz&7byj9esv1ncrv(7805g7w%h+-(-k_q82q(woh%1pcxr)^jf"), # Platta uses mSECRET_KEY as the name for the variable. mSECRET_KEY=(str, ""), - ALLOWED_HOSTS=(list, ["devreg", "localhost", "127.0.0.1", "[::1]"]), + ALLOWED_HOSTS=(list, ["deviceregistry", "localhost", "127.0.0.1", "[::1]"]), DJANGO_DB_NAME=(str, "postgres"), DJANGO_DB_USER=(str, "postgres"), DJANGO_DB_PASSWORD=(str, "postgres"), diff --git a/mittaridatapumppu-endpoint b/mittaridatapumppu-endpoint new file mode 160000 index 0000000..e71bd8f --- /dev/null +++ b/mittaridatapumppu-endpoint @@ -0,0 +1 @@ +Subproject commit e71bd8f5a24d5ff8b5cd07cd4f67e5f5d15f20d8 diff --git a/mittaridatapumppu-parser b/mittaridatapumppu-parser new file mode 160000 index 0000000..9510b50 --- /dev/null +++ b/mittaridatapumppu-parser @@ -0,0 +1 @@ +Subproject commit 9510b5071db9a59683d901796eb161bc066004a3 diff --git a/mittaridatapumppu-persister b/mittaridatapumppu-persister new file mode 160000 index 0000000..b7086fa --- /dev/null +++ b/mittaridatapumppu-persister @@ -0,0 +1 @@ +Subproject commit b7086fa1c4d32f7ea4185be4f3c64ba0f596c634 diff --git a/tests/README.md b/tests/README.md index 07df2b7..ae03603 100644 --- a/tests/README.md +++ b/tests/README.md @@ -32,8 +32,8 @@ docker-compose up --build -d ### Get deviceregstry up and functional ``` -docker-compose exec devreg python manage.py migrate -docker-compose exec devreg python manage.py createsuperuser +docker-compose exec deviceregistry python manage.py migrate +docker-compose exec deviceregistry python manage.py createsuperuser ``` @@ -44,14 +44,14 @@ This loads authtoken.token, which is also configured in docker-compose.yml and some test devices. ``` -docker-compose exec devreg python manage.py loaddata devices/fixtures/initialdata.json +docker-compose exec deviceregistry python manage.py loaddata devices/fixtures/initialdata.json ``` #### to run device registry tests: ``` -docker-compose exec devreg python manage.py test devices/tests -docker-compose exec devreg pytest devices/tests/test_api_device.py -docker-compose exec devreg pytest devices/tests/test_api_devicetype.py +docker-compose exec deviceregistry python manage.py test devices/tests +docker-compose exec deviceregistry pytest devices/tests/test_api_device.py +docker-compose exec deviceregistry pytest devices/tests/test_api_devicetype.py ``` ### Access Tokens From 5e0e211bf3052d770ae4c981b459cddb511aba5f Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Tue, 17 Oct 2023 11:59:49 +0300 Subject: [PATCH 2/2] Update InfluxDB env variable names These were changed in FVHIoT-python --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7126f8c..cdfc1b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,10 +68,10 @@ services: DEVICE_REGISTRY_URL: "http://deviceregistry:8000/api/v1" DEVICE_REGISTRY_TOKEN: abcdef1234567890abcdef1234567890abcdef12 # InfluxDB config - INFLUXDB_URL: "http://influxdb2:8086" - INFLUXDB_ORG: "Development" - INFLUXDB_BUCKET: "devbucket" - INFLUXDB_TOKEN: "insercure-token-for-development-and-testing-purposes-only-KxU4jAU23H3vTk" + INFLUX_HOST: "http://influxdb2:8086" + INFLUX_ORG: "Development" + INFLUX_BUCKET: "devbucket" + INFLUX_TOKEN: "insercure-token-for-development-and-testing-purposes-only-KxU4jAU23H3vTk" # Debug config DEBUG: 1