Use DuckDB to store device states (#2104)

.github/workflows/docker-dev-build.yml

@@ -10,7 +10,7 @@ on:
       platforms:
         description: 'Docker platform to build'
         required: true
-        default: 'linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8'
+        default: 'linux/amd64,linux/arm/v7,linux/arm64/v8'
 
 jobs:
   build-front:

.github/workflows/docker-release-build.yml

@@ -168,7 +168,7 @@ jobs:
         with:
           context: .
           file: ./docker/Dockerfile.buildx
-          platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8
+          platforms: linux/amd64,linux/arm/v7,linux/arm64/v8
           push: true
           pull: true
           tags: ${{ steps.docker_meta.outputs.tags }}

.gitignore

@@ -1,5 +1,7 @@
 node_modules
 *.db
+*.duckdb
+*.duckdb.wal
 *.db-shm
 *.db-wal
 *.dbfile-shm

docker/Dockerfile

@@ -2,7 +2,7 @@ ARG TARGET
 ARG VERSION
 ARG BUILD_DATE
 
-FROM $TARGET/node:18-alpine
+FROM ${TARGET}/node:18-slim
 
 LABEL \
   org.label-schema.build-date=$BUILD_DATE \
@@ -11,28 +11,43 @@ LABEL \
 COPY qemu-* /usr/bin/
 
 # System dependencies
-RUN apk add --no-cache tzdata nmap ffmpeg sqlite openssl gzip eudev
+RUN apt-get update && apt-get install -y --no-install-recommends \
+         tzdata \
+         nmap \
+         ffmpeg \
+         sqlite3 \
+         openssl \
+         gzip \
+         udev \
+         bluez \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /tmp
 
-# Install Bluez dependencies
-RUN apk add --no-cache bluez
-
 # Install Gladys
 RUN mkdir /src
 WORKDIR /src
 ADD . /src
 COPY ./static /src/server/static
 WORKDIR /src/server
-RUN apk add --no-cache --virtual .build-deps make gcc g++ python3 py3-setuptools git libffi-dev linux-headers \
-    && npm ci --unsafe-perm --production \
-    && npm cache clean --force \
-    && apk del .build-deps
+
+# Install build dependencies and application dependencies
+RUN apt-get update && apt-get install -y \
+         build-essential \
+         python3 \
+         python3-pip \
+         git \
+         libffi-dev \
+         && npm ci --unsafe-perm --production \
+         && npm cache clean --force \
+         && apt-get autoremove -y build-essential python3 python3-pip git libffi-dev \
+         && apt-get purge -y --auto-remove \
+         && rm -rf /var/lib/apt/lists/*
 
 ENV NODE_ENV production
 ENV SERVER_PORT 80
 
 # Export listening port
 EXPOSE 80
 
-CMD ["node", "index.js"]
+CMD ["node", "index.js"]

docker/Dockerfile.buildx

@@ -1,38 +1,46 @@
 # STEP 1
 # Prepare server package*.json files
-FROM node:18-alpine as json-files
+FROM node:18-slim as json-files
 COPY ./server /json-files/server
 WORKDIR /json-files/server/
 RUN find . -type f \! -name "package*.json" -exec rm -r {} \;
 COPY ./server/cli /json-files/server/cli
 COPY ./server/utils /json-files/server/utils
 
-
 # STEP 3
 # Gladys Bundle
-FROM node:18-alpine as gladys
+FROM node:18-slim as gladys
 
 # System dependencies
-RUN apk add --no-cache \
+RUN apt-get update && apt-get install -y --no-install-recommends \
          tzdata \
          nmap \
          ffmpeg \
-         sqlite \
+         sqlite3 \
          openssl \
          gzip \
-         eudev \
-         bluez
+         udev \
+         bluez \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --from=json-files /json-files/server /src/server
 
 ENV LD_LIBRARY_PATH /lib
 
 WORKDIR /src/server
 
-RUN apk add --no-cache --virtual .build-deps make gcc g++ python3 py3-setuptools git libffi-dev linux-headers \
-    && npm ci --unsafe-perm --production \
-    && npm cache clean --force \
-    && apk del .build-deps
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+         build-essential \
+         python3 \
+         python3-pip \
+         git \
+         libffi-dev \
+         && npm ci --unsafe-perm --production \
+         && npm cache clean --force \
+         && apt-get autoremove -y build-essential python3 python3-pip git libffi-dev \
+         && apt-get purge -y --auto-remove \
+         && rm -rf /var/lib/apt/lists/*
 
 # Copy builded front
 COPY ./static /src/server/static

front/src/actions/signup/signupSetPreferences.js

@@ -59,12 +59,6 @@ function createActions(store) {
         await state.httpClient.post(`/api/v1/variable/${SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HISTORY_IN_DAYS}`, {
           value: state.signupSystemPreferences[SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HISTORY_IN_DAYS]
         });
-        await state.httpClient.post(
-          `/api/v1/variable/${SYSTEM_VARIABLE_NAMES.DEVICE_AGGREGATE_STATE_HISTORY_IN_DAYS}`,
-          {
-            value: state.signupSystemPreferences[SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HISTORY_IN_DAYS]
-          }
-        );
         store.setState({
           signupSaveSystemPreferences: RequestStatus.Success
         });

front/src/components/boxs/chart/ApexChartAreaOptions.js

@@ -51,7 +51,14 @@ const getApexChartAreaOptions = ({ displayAxes, height, series, colors, locales,
     },
     yaxis: {
       labels: {
-        padding: 4
+        padding: 4,
+        formatter: function(value) {
+          if (Math.abs(value) < 1) {
+            return value; // For very low values, like crypto prices, use the normal value
+          } else {
+            return value.toFixed(2); // 2 decimal places for other values
+          }
+        }
       }
     },
     colors,

front/src/components/boxs/chart/ApexChartBarOptions.js

@@ -59,7 +59,14 @@ const getApexChartBarOptions = ({ displayAxes, series, colors, locales, defaultL
     },
     yaxis: {
       labels: {
-        padding: 4
+        padding: 4,
+        formatter: function(value) {
+          if (Math.abs(value) < 1) {
+            return value; // For very low values, like crypto prices, use the normal value
+          } else {
+            return value.toFixed(2); // 2 decimal places for other values
+          }
+        }
       }
     },
     colors,

front/src/components/boxs/chart/ApexChartLineOptions.js

@@ -50,7 +50,14 @@ const getApexChartLineOptions = ({ height, displayAxes, series, colors, locales,
     },
     yaxis: {
       labels: {
-        padding: 4
+        padding: 4,
+        formatter: function(value) {
+          if (Math.abs(value) < 1) {
+            return value; // For very low values, like crypto prices, use the normal value
+          } else {
+            return value.toFixed(2); // 2 decimal places for other values
+          }
+        }
       }
     },
     colors,

front/src/components/boxs/chart/ApexChartStepLineOptions.js

@@ -49,7 +49,14 @@ const getApexChartStepLineOptions = ({ height, displayAxes, series, colors, loca
     },
     yaxis: {
       labels: {
-        padding: 4
+        padding: 4,
+        formatter: function(value) {
+          if (Math.abs(value) < 1) {
+            return value; // For very low values, like crypto prices, use the normal value
+          } else {
+            return value.toFixed(2); // 2 decimal places for other values
+          }
+        }
       }
     },
     colors,

front/src/config/i18n/de.json

@@ -18,7 +18,9 @@
     "fahrenheit": "F",
     "metersPerSec": "m/s",
     "milesPerHour": "m/h",
-    "selectPlaceholder": "Auswählen …"
+    "selectPlaceholder": "Auswählen …",
+    "yes": "Ja",
+    "no": "Nein"
   },
   "color": {
     "aqua": "Aqua",
@@ -231,6 +233,7 @@
       "houseLabel": "Zuhause",
       "tabletModeDisabled": "Tablet-Modus deaktiviert"
     },
+    "duckDbMigrationInProgress": "Ihre Instanz Gladys ist dabei, ihre Datenbank auf DuckDB umzustellen, ein neues, leistungsfähigeres Datenbanksystem für Zeitseriendaten. Diese Aufgabe kann einige Zeit in Anspruch nehmen, und während dieser Zeit werden nicht alle Ihre Grafiken verfügbar sein. Fortschritt der Migration = {{progress}}%",
     "editDashboardSaveButton": "Sichern",
     "emptyDashboardSentenceTop": "Dein Dashboard wurde noch nicht konfiguriert.",
     "emptyDashboardSentenceBottom": "Klicke auf \"Bearbeiten\", um dein Dashboard zu gestalten.",
@@ -2362,7 +2365,9 @@
       "gladys-gateway-backup": "Gladys-Plus-Backup",
       "device-state-purge-single-feature": "Zustände einzelner Gerätemerkmale aufräumen",
       "vacuum": "Datenbank reinigen",
-      "service-zigbee2mqtt-backup": "Zigbee2MQTT-Backup"
+      "service-zigbee2mqtt-backup": "Zigbee2MQTT-Backup",
+      "migrate-sqlite-to-duckdb": "DuckDB migration",
+      "device-state-purge-all-sqlite-states": "Bereinigung aller SQLite-Berichte"
     },
     "jobErrors": {
       "purged-when-restarted": "Gladys Assistant wurde neu gestartet, während diese Aufgabe noch lief. Daher wurde sie gelöscht. Das bedeutet nicht, dass die Aufgabe fehlgeschlagen ist – es handelt sich um normales Verhalten."
@@ -2415,7 +2420,18 @@
       "dead": "Beendet"
     },
     "batteryLevel": "Batteriestandswarnung",
-    "batteryLevelDescription": "Jeden Samstag um 9:00 Uhr wird eine Nachricht an alle Administratoren gesendet, wenn der Batteriestand eines Geräts unter den gewählten Schwellenwert fällt."
+    "batteryLevelDescription": "Jeden Samstag um 9:00 Uhr wird eine Nachricht an alle Administratoren gesendet, wenn der Batteriestand eines Geräts unter den gewählten Schwellenwert fällt.",
+    "duckDbMigrationTitle": "Migration zu DuckDB",
+    "duckDbMigrationDescription": "Wir migrieren zu einem neuen Datenbanksystem für Sensordaten. Gladys wird Ihre Daten automatisch migrieren, aber Ihre Daten in Ihrer SQLite-Datenbank nicht löschen, um Datenverlust zu vermeiden.",
+    "duckDbMigrationProgressTitle": "Migrationsfortschritt",
+    "duckDbMigrationMigrationDone": "Migration abgeschlossen:",
+    "duckDbNumberOfStatesinSQlite": "Anzahl der Zustände in SQLite",
+    "duckDbNumberOfStatesinDuckDb": "Anzahl der Zustände in DuckDB",
+    "restartMigrationTitle": "Migration neu starten",
+    "confirm": "Bestätigen",
+    "cancel": "Abbrechen",
+    "purgeSQliteTitle": "SQLite-Zustände löschen",
+    "purgeSQliteDescription": "Sobald alle Zustände aus SQLite gelöscht wurden, müssen Sie auf der rechten Registerkarte eine \"Datenbankbereinigung\" durchführen, um Speicherplatz freizugeben. Dieser Schritt kann einige Zeit in Anspruch nehmen."
   },
   "newArea": {
     "createNewZoneButton": "Neue Zone erstellen",

front/src/config/i18n/en.json

@@ -18,7 +18,9 @@
     "fahrenheit": "F",
     "metersPerSec": "m/s",
     "milesPerHour": "m/h",
-    "selectPlaceholder": "Select..."
+    "selectPlaceholder": "Select...",
+    "yes": "Yes",
+    "no": "No"
   },
   "color": {
     "aqua": "Aqua",
@@ -231,6 +233,7 @@
       "houseLabel": "House",
       "tabletModeDisabled": "Tablet Mode Disabled"
     },
+    "duckDbMigrationInProgress": "Votre instance Gladys est entrain de migrer sa base de donnée à DuckDB, un nouveau système de base de donnée plus performant pour les données time-series. Cette tâche peut prendre un certain temps, et pendant ce temps vos graphiques ne seront pas tous disponibles. Progression de la migration = {{progress}}%",
     "editDashboardSaveButton": "Save",
     "emptyDashboardSentenceTop": "Looks like your dashboard is not configured yet.",
     "emptyDashboardSentenceBottom": "Click on the \"Edit\" button to design your dashboard.",
@@ -2362,7 +2365,9 @@
       "gladys-gateway-backup": "Gladys Plus backup",
       "device-state-purge-single-feature": "Single device feature states clean",
       "vacuum": "Database cleaning",
-      "service-zigbee2mqtt-backup": "Zigbee2MQTT backup"
+      "service-zigbee2mqtt-backup": "Zigbee2MQTT backup",
+      "migrate-sqlite-to-duckdb": "DuckDB migration",
+      "device-state-purge-all-sqlite-states": "Purge all SQLite states"
     },
     "jobErrors": {
       "purged-when-restarted": "Gladys Assistant restarted while this job was still running, so it was purged. It doesn't mean the job has failed, it's a normal behavior."
@@ -2415,7 +2420,18 @@
       "dead": "Dead"
     },
     "batteryLevel": "Battery level alert",
-    "batteryLevelDescription": "Every Saturday at 9:00 am, a message will be sent to all administrators if a device's battery level falls below the chosen threshold."
+    "batteryLevelDescription": "Every Saturday at 9:00 am, a message will be sent to all administrators if a device's battery level falls below the chosen threshold.",
+    "duckDbMigrationTitle": "Migration to DuckDB",
+    "duckDbMigrationDescription": "We are migrating to a new database system for sensor data. Gladys will automatically migrate your data, but will not delete your data from your SQLite database to prevent data loss.",
+    "duckDbMigrationProgressTitle": "Migration Progress",
+    "duckDbMigrationMigrationDone": "Migration completed:",
+    "duckDbNumberOfStatesinSQlite": "Number of states in SQLite",
+    "duckDbNumberOfStatesinDuckDb": "Number of states in DuckDB",
+    "restartMigrationTitle": "Restart Migration",
+    "confirm": "Confirm",
+    "cancel": "Cancel",
+    "purgeSQliteTitle": "Purge SQLite states",
+    "purgeSQliteDescription": "Once all states are deleted from SQLite, you will need to perform a \"database cleanup\" on the tab to the right to free up disk space. This step may take some time."
   },
   "newArea": {
     "createNewZoneButton": "Create new zone",

front/src/config/i18n/fr.json

@@ -18,7 +18,9 @@
     "fahrenheit": "F",
     "metersPerSec": "m/s",
     "milesPerHour": "m/h",
-    "selectPlaceholder": "Sélectionnez..."
+    "selectPlaceholder": "Sélectionnez...",
+    "yes": "Oui",
+    "no": "Non"
   },
   "color": {
     "aqua": "Aqua",
@@ -229,6 +231,7 @@
       "houseLabel": "Maison",
       "tabletModeDisabled": "Mode tablette désactivé"
     },
+    "duckDbMigrationInProgress": "Votre instance Gladys est en train de migrer sa base de données à DuckDb, un nouveau système de base de données plus performant pour les données time-series. Cette tâche peut prendre un certain temps, et pendant ce temps vos graphiques ne seront pas tous disponibles. Progression de la migration = {{progress}}%",
     "enableFullScreen": "Plein écran",
     "disableFullScreen": "Quitter plein écran",
     "editDashboardTitle": "Éditer le tableau de bord",
@@ -2362,7 +2365,9 @@
       "device-state-purge-single-feature": "Nettoyage des états d'un appareil",
       "device-state-purge": "Nettoyage des vieux états d'appareils",
       "vacuum": "Nettoyage de la base de données",
-      "service-zigbee2mqtt-backup": "Sauvegarde Zigbee2MQTT"
+      "service-zigbee2mqtt-backup": "Sauvegarde Zigbee2MQTT",
+      "migrate-sqlite-to-duckdb": "Migration vers DuckDb",
+      "device-state-purge-all-sqlite-states": "Purge de tous les états SQLite"
     },
     "jobErrors": {
       "purged-when-restarted": "Gladys Assistant a redémarré alors que cette tâche était en cours. Cela ne veut pas dire que cette tâche a échouée, c'est un comportement normal."
@@ -2415,7 +2420,18 @@
       "dead": "Mort"
     },
     "batteryLevel": "Alerte sur le niveau de batterie",
-    "batteryLevelDescription": "Tous les samedis à 9h00, un message sera envoyé à tous les administrateurs si le niveau de batterie d'un appareil passe en dessous du seuil choisi."
+    "batteryLevelDescription": "Tous les samedis à 9h00, un message sera envoyé à tous les administrateurs si le niveau de batterie d'un appareil passe en dessous du seuil choisi.",
+    "duckDbMigrationTitle": "Migration vers DuckDB",
+    "duckDbMigrationDescription": "Nous migrons vers un nouveau système de base de donnée pour les données de capteurs. Gladys va effectuer une migration de vos données automatiquement, mais ne supprimera pas vos données de votre base SQLite afin d'éviter les pertes de données.",
+    "duckDbMigrationProgressTitle": "Avancement de la migration",
+    "duckDbMigrationMigrationDone": "Migration effectuée :",
+    "duckDbNumberOfStatesinSQlite": "Nombre d'états dans SQLite",
+    "duckDbNumberOfStatesinDuckDb": "Nombre d'états dans DuckDB",
+    "restartMigrationTitle": "Relancer la migration",
+    "confirm": "Confirmer",
+    "cancel": "Annuler",
+    "purgeSQliteTitle": "Purger les états SQLite",
+    "purgeSQliteDescription": "Une fois que tous les états seront supprimés de SQLite, vous devrez faire un \"nettoyage de la base de données\" sur l'onglet à droite afin de relâcher l'espace disque. Cette étape peut prendre un certain temps."
   },
   "newArea": {
     "createNewZoneButton": "Créer une zone",

front/src/routes/dashboard/DashboardPage.jsx

@@ -4,6 +4,7 @@ import cx from 'classnames';
 import BoxColumns from './BoxColumns';
 import EmptyState from './EmptyState';
 import SetTabletMode from './SetTabletMode';
+import { JOB_STATUS } from '../../../../server/utils/constants';
 
 import style from './style.css';
 
@@ -85,6 +86,11 @@ const DashboardPage = ({ children, ...props }) => (
                 toggleDefineTabletMode={props.toggleDefineTabletMode}
                 defineTabletModeOpened={props.defineTabletModeOpened}
               />
+              {props.duckDbMigrationJob && props.duckDbMigrationJob.status === JOB_STATUS.IN_PROGRESS && (
+                <div class="alert alert-info">
+                  <Text id="dashboard.duckDbMigrationInProgress" fields={props.duckDbMigrationJob} />
+                </div>
+              )}
               {props.dashboardNotConfigured && <EmptyState dashboardListEmpty={props.dashboardListEmpty} />}
               {!props.dashboardNotConfigured && <BoxColumns homeDashboard={props.currentDashboard} />}
             </div>