From 779cb94238a250d015c82a1fbf5b70f6854e3f46 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Tue, 2 Apr 2024 15:24:48 +0200 Subject: [PATCH 1/3] Slapper --- src/main/java/telraam/App.java | 2 + .../telraam/logic/lapper/slapper/Slapper.java | 139 ++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 src/main/java/telraam/logic/lapper/slapper/Slapper.java diff --git a/src/main/java/telraam/App.java b/src/main/java/telraam/App.java index 5b1644e..b94b2e8 100644 --- a/src/main/java/telraam/App.java +++ b/src/main/java/telraam/App.java @@ -22,6 +22,7 @@ import telraam.logic.lapper.Lapper; import telraam.logic.lapper.external.ExternalLapper; import telraam.logic.lapper.robust.RobustLapper; +import telraam.logic.lapper.slapper.Slapper; import telraam.logic.positioner.Positioner; import telraam.logic.positioner.simple.SimplePositioner; import telraam.station.Fetcher; @@ -130,6 +131,7 @@ public void run(AppConfiguration configuration, Environment environment) { lappers.add(new ExternalLapper(this.database)); lappers.add(new RobustLapper(this.database)); + lappers.add(new Slapper(this.database)); // Enable lapper APIs for (Lapper lapper : lappers) { diff --git a/src/main/java/telraam/logic/lapper/slapper/Slapper.java b/src/main/java/telraam/logic/lapper/slapper/Slapper.java new file mode 100644 index 0000000..41ed3fc --- /dev/null +++ b/src/main/java/telraam/logic/lapper/slapper/Slapper.java @@ -0,0 +1,139 @@ +package telraam.logic.lapper.slapper; + +import io.dropwizard.jersey.setup.JerseyEnvironment; +import org.jdbi.v3.core.Jdbi; +import telraam.database.daos.LapSourceDAO; +import telraam.database.models.Detection; +import telraam.database.models.LapSource; +import telraam.logic.lapper.Lapper; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +// Lapper that only uses a single sql query +public class Slapper implements Lapper { + private final String SOURCE_NAME = "slapper"; + private final int DEBOUNCE_TIMEOUT = 10; + private final ScheduledExecutorService scheduler; + private boolean debounceScheduled; + private final Logger logger; + private final Jdbi jdbi; + + public Slapper(Jdbi jdbi) { + this.jdbi = jdbi; + this.scheduler = Executors.newScheduledThreadPool(1); + this.logger = Logger.getLogger(Slapper.class.getName()); + this.debounceScheduled = false; + + // Get the lapSourceId, create the source if needed + LapSourceDAO lapSourceDAO = jdbi.onDemand(LapSourceDAO.class); + if (lapSourceDAO.getByName(SOURCE_NAME).isEmpty()) { + lapSourceDAO.insert(new LapSource(SOURCE_NAME)); + } + } + + @Override + public void handle(Detection msg) { + if (!this.debounceScheduled) { + this.debounceScheduled = true; + this.scheduler.schedule(() -> { + try { + this.calculateLaps(); + } catch (Exception e) { + logger.severe(e.getMessage()); + } + this.debounceScheduled = false; + }, DEBOUNCE_TIMEOUT, TimeUnit.SECONDS); + } + } + + private void calculateLaps() { + logger.info("Slapper: Calculating laps..."); + this.jdbi.useHandle(handle -> handle.execute( + """ + WITH switchovers AS ( + SELECT teamid AS team_id, + newbatonid, + timestamp AS start_timestamp, + COALESCE( + LEAD(timestamp) OVER (PARTITION BY teamid ORDER BY timestamp), + timestamp + INTERVAL '6 MONTHS' + ) AS next_baton_switch + FROM batonswitchover + ), + team_detections AS ( + SELECT MIN(station_id) AS station_id, + MAX(rssi) as rssi, + date_trunc('second', timestamp) AS timestamp_seconds, + team_id + FROM detection d + LEFT JOIN switchovers s ON d.baton_id = s.newbatonid + AND d.timestamp BETWEEN s.start_timestamp AND s.next_baton_switch + WHERE station_id NOT BETWEEN 3 AND 5 + AND rssi > -85 + AND team_id IS NOT NULL + GROUP BY date_trunc('second', timestamp), team_id + ), + no_duplicates AS ( + SELECT team_id, + timestamp_seconds, + FIRST_VALUE(timestamp_seconds) OVER ( + PARTITION BY team_id + ORDER BY timestamp_seconds + ) AS first_timestamp_seconds + FROM ( + SELECT *, + LAG(station_id) OVER ( + PARTITION BY team_id + ORDER BY timestamp_seconds + ) AS prev_station_id + FROM team_detections + ) AS previous + WHERE station_id - prev_station_id <= -5 + ), + new_laps AS ( + SELECT team_id, + timestamp_seconds + FROM no_duplicates n_d + WHERE timestamp_seconds > first_timestamp_seconds + ), + cst_source_id AS ( + SELECT COALESCE(id, -1) AS source_id + FROM lap_source + WHERE name ILIKE 'slapper' + ), + deletions AS ( + DELETE FROM lap l + WHERE lap_source_id = (SELECT source_id FROM cst_source_id) + AND NOT EXISTS ( + SELECT 1 + FROM new_laps n_l + WHERE l.team_id = n_l.team_id + AND l.timestamp = n_l.timestamp_seconds + ) + ) + INSERT INTO lap (team_id, timestamp, lap_source_id) + SELECT team_id, + timestamp_seconds, + source_id + FROM new_laps n_l, cst_source_id + WHERE NOT EXISTS ( + SELECT 1 + FROM lap l, cst_source_id + WHERE l.lap_source_id = source_id + AND l.team_id = n_l.team_id + AND l.timestamp = n_l.timestamp_seconds + ) + """ + ) + ); + logger.info("Slapper: Done calculating laps"); + } + + @Override + public void registerAPI(JerseyEnvironment jersey) { + + } +} From 521dc8f9a3e5f1bb28b28c317aa38abf3b714350 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Thu, 4 Apr 2024 11:41:29 +0200 Subject: [PATCH 2/3] slapper query change --- .../telraam/logic/lapper/slapper/Slapper.java | 114 +++++++++--------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/src/main/java/telraam/logic/lapper/slapper/Slapper.java b/src/main/java/telraam/logic/lapper/slapper/Slapper.java index 41ed3fc..17b7f9f 100644 --- a/src/main/java/telraam/logic/lapper/slapper/Slapper.java +++ b/src/main/java/telraam/logic/lapper/slapper/Slapper.java @@ -54,77 +54,77 @@ private void calculateLaps() { this.jdbi.useHandle(handle -> handle.execute( """ WITH switchovers AS ( - SELECT teamid AS team_id, - newbatonid, - timestamp AS start_timestamp, - COALESCE( - LEAD(timestamp) OVER (PARTITION BY teamid ORDER BY timestamp), - timestamp + INTERVAL '6 MONTHS' - ) AS next_baton_switch - FROM batonswitchover + SELECT teamid AS team_id, + newbatonid, + timestamp AS start_timestamp, + COALESCE( + LEAD(timestamp) OVER (PARTITION BY teamid ORDER BY timestamp), + timestamp + INTERVAL '1 MONTH' + ) AS next_baton_switch + FROM batonswitchover ), team_detections AS ( - SELECT MIN(station_id) AS station_id, - MAX(rssi) as rssi, - date_trunc('second', timestamp) AS timestamp_seconds, - team_id - FROM detection d - LEFT JOIN switchovers s ON d.baton_id = s.newbatonid - AND d.timestamp BETWEEN s.start_timestamp AND s.next_baton_switch - WHERE station_id NOT BETWEEN 3 AND 5 - AND rssi > -85 - AND team_id IS NOT NULL - GROUP BY date_trunc('second', timestamp), team_id + SELECT MIN(station_id) AS station_id, + MAX(rssi) as rssi, + date_trunc('second', timestamp) AS timestamp_seconds, + team_id + FROM detection d + LEFT JOIN switchovers s ON d.baton_id = s.newbatonid + AND d.timestamp BETWEEN s.start_timestamp AND s.next_baton_switch + WHERE station_id NOT BETWEEN 3 AND 5 + AND rssi > -84 + AND team_id IS NOT NULL + GROUP BY date_trunc('second', timestamp), team_id ), - no_duplicates AS ( - SELECT team_id, - timestamp_seconds, - FIRST_VALUE(timestamp_seconds) OVER ( - PARTITION BY team_id - ORDER BY timestamp_seconds - ) AS first_timestamp_seconds - FROM ( - SELECT *, - LAG(station_id) OVER ( - PARTITION BY team_id - ORDER BY timestamp_seconds - ) AS prev_station_id - FROM team_detections - ) AS previous - WHERE station_id - prev_station_id <= -5 + start_times AS ( + SELECT DISTINCT ON (team_id) team_id, + timestamp_seconds AS start_seconds + FROM team_detections + WHERE station_id BETWEEN 2 AND 3 + ORDER BY team_id, timestamp_seconds ), new_laps AS ( - SELECT team_id, - timestamp_seconds - FROM no_duplicates n_d - WHERE timestamp_seconds > first_timestamp_seconds + SELECT previous.team_id, + timestamp_seconds + FROM ( + SELECT *, + LAG(station_id) OVER ( + PARTITION BY team_id + ORDER BY timestamp_seconds + ) AS prev_station_id + FROM team_detections + ) AS previous + LEFT JOIN start_times s_t + ON previous.team_id = s_t.team_id + WHERE station_id - prev_station_id < -4 + AND timestamp_seconds > start_seconds ), cst_source_id AS ( - SELECT COALESCE(id, -1) AS source_id - FROM lap_source - WHERE name ILIKE 'slapper' + SELECT COALESCE(id, -1) AS source_id + FROM lap_source + WHERE name ILIKE 'slapper' ), deletions AS ( - DELETE FROM lap l - WHERE lap_source_id = (SELECT source_id FROM cst_source_id) - AND NOT EXISTS ( - SELECT 1 - FROM new_laps n_l - WHERE l.team_id = n_l.team_id - AND l.timestamp = n_l.timestamp_seconds - ) + DELETE FROM lap l + WHERE lap_source_id = (SELECT source_id FROM cst_source_id) + AND NOT EXISTS ( + SELECT 1 + FROM new_laps n_l + WHERE l.team_id = n_l.team_id + AND l.timestamp = n_l.timestamp_seconds + ) ) INSERT INTO lap (team_id, timestamp, lap_source_id) SELECT team_id, - timestamp_seconds, - source_id + timestamp_seconds, + source_id FROM new_laps n_l, cst_source_id WHERE NOT EXISTS ( - SELECT 1 - FROM lap l, cst_source_id - WHERE l.lap_source_id = source_id - AND l.team_id = n_l.team_id - AND l.timestamp = n_l.timestamp_seconds + SELECT 1 + FROM lap l, cst_source_id + WHERE l.lap_source_id = source_id + AND l.team_id = n_l.team_id + AND l.timestamp = n_l.timestamp_seconds ) """ ) From ae537f80530c923f8e6932e996e4ff64e6b1172e Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 8 Apr 2024 17:59:21 +0200 Subject: [PATCH 3/3] remove min station --- .../telraam/logic/lapper/slapper/Slapper.java | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/src/main/java/telraam/logic/lapper/slapper/Slapper.java b/src/main/java/telraam/logic/lapper/slapper/Slapper.java index 17b7f9f..e4a34af 100644 --- a/src/main/java/telraam/logic/lapper/slapper/Slapper.java +++ b/src/main/java/telraam/logic/lapper/slapper/Slapper.java @@ -54,78 +54,78 @@ private void calculateLaps() { this.jdbi.useHandle(handle -> handle.execute( """ WITH switchovers AS ( - SELECT teamid AS team_id, - newbatonid, - timestamp AS start_timestamp, - COALESCE( - LEAD(timestamp) OVER (PARTITION BY teamid ORDER BY timestamp), - timestamp + INTERVAL '1 MONTH' - ) AS next_baton_switch - FROM batonswitchover - ), - team_detections AS ( - SELECT MIN(station_id) AS station_id, - MAX(rssi) as rssi, - date_trunc('second', timestamp) AS timestamp_seconds, - team_id - FROM detection d - LEFT JOIN switchovers s ON d.baton_id = s.newbatonid - AND d.timestamp BETWEEN s.start_timestamp AND s.next_baton_switch - WHERE station_id NOT BETWEEN 3 AND 5 - AND rssi > -84 - AND team_id IS NOT NULL - GROUP BY date_trunc('second', timestamp), team_id - ), - start_times AS ( - SELECT DISTINCT ON (team_id) team_id, - timestamp_seconds AS start_seconds - FROM team_detections - WHERE station_id BETWEEN 2 AND 3 - ORDER BY team_id, timestamp_seconds - ), - new_laps AS ( - SELECT previous.team_id, - timestamp_seconds - FROM ( - SELECT *, - LAG(station_id) OVER ( - PARTITION BY team_id - ORDER BY timestamp_seconds - ) AS prev_station_id - FROM team_detections - ) AS previous - LEFT JOIN start_times s_t - ON previous.team_id = s_t.team_id - WHERE station_id - prev_station_id < -4 - AND timestamp_seconds > start_seconds - ), - cst_source_id AS ( - SELECT COALESCE(id, -1) AS source_id - FROM lap_source - WHERE name ILIKE 'slapper' - ), - deletions AS ( - DELETE FROM lap l - WHERE lap_source_id = (SELECT source_id FROM cst_source_id) - AND NOT EXISTS ( - SELECT 1 - FROM new_laps n_l - WHERE l.team_id = n_l.team_id - AND l.timestamp = n_l.timestamp_seconds - ) - ) - INSERT INTO lap (team_id, timestamp, lap_source_id) - SELECT team_id, - timestamp_seconds, - source_id - FROM new_laps n_l, cst_source_id - WHERE NOT EXISTS ( - SELECT 1 - FROM lap l, cst_source_id - WHERE l.lap_source_id = source_id - AND l.team_id = n_l.team_id - AND l.timestamp = n_l.timestamp_seconds - ) + SELECT teamid AS team_id, + newbatonid, + timestamp AS start_timestamp, + COALESCE( + LEAD(timestamp) OVER (PARTITION BY teamid ORDER BY timestamp), + timestamp + INTERVAL '1 MONTH' + ) AS next_baton_switch + FROM batonswitchover + ), + team_detections AS ( + SELECT station_id, + MAX(rssi) as rssi, + date_trunc('second', timestamp) AS timestamp_seconds, + team_id + FROM detection d + LEFT JOIN switchovers s ON d.baton_id = s.newbatonid + AND d.timestamp BETWEEN s.start_timestamp AND s.next_baton_switch + WHERE station_id NOT BETWEEN 3 AND 5 + AND rssi > -84 + AND team_id IS NOT NULL + GROUP BY date_trunc('second', timestamp), team_id, station_id + ), + start_times AS ( + SELECT DISTINCT ON (team_id) team_id, + timestamp_seconds AS start_seconds + FROM team_detections + WHERE station_id BETWEEN 2 AND 3 + ORDER BY team_id, timestamp_seconds + ), + new_laps AS ( + SELECT previous.team_id, + timestamp_seconds + FROM ( + SELECT *, + LAG(station_id) OVER ( + PARTITION BY team_id + ORDER BY timestamp_seconds + ) AS prev_station_id + FROM team_detections + ) AS previous + LEFT JOIN start_times s_t + ON previous.team_id = s_t.team_id + WHERE station_id - prev_station_id < -4 + AND timestamp_seconds > start_seconds + ), + cst_source_id AS ( + SELECT COALESCE(id, -1) AS source_id + FROM lap_source + WHERE name ILIKE 'slapper' + ), + deletions AS ( + DELETE FROM lap l + WHERE lap_source_id = (SELECT source_id FROM cst_source_id) + AND NOT EXISTS ( + SELECT 1 + FROM new_laps n_l + WHERE l.team_id = n_l.team_id + AND l.timestamp = n_l.timestamp_seconds + ) + ) + INSERT INTO lap (team_id, timestamp, lap_source_id) + SELECT team_id, + timestamp_seconds, + source_id + FROM new_laps n_l, cst_source_id + WHERE NOT EXISTS ( + SELECT 1 + FROM lap l, cst_source_id + WHERE l.lap_source_id = source_id + AND l.team_id = n_l.team_id + AND l.timestamp = n_l.timestamp_seconds + ) """ ) );