feature(graphs): filter data points by date filters

In order to narrow down graphs range, date filters were added (by start
and end date) with ability to quickly switch between 1,3 and 6 recent
months. Last 3 months filter is on by default.

closes: #474

argus/backend/controller/api.py

@@ -1,4 +1,5 @@
 import logging
+from datetime import datetime, timezone
 from uuid import UUID
 import requests
 from flask import (
@@ -386,13 +387,21 @@ def test_info():
 @api_login_required
 def test_results():
     test_id = request.args.get("testId")
+    start_date_str = request.args.get("startDate")
+    end_date_str = request.args.get("endDate")
+
     if not test_id:
         raise Exception("No testId provided")
+
+    start_date = datetime.fromisoformat(start_date_str).astimezone(timezone.utc) if start_date_str else None
+    end_date = datetime.fromisoformat(end_date_str).astimezone(timezone.utc) if end_date_str else None
+
     service = ResultsService()
     if request.method == 'HEAD':
         exists = service.is_results_exist(test_id=UUID(test_id))
         return Response(status=200 if exists else 404)
-    graphs, ticks, releases_filters = service.get_test_graphs(test_id=UUID(test_id))
+
+    graphs, ticks, releases_filters = service.get_test_graphs(test_id=UUID(test_id), start_date=start_date, end_date=end_date)
 
     return {
         "status": "ok",

argus/backend/service/results_service.py

@@ -295,9 +295,12 @@ def calculate_graph_ticks(graphs: List[Dict]) -> dict[str, str]:
                 min_x = first_x
             if max_x is None or last_x > max_x:
                 max_x = last_x
+    if not max_x or not min_x:
+        return {}  # no data
     return {"min": min_x[:10], "max": max_x[:10]}
 
 
+
 def _identify_most_changed_package(packages_list: list[PackageVersion]) -> str:
     version_date_changes: dict[str, set[tuple[str, str]]] = defaultdict(set)
 
@@ -365,12 +368,25 @@ def _get_tables_metadata(self, test_id: UUID) -> list[ArgusGenericResultMetadata
         tables_meta = self.cluster.session.execute(query=query, parameters=(test_id,))
         return [ArgusGenericResultMetadata(**table) for table in tables_meta]
 
-    def _get_tables_data(self, test_id: UUID, table_name: str, ignored_runs: list[RunId]) -> list[ArgusGenericResultData]:
+    def _get_tables_data(self, test_id: UUID, table_name: str, ignored_runs: list[RunId],
+                         start_date: datetime | None = None, end_date: datetime | None = None) -> list[ArgusGenericResultData]:
         query_fields = ["run_id", "column", "row", "value", "status", "sut_timestamp"]
         raw_query = (f"SELECT {','.join(query_fields)}"
-                     f" FROM generic_result_data_v1 WHERE test_id = ? AND name = ? LIMIT 2147483647")
+                     f" FROM generic_result_data_v1 WHERE test_id = ? AND name = ?")
+
+        parameters = [test_id, table_name]
+
+        if start_date:
+            raw_query += " AND sut_timestamp >= ?"
+            parameters.append(start_date)
+        if end_date:
+            raw_query += " AND sut_timestamp <= ?"
+            parameters.append(end_date)
+
+        if start_date or end_date:
+            raw_query += " ALLOW FILTERING"
         query = self.cluster.prepare(raw_query)
-        data = self.cluster.session.execute(query=query, parameters=(test_id, table_name))
+        data = self.cluster.session.execute(query=query, parameters=tuple(parameters))
         return [ArgusGenericResultData(**cell) for cell in data if cell["run_id"] not in ignored_runs]
 
     def get_table_metadata(self, test_id: UUID, table_name: str) -> ArgusGenericResultMetadata:
@@ -401,13 +417,14 @@ def get_run_results(self, test_id: UUID, run_id: UUID) -> list[dict]:
                 'order': min([cell['ordering'] for cell in cells] or [0])})
         return sorted(tables, key=lambda x: x['order'])
 
-    def get_test_graphs(self, test_id: UUID):
+    def get_test_graphs(self, test_id: UUID, start_date: datetime | None = None, end_date: datetime | None = None):
         runs_details = self._get_runs_details(test_id)
         tables_meta = self._get_tables_metadata(test_id=test_id)
         graphs = []
         releases_filters = set()
         for table in tables_meta:
-            data = self._get_tables_data(test_id=test_id, table_name=table.name, ignored_runs=runs_details.ignored)
+            data = self._get_tables_data(test_id=test_id, table_name=table.name, ignored_runs=runs_details.ignored,
+                                         start_date=start_date, end_date=end_date)
             if not data:
                 continue
             best_results = self.get_best_results(test_id=test_id, name=table.name)

argus/backend/tests/results_service/test_chartjs_additional_functions.py

@@ -197,7 +197,7 @@ def test_calculate_limits():
         assert 'limit' in point
 
 
-def test_calculate_graph_ticks():
+def test_calculate_graph_ticks_with_data_returns_min_max_ticks():
     graphs = [
         {
             "data": {
@@ -217,3 +217,23 @@ def test_calculate_graph_ticks():
     ticks = calculate_graph_ticks(graphs)
     assert ticks["min"] == "2023-10-22"
     assert ticks["max"] == "2023-10-25"
+
+def test_calculate_graph_ticks_without_data_does_not_fail():
+    graphs = [
+        {
+            "data": {
+                "datasets": [
+                    {"data": []}
+                ]
+            }
+        },
+        {
+            "data": {
+                "datasets": [
+                    {"data": []}
+                ]
+            }
+        }
+    ]
+    ticks = calculate_graph_ticks(graphs)
+    assert ticks == {}

argus/backend/tests/results_service/test_results_service.py

@@ -0,0 +1,100 @@
+from datetime import datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from argus.backend.models.result import ArgusGenericResultMetadata, ArgusGenericResultData, ColumnMetadata
+from argus.backend.service.results_service import ResultsService
+
+@pytest.fixture
+def setup_data(argus_db):
+    test_id = uuid4()
+    table = ArgusGenericResultMetadata(
+        test_id=test_id,
+        name='Test Table',
+        columns_meta=[
+            ColumnMetadata(name='col1', unit='ms', type='FLOAT', higher_is_better=False)
+        ],
+        rows_meta=['row1'],
+        validation_rules={}
+    )
+    data = [
+        ArgusGenericResultData(
+            test_id=test_id,
+            name=table.name,
+            run_id=uuid4(),
+            column='col1',
+            row='row1',
+            sut_timestamp=datetime.today() - timedelta(days=10),
+            value=100.0,
+            status='UNSET'
+        ).save(),
+        ArgusGenericResultData(
+            test_id=test_id,
+            name=table.name,
+            run_id=uuid4(),
+            column='col1',
+            row='row1',
+            sut_timestamp=datetime.today() - timedelta(days=5),
+            value=150.0,
+            status='UNSET'
+        ).save(),
+        ArgusGenericResultData(
+            test_id=test_id,
+            name=table.name,
+            run_id=uuid4(),
+            column='col1',
+            row='row1',
+            sut_timestamp=datetime.today() - timedelta(days=1),
+            value=200.0,
+            status='UNSET'
+        ).save()
+    ]
+    return test_id, table, data
+
+
+def test_results_service_should_return_results_within_date_range(setup_data):
+    test_id, table, data = setup_data
+    service = ResultsService()
+
+    start_date = datetime.today() - timedelta(days=7)
+    end_date = datetime.today() - timedelta(days=2)
+
+    filtered_data = service._get_tables_data(
+        test_id=test_id,
+        table_name=table.name,
+        ignored_runs=[],
+        start_date=start_date,
+        end_date=end_date
+    )
+
+    assert len(filtered_data) == 1
+    assert filtered_data[0].value == 150.0
+
+def test_results_service_should_return_no_results_outside_date_range(setup_data):
+    test_id, table, data = setup_data
+    service = ResultsService()
+
+    start_date = datetime.today() - timedelta(days=20)
+    end_date = datetime.today() - timedelta(days=15)
+
+    filtered_data = service._get_tables_data(
+        test_id=test_id,
+        table_name=table.name,
+        ignored_runs=[],
+        start_date=start_date,
+        end_date=end_date
+    )
+
+    assert len(filtered_data) == 0
+
+def test_results_service_should_return_all_results_with_no_date_range(setup_data):
+    test_id, table, data = setup_data
+    service = ResultsService()
+
+    filtered_data = service._get_tables_data(
+        test_id=test_id,
+        table_name=table.name,
+        ignored_runs=[]
+    )
+
+    assert len(filtered_data) == 3

frontend/TestRun/ResultsGraphs.svelte

@@ -2,6 +2,8 @@
     import {createEventDispatcher, onMount} from "svelte";
     import {sendMessage} from "../Stores/AlertStore";
     import ResultsGraph from "./ResultsGraph.svelte";
+    import dayjs from "dayjs";
+    import queryString from "query-string";
 
     export let test_id = "";
     let graphs = [];
@@ -12,17 +14,20 @@
     let selectedTableFilters = [];
     let selectedColumnFilters = [];
     let filteredGraphs = [];
+    let startDate = "";
+    let endDate = "";
     let width = 500;  // default width for each chart
     let height = 300;  // default height for each chart
     const dispatch = createEventDispatcher();
 
     const dispatch_run_click = (e) => {
-        dispatch("runClick", {runId: e.detail.runId});
+        dispatch("runClick", { runId: e.detail.runId });
     };
 
     const fetchTestResults = async function (testId) {
         try {
-            let res = await fetch(`/api/v1/test-results?testId=${testId}`);
+            const params = queryString.stringify({testId, startDate, endDate});
+            let res = await fetch(`/api/v1/test-results?${params}`);
             if (res.status != 200) {
                 return Promise.reject(`HTTP Error ${res.status} trying to fetch test results`);
             }
@@ -59,6 +64,21 @@
         return Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
     };
 
+    const setDefaultDateRange = () => {
+        const now = dayjs();
+        endDate = now.format('YYYY-MM-DD');
+        const pastDate = now.subtract(3, 'month');
+        startDate = pastDate.format('YYYY-MM-DD');
+    };
+
+    const setDateRange = (months) => {
+        const now = dayjs();
+        endDate = now.format('YYYY-MM-DD');
+        const pastDate = now.subtract(months, 'month');
+        startDate = pastDate.format('YYYY-MM-DD');
+        fetchTestResults(test_id);
+    };
+
     const extractTableFilters = () => {
         let fltrs = new Map();
         graphs.forEach(graph => {
@@ -147,10 +167,19 @@
     };
 
     onMount(() => {
+        setDefaultDateRange();
         fetchTestResults(test_id);
     });
 </script>
-
+<div class="filters-container">
+    <div class="input-group input-group-sm mx-2">
+        <input type="date" class="form-control date-input" bind:value={startDate} on:change={() => fetchTestResults(test_id)} />
+        <input type="date" class="form-control date-input" bind:value={endDate} on:change={() => fetchTestResults(test_id)} />
+        <button class="btn btn-info btn-sm" on:click={() => setDateRange(1)}>Last Month</button>
+        <button class="btn btn-info btn-sm" on:click={() => setDateRange(3)}>Last 3 Months</button>
+        <button class="btn btn-info btn-sm" on:click={() => setDateRange(6)}>Last 6 Months</button>
+    </div>
+</div>
 <div class="filters-container">
     {#each tableFilters as filterGroup}
         {#each filterGroup.items as filter}
@@ -245,4 +274,8 @@
     .filters-container button.selected {
         border: 2px solid #333;
     }
+
+    .date-input {
+        max-width: 200px;
+    }
 </style>