Merge pull request #1154 from JGreenlee/use-server-ble-matching

🎆 Only do BLE matching on the phone for unprocessed BLE Scans

jest.config.js

@@ -12,7 +12,7 @@ module.exports = {
     "^.+\\.(ts|tsx|js|jsx)$": "babel-jest"
   },
   transformIgnorePatterns: [
-    "node_modules/(?!((enketo-transformer/dist/enketo-transformer/web)|(jest-)?react-native(-.*)?|@react-native(-community)?)/)",
+    "node_modules/(?!((enketo-transformer/dist/enketo-transformer/web)|(jest-)?react-native(-.*)?|@react-native(-community)?|e-mission-common)/)"
   ],
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
   moduleDirectories: ["node_modules", "src"],

package.cordovabuild.json

@@ -139,6 +139,7 @@
     "cordova-custom-config": "^5.1.1",
     "cordova-plugin-ibeacon": "git+https://github.com/louisg1337/cordova-plugin-ibeacon.git",
     "core-js": "^2.5.7",
+    "e-mission-common": "git+https://github.com/JGreenlee/e-mission-common.git#0.4.4",
     "enketo-core": "^6.1.7",
     "enketo-transformer": "^4.0.0",
     "fast-xml-parser": "^4.2.2",

package.serve.json

@@ -65,6 +65,7 @@
     "chartjs-adapter-luxon": "^1.3.1",
     "chartjs-plugin-annotation": "^3.0.1",
     "core-js": "^2.5.7",
+    "e-mission-common": "git+https://github.com/JGreenlee/e-mission-common.git#0.4.4",
     "enketo-core": "^6.1.7",
     "enketo-transformer": "^4.0.0",
     "fast-xml-parser": "^4.2.2",

www/__tests__/timelineHelper.test.ts

@@ -291,14 +291,15 @@ jest.mock('../js/services/unifiedDataLoader', () => ({
 }));
 
 it('works when there are no unprocessed trips...', async () => {
-  expect(readUnprocessedTrips(-1, -1, {} as any)).resolves.toEqual([]);
+  expect(readUnprocessedTrips(-1, -1, {} as any, {} as any)).resolves.toEqual([]);
 });
 
 it('works when there are one or more unprocessed trips...', async () => {
   const testValueOne = await readUnprocessedTrips(
     mockTLH.fakeStartTsOne,
     mockTLH.fakeEndTsOne,
     {} as any,
+    {} as any,
   );
   expect(testValueOne.length).toEqual(1);
   expect(testValueOne[0]).toEqual(

www/js/diary/LabelTab.tsx

@@ -29,11 +29,7 @@ import { getLabelOptions, labelOptionByValue } from '../survey/multilabel/confir
 import { displayError, displayErrorMsg, logDebug, logWarn } from '../plugin/logger';
 import { useTheme } from 'react-native-paper';
 import { getPipelineRangeTs } from '../services/commHelper';
-import {
-  getNotDeletedCandidates,
-  mapBleScansToTimelineEntries,
-  mapInputsToTimelineEntries,
-} from '../survey/inputMatcher';
+import { getNotDeletedCandidates, mapInputsToTimelineEntries } from '../survey/inputMatcher';
 import { configuredFilters as multilabelConfiguredFilters } from '../survey/multilabel/infinite_scroll_filters';
 import { configuredFilters as enketoConfiguredFilters } from '../survey/enketo/infinite_scroll_filters';
 import LabelTabContext, {
@@ -45,6 +41,7 @@ import LabelTabContext, {
 import { readAllCompositeTrips, readUnprocessedTrips } from './timelineHelper';
 import { LabelOptions, MultilabelKey } from '../types/labelTypes';
 import { CompositeTrip, TimelineEntry, TimestampRange, UserInputEntry } from '../types/diaryTypes';
+import { primarySectionForTrip } from './diaryHelper';
 
 let showPlaces;
 const ONE_DAY = 24 * 60 * 60; // seconds
@@ -63,7 +60,6 @@ const LabelTab = () => {
   const [timelineMap, setTimelineMap] = useState<TimelineMap | null>(null);
   const [timelineLabelMap, setTimelineLabelMap] = useState<TimelineLabelMap | null>(null);
   const [timelineNotesMap, setTimelineNotesMap] = useState<TimelineNotesMap | null>(null);
-  const [timelineBleMap, setTimelineBleMap] = useState<any>(null);
   const [displayedEntries, setDisplayedEntries] = useState<TimelineEntry[] | null>(null);
   const [refreshTime, setRefreshTime] = useState<Date | null>(null);
   const [isLoading, setIsLoading] = useState<string | false>('replace');
@@ -105,15 +101,8 @@ const LabelTab = () => {
         allEntries,
         appConfig,
       );
-
       setTimelineLabelMap(newTimelineLabelMap);
       setTimelineNotesMap(newTimelineNotesMap);
-
-      if (appConfig.vehicle_identities?.length) {
-        const newTimelineBleMap = mapBleScansToTimelineEntries(allEntries, appConfig);
-        setTimelineBleMap(newTimelineBleMap);
-      }
-
       applyFilters(timelineMap, newTimelineLabelMap);
     } catch (e) {
       displayError(e, t('errors.while-updating-timeline'));
@@ -171,7 +160,7 @@ const LabelTab = () => {
         unprocessedNotes = ${JSON.stringify(unprocessedNotes)}`);
       if (appConfig.vehicle_identities?.length) {
         await updateUnprocessedBleScans({
-          start_ts: pipelineRange.start_ts,
+          start_ts: pipelineRange.end_ts,
           end_ts: Date.now() / 1000,
         });
         logDebug(`LabelTab: After updating unprocessedBleScans,
@@ -301,7 +290,12 @@ const LabelTab = () => {
           .reverse()
           .find((trip) => trip.origin_key.includes('trip')) as CompositeTrip;
       }
-      readUnprocessedPromise = readUnprocessedTrips(pipelineRange.end_ts, nowTs, lastProcessedTrip);
+      readUnprocessedPromise = readUnprocessedTrips(
+        pipelineRange.end_ts,
+        nowTs,
+        appConfig,
+        lastProcessedTrip,
+      );
     } else {
       readUnprocessedPromise = Promise.resolve([]);
     }
@@ -336,8 +330,8 @@ const LabelTab = () => {
    * @returns Confirmed mode, which could be a vehicle identity as determined by Bluetooth scans,
    *  or the label option from a user-given 'MODE' label, or undefined if neither exists.
    */
-  const confirmedModeFor = (tlEntry: TimelineEntry) =>
-    timelineBleMap?.[tlEntry._id.$oid] || labelFor(tlEntry, 'MODE');
+  const confirmedModeFor = (tlEntry: CompositeTrip) =>
+    primarySectionForTrip(tlEntry)?.ble_sensed_mode || labelFor(tlEntry, 'MODE');
 
   function addUserInputToEntry(oid: string, userInput: any, inputType: 'label' | 'note') {
     const tlEntry = timelineMap?.get(oid);

www/js/diary/LabelTabContext.ts

@@ -1,5 +1,5 @@
 import { createContext } from 'react';
-import { TimelineEntry, TimestampRange, UserInputEntry } from '../types/diaryTypes';
+import { CompositeTrip, TimelineEntry, TimestampRange, UserInputEntry } from '../types/diaryTypes';
 import { LabelOption, LabelOptions, MultilabelKey } from '../types/labelTypes';
 import { EnketoUserInputEntry } from '../survey/enketo/enketoHelper';
 import { VehicleIdentity } from '../types/appConfigTypes';
@@ -35,7 +35,7 @@ type ContextProps = {
   userInputFor: (tlEntry: TimelineEntry) => UserInputMap | undefined;
   notesFor: (tlEntry: TimelineEntry) => UserInputEntry[] | undefined;
   labelFor: (tlEntry: TimelineEntry, labelType: MultilabelKey) => LabelOption | undefined;
-  confirmedModeFor: (tlEntry: TimelineEntry) => VehicleIdentity | LabelOption | undefined;
+  confirmedModeFor: (tlEntry: CompositeTrip) => VehicleIdentity | LabelOption | undefined;
   addUserInputToEntry: (oid: string, userInput: any, inputType: 'label' | 'note') => void;
   displayedEntries: TimelineEntry[] | null;
   filterInputs: LabelTabFilter[];

www/js/diary/diaryHelper.ts

@@ -192,6 +192,15 @@ export function getFormattedSectionProperties(trip: CompositeTrip, imperialConfi
   }));
 }
 
+/**
+ * @param trip A composite trip object
+ * @return the primary section of the trip, i.e. the section with the greatest distance
+ */
+export function primarySectionForTrip(trip: CompositeTrip) {
+  if (!trip.sections?.length) return undefined;
+  return trip.sections.reduce((prev, curr) => (prev.distance > curr.distance ? prev : curr));
+}
+
 export function getLocalTimeString(dt?: LocalDt) {
   if (!dt) return;
   const dateTime = DateTime.fromObject({

www/js/diary/timelineHelper.ts

@@ -17,12 +17,14 @@ import {
   BluetoothBleData,
   SectionData,
   CompositeTripLocation,
+  SectionSummary,
 } from '../types/diaryTypes';
 import { getLabelInputDetails, getLabelInputs } from '../survey/multilabel/confirmHelper';
 import { LabelOptions } from '../types/labelTypes';
 import { EnketoUserInputEntry, filterByNameAndVersion } from '../survey/enketo/enketoHelper';
 import { AppConfig } from '../types/appConfigTypes';
 import { Point, Feature } from 'geojson';
+import { ble_matching } from 'e-mission-common';
 
 const cachedGeojsons: Map<string, GeoJSONData> = new Map();
 
@@ -306,10 +308,25 @@ const dateTime2localdate = (currtime: DateTime, tz: string) => ({
   second: currtime.second,
 });
 
+/* Compute a section summary, which is really simple for unprocessed trips because they are
+  always assumed to be unimodal.
+/* maybe unify with eaum.get_section_summary on e-mission-server at some point */
+const getSectionSummaryForUnprocessed = (section: SectionData, modeProp): SectionSummary => {
+  const baseMode = section[modeProp] || 'UNKNOWN';
+  return {
+    count: { [baseMode]: 1 },
+    distance: { [baseMode]: section.distance },
+    duration: { [baseMode]: section.duration },
+  };
+};
+
 /**
  * @description Given an array of location points, creates an UnprocessedTrip object.
  */
-function points2UnprocessedTrip(locationPoints: Array<BEMData<FilteredLocation>>): UnprocessedTrip {
+function points2UnprocessedTrip(
+  locationPoints: Array<BEMData<FilteredLocation>>,
+  appConfig: AppConfig,
+): UnprocessedTrip {
   const startPoint = locationPoints[0];
   const endPoint = locationPoints[locationPoints.length - 1];
   const tripAndSectionId = `unprocessed_${startPoint.data.ts}_${endPoint.data.ts}`;
@@ -369,6 +386,12 @@ function points2UnprocessedTrip(locationPoints: Array<BEMData<FilteredLocation>>
     origin_key: 'UNPROCESSED_section',
     sensed_mode: 4, // MotionTypes.UNKNOWN (4)
     sensed_mode_str: 'UNKNOWN',
+    ble_sensed_mode: ble_matching.get_ble_sensed_vehicle_for_section(
+      unprocessedBleScans,
+      baseProps.start_ts,
+      baseProps.end_ts,
+      appConfig,
+    ),
     trip_id: { $oid: tripAndSectionId },
   };
 
@@ -377,6 +400,9 @@ function points2UnprocessedTrip(locationPoints: Array<BEMData<FilteredLocation>>
     ...baseProps,
     _id: { $oid: tripAndSectionId },
     additions: [],
+    ble_sensed_summary: getSectionSummaryForUnprocessed(singleSection, 'ble_sensed_mode'),
+    cleaned_section_summary: getSectionSummaryForUnprocessed(singleSection, 'sensed_mode_str'),
+    inferred_section_summary: getSectionSummaryForUnprocessed(singleSection, 'sensed_mode_str'),
     confidence_threshold: 0,
     expectation: { to_label: true },
     inferred_labels: [],
@@ -395,7 +421,10 @@ const tsEntrySort = (e1: BEMData<FilteredLocation>, e2: BEMData<FilteredLocation
  * @description Given an array of 2 transitions, queries the location data during that time and promises an UnprocessedTrip object.
  * @param trip An array of transitions representing one trip; i.e. [start transition, end transition]
  */
-function tripTransitions2UnprocessedTrip(trip: Array<any>): Promise<UnprocessedTrip | undefined> {
+function tripTransitions2UnprocessedTrip(
+  trip: Array<any>,
+  appConfig: AppConfig,
+): Promise<UnprocessedTrip | undefined> {
   const tripStartTransition = trip[0];
   const tripEndTransition = trip[1];
   const tq = {
@@ -437,7 +466,7 @@ function tripTransitions2UnprocessedTrip(trip: Array<any>): Promise<UnprocessedT
         logDebug(`transitions: start = ${JSON.stringify(tripStartTransition.data)}; 
           end = ${JSON.stringify(tripEndTransition.data)}`);
       }
-      return points2UnprocessedTrip(filteredLocationList);
+      return points2UnprocessedTrip(filteredLocationList, appConfig);
     },
   );
 }
@@ -551,6 +580,7 @@ function linkTrips(trip1, trip2) {
 export function readUnprocessedTrips(
   startTs: number,
   endTs: number,
+  appConfig: AppConfig,
   lastProcessedTrip?: CompositeTrip,
 ) {
   const tq = { key: 'write_ts', startTs, endTs };
@@ -568,7 +598,9 @@ export function readUnprocessedTrips(
         tripsList.forEach((trip) => {
           logDebug(JSON.stringify(trip, null, 2));
         });
-        const tripFillPromises = tripsList.map(tripTransitions2UnprocessedTrip);
+        const tripFillPromises = tripsList.map((t) =>
+          tripTransitions2UnprocessedTrip(t, appConfig),
+        );
         return Promise.all(tripFillPromises).then(
           (rawTripObjs: (UnprocessedTrip | undefined)[]) => {
             // Now we need to link up the trips. linking unprocessed trips

www/js/types/diaryTypes.ts

@@ -4,6 +4,7 @@
 
 import { BaseModeKey, MotionTypeKey } from '../diary/diaryHelper';
 import useDerivedProperties from '../diary/useDerivedProperties';
+import { VehicleIdentity } from './appConfigTypes';
 import { MultilabelKey } from './labelTypes';
 import { BEMData, LocalDt } from './serverData';
 import { FeatureCollection, Feature, Geometry, Point, Position } from 'geojson';
@@ -58,6 +59,8 @@ export type CompositeTripLocation = {
 export type UnprocessedTrip = {
   _id: ObjectId;
   additions: []; // unprocessed trips won't have any matched processed inputs, so this is always empty
+  ble_sensed_summary: SectionSummary;
+  cleaned_section_summary: SectionSummary;
   confidence_threshold: number;
   distance: number;
   duration: number;
@@ -67,6 +70,7 @@ export type UnprocessedTrip = {
   end_ts: number;
   expectation: { to_label: true }; // unprocessed trips are always expected to be labeled
   inferred_labels: []; // unprocessed trips won't have inferred labels
+  inferred_section_summary: SectionSummary;
   key: 'UNPROCESSED_trip';
   locations?: CompositeTripLocation[];
   origin_key: 'UNPROCESSED_trip';
@@ -85,6 +89,7 @@ export type UnprocessedTrip = {
 export type CompositeTrip = {
   _id: ObjectId;
   additions: UserInputEntry[];
+  ble_sensed_summary: SectionSummary;
   cleaned_section_summary: SectionSummary;
   cleaned_trip: ObjectId;
   confidence_threshold: number;
@@ -202,6 +207,7 @@ export type SectionData = {
   key: string;
   origin_key: string;
   trip_id: ObjectId;
+  ble_sensed_mode: VehicleIdentity;
   sensed_mode: number;
   source: string; // e.x., "SmoothedHighConfidenceMotion"
   start_ts: number; // Unix