WIP: Replace dict usage with protos for workers

mapadroid/data_handler/grpc/MitmMapperClient.py

@@ -126,10 +126,14 @@ def __transform_proto_data_entry(self, entry: mitm_mapper_pb2.LatestMitmDataEntr
         elif entry.HasField(
                 "some_list"):
             data = entry.some_list
+        elif entry.HasField("raw_message"):
+            data = entry.raw_message
         else:
             data = None
-        if data:
+        if data is not None and not isinstance(data, bytes):
             formatted = json_format.MessageToDict(data)
+        elif data is not None:
+            formatted = data
         else:
             formatted = None
         entry: LatestMitmDataEntry = LatestMitmDataEntry(location=location,

mapadroid/data_handler/mitm_data/holder/latest_mitm_data/LatestMitmDataEntry.py

@@ -35,7 +35,8 @@ async def from_json(json_data: Union[bytes, str]) -> Optional[LatestMitmDataEntr
 
         timestamp_received: Optional[int] = loaded.get("timestamp_received")
         timestamp_of_data_retrieval: Optional[int] = loaded.get("timestamp_of_data_retrieval")
-        data: Union[List, Dict] = loaded.get("data")
+        # TODO: Likely data is a str which needs to be translated to bytes?
+        data: Union[List, Dict, bytes] = loaded.get("data")
         obj: LatestMitmDataEntry = LatestMitmDataEntry(location,
                                                        timestamp_received,
                                                        timestamp_of_data_retrieval,

mapadroid/utils/global_variables.py

@@ -16,3 +16,5 @@
 MIN_LEVEL_IV = 30
 # Speed can be 60 km/h up to distances of 3km
 QUEST_WALK_SPEED_CALCULATED = 16.67
+# Distance to consider mons being available for encounters in meters
+DISTANCE_CONSIDER_ENCOUNTERS = 65

mapadroid/worker/strategy/AbstractMitmBaseStrategy.py

@@ -1,6 +1,8 @@
 import asyncio
 import sys
 
+from google.protobuf.internal.containers import RepeatedCompositeFieldContainer
+
 from mapadroid.db.helper.PokemonHelper import PokemonHelper
 
 if sys.version_info.major == 3 and sys.version_info.minor >= 10:
@@ -11,7 +13,7 @@
 import math
 import time
 from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Dict, List, Optional, Set, Tuple, Union, Any
 
 from loguru import logger
 
@@ -35,8 +37,7 @@
 from mapadroid.utils.madConstants import (
     FALLBACK_MITM_WAIT_TIMEOUT, MINIMUM_DISTANCE_ALLOWANCE_FOR_GMO,
     SECONDS_BEFORE_ARRIVAL_OF_WALK_BUFFER, TIMESTAMP_NEVER)
-from mapadroid.utils.madGlobals import (FortSearchResultTypes,
-                                        InternalStopWorkerException,
+from mapadroid.utils.madGlobals import (InternalStopWorkerException,
                                         MadGlobals, PositionType,
                                         TransportType,
                                         WebsocketWorkerRemovedException)
@@ -47,6 +48,7 @@
     AbstractWorkerStrategy
 from mapadroid.worker.WorkerState import WorkerState
 from mapadroid.worker.WorkerType import WorkerType
+import mapadroid.mitm_receiver.protos.Rpc_pb2 as pogoprotos
 
 
 class AbstractMitmBaseStrategy(AbstractWorkerStrategy, ABC):
@@ -79,7 +81,7 @@ async def _clear_game_data(self):
     async def _check_for_data_content(self, latest: Optional[LatestMitmDataEntry],
                                       proto_to_wait_for: ProtoIdentifier,
                                       timestamp: int) \
-            -> Tuple[ReceivedType, Optional[object]]:
+            -> Tuple[ReceivedType, Optional[Any]]:
         """
         Wait_for_data for each worker
         :return:
@@ -140,7 +142,7 @@ async def pre_work_loop(self) -> None:
 
     async def _wait_for_data(self, timestamp: float = None,
                              proto_to_wait_for: ProtoIdentifier = ProtoIdentifier.GMO, timeout=None) \
-            -> Tuple[ReceivedType, Optional[Union[dict, FortSearchResultTypes]], float]:
+            -> Tuple[ReceivedType, Optional[Any], float]:
         key = str(proto_to_wait_for.value)
         if timestamp is None:
             timestamp = time.time()
@@ -158,9 +160,9 @@ async def _wait_for_data(self, timestamp: float = None,
         position_type = await self._mapping_manager.routemanager_get_position_type(self._area_id,
                                                                                    self._worker_state.origin)
         type_of_data_returned = ReceivedType.UNDEFINED
-        data = None
+        data: Optional[Any] = None
         last_time_received = TIMESTAMP_NEVER
-
+        latest: Optional[LatestMitmDataEntry] = None
         data, latest, type_of_data_returned = await self._request_data(data, key, proto_to_wait_for, timestamp,
                                                                        type_of_data_returned)
         if latest:
@@ -207,7 +209,8 @@ async def _wait_for_data(self, timestamp: float = None,
         # await self.worker_stats()
         return type_of_data_returned, data, last_time_received
 
-    async def _request_data(self, data, key, proto_to_wait_for, timestamp, type_of_data_returned):
+    async def _request_data(self, data, key, proto_to_wait_for, timestamp, type_of_data_returned)\
+            -> Tuple[Optional[Any], Optional[LatestMitmDataEntry], ReceivedType]:
         latest_location: Optional[Location] = await self._mitm_mapper.get_last_known_location(
             self._worker_state.origin)
         check_data = True
@@ -452,20 +455,21 @@ async def _wait_for_injection(self):
         return True
 
     @staticmethod
-    def _gmo_cells_contain_multiple_of_key(gmo: dict, keys_in_cell: Union[str, Collection[str]]) -> bool:
-        if not gmo or not keys_in_cell or "cells" not in gmo:
+    def _gmo_cells_contain_multiple_of_key(gmo: pogoprotos.GetMapObjectsOutProto,
+                                           keys_in_cell: Union[str, Collection[str]]) -> bool:
+        if not gmo or not keys_in_cell or not gmo.map_cell:
             return False
         keys: List[str] = []
         if isinstance(keys_in_cell, str):
             keys.append(keys_in_cell)
         else:
             keys.extend(keys_in_cell)
-        cells = gmo.get("cells", [])
-        if not cells or not isinstance(cells, list):
+        cells: RepeatedCompositeFieldContainer[pogoprotos.ClientMapCellProto] = gmo.map_cell
+        if not cells or not isinstance(cells, RepeatedCompositeFieldContainer):
             return False
         for cell in cells:
             for key in keys:
-                value_of_key = cell.get(key, None)
+                value_of_key: Optional[Any] = getattr(cell, key, None)
                 if value_of_key and isinstance(value_of_key, list) and len(value_of_key) > 0:
                     return True
         return False

mapadroid/worker/strategy/plain/AbstractWorkerMitmStrategy.py

@@ -2,8 +2,9 @@
 import math
 import time
 from abc import ABC, abstractmethod
-from typing import Optional, Set, Tuple, Union
+from typing import Optional, Set, Tuple, Union, Any
 
+from google.protobuf.internal.containers import RepeatedCompositeFieldContainer
 from redis import Redis
 
 from mapadroid.data_handler.mitm_data.holder.latest_mitm_data.LatestMitmDataEntry import \
@@ -12,6 +13,7 @@
     MappingManagerDevicemappingKey
 from mapadroid.utils.collections import Location
 from mapadroid.utils.geo import get_distance_of_two_points_in_meters
+from mapadroid.utils.global_variables import DISTANCE_CONSIDER_ENCOUNTERS
 from mapadroid.utils.logging import LoggerEnums, get_logger
 from mapadroid.utils.madGlobals import (FortSearchResultTypes,
                                         InternalStopWorkerException,
@@ -20,6 +22,7 @@
 from mapadroid.worker.ReceivedTypeEnum import ReceivedType
 from mapadroid.worker.strategy.AbstractMitmBaseStrategy import \
     AbstractMitmBaseStrategy
+import mapadroid.mitm_receiver.protos.Rpc_pb2 as pogoprotos
 
 logger = get_logger(LoggerEnums.worker)
 
@@ -97,10 +100,7 @@ async def move_to_location(self):
         self._worker_state.last_location = self._worker_state.current_location
         return timestamp_to_use
 
-    async def post_move_location_routine(self, timestamp) -> Optional[Tuple[ReceivedType,
-                                                                            Optional[Union[dict,
-                                                                                           FortSearchResultTypes]],
-                                                                            float]]:
+    async def post_move_location_routine(self, timestamp) -> Optional[Tuple[ReceivedType, Optional[Any], float]]:
         # TODO: pass the appropriate proto number if IV?
         type_received, data_gmo, time_received = await self._wait_for_data(timestamp)
         if type_received != ReceivedType.GMO or not data_gmo:
@@ -110,28 +110,28 @@ async def post_move_location_routine(self, timestamp) -> Optional[Tuple[Received
             return None
         return type_received, data_gmo, time_received
 
-    async def _gmo_contains_wild_mons_closeby(self, gmo) -> bool:
-        cells = gmo.get("cells", None)
+    async def _gmo_contains_wild_mons_closeby(self, gmo: pogoprotos.GetMapObjectsOutProto) -> bool:
+        cells: RepeatedCompositeFieldContainer[pogoprotos.ClientMapCellProto] = gmo.map_cell
         if not cells:
             return False
         for cell in cells:
-            for wild_mon in cell["wild_pokemon"]:
-                lat = wild_mon["latitude"]
-                lon = wild_mon["longitude"]
-                distance_to_mon: float = get_distance_of_two_points_in_meters(lat, lon,
+            for wild_mon in cell.wild_pokemon:
+                distance_to_mon: float = get_distance_of_two_points_in_meters(wild_mon.latitude, wild_mon.longitude,
                                                                               self._worker_state.current_location.lat,
                                                                               self._worker_state.current_location.lng)
-                # TODO: Distance probably incorrect
-                if distance_to_mon > 70:
+                # TODO: Distance probably incorrect -> Should be read from proto indicating distances allowed
+                if distance_to_mon > DISTANCE_CONSIDER_ENCOUNTERS:
                     logger.debug("Distance to mon around considered to be too far away to await encounter")
                     continue
                 else:
-                    logger.debug2("Mon at {:.5f}, {:.5f} at distance {}", lat, lon, distance_to_mon)
+                    logger.debug2("Mon at {:.5f}, {:.5f} at distance {}",
+                                  wild_mon.latitude, wild_mon.longitude, distance_to_mon)
                     return True
         return False
 
-    async def _gmo_contains_mons_to_be_encountered(self, gmo, check_encounter_id: bool = False) -> bool:
-        cells = gmo.get("cells", None)
+    async def _gmo_contains_mons_to_be_encountered(self, gmo: pogoprotos.GetMapObjectsOutProto,
+                                                   check_encounter_id: bool = False) -> bool:
+        cells: RepeatedCompositeFieldContainer[pogoprotos.ClientMapCellProto] = gmo.map_cell
         if not cells:
             return False
         ids_to_encounter: Set[int] = set()
@@ -146,23 +146,21 @@ async def _gmo_contains_mons_to_be_encountered(self, gmo, check_encounter_id: bo
             ids_to_encounter = {id_to_encounter for id_to_encounter in ids_iv}
 
         for cell in cells:
-            for wild_mon in cell["wild_pokemon"]:
-                spawnid = int(str(wild_mon["spawnpoint_id"]), 16)
-                lat = wild_mon["latitude"]
-                lon = wild_mon["longitude"]
-                distance_to_mon: float = get_distance_of_two_points_in_meters(lat, lon,
+            for wild_mon in cell.wild_pokemon:
+                distance_to_mon: float = get_distance_of_two_points_in_meters(wild_mon.latitude, wild_mon.longitude,
                                                                               self._worker_state.current_location.lat,
                                                                               self._worker_state.current_location.lng)
-                # TODO: Distance probably incorrect
-                if distance_to_mon > 65:
+                # TODO: Distance probably incorrect -> Should be read from proto indicating distances allowed
+                if distance_to_mon > DISTANCE_CONSIDER_ENCOUNTERS:
                     logger.debug("Distance to mon around considered to be too far away to await encounter")
                     continue
                 # If the mon has been encountered before, continue as it cannot be expected to be encountered again
-                encounter_id = wild_mon["encounter_id"]
-                pokemon_data = wild_mon.get("pokemon_data")
-                mon_id = pokemon_data.get("id")
-                pokemon_display = pokemon_data.get("display", {})
-                weather_boosted = pokemon_display.get('weather_boosted_value', None)
+                encounter_id: int = wild_mon.encounter_id
+                pokemon_data: pogoprotos.PokemonProto = wild_mon.pokemon
+                mon_id: int = pokemon_data.id
+                pokemon_display: pogoprotos.PokemonDisplayProto = pokemon_data.pokemon_display
+                # TODO: Ensure this works
+                weather_boosted: int = pokemon_display.weather_boosted_condition
                 if encounter_id < 0:
                     encounter_id = encounter_id + 2 ** 64
                 cache_key = "moniv{}-{}-{}".format(encounter_id, weather_boosted, mon_id)

mapadroid/worker/strategy/plain/WorkerInitStrategy.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Any, Union, Dict
 
 from mapadroid.data_handler.mitm_data.holder.latest_mitm_data.LatestMitmDataEntry import \
     LatestMitmDataEntry
@@ -10,6 +10,7 @@
 from mapadroid.worker.ReceivedTypeEnum import ReceivedType
 from mapadroid.worker.strategy.plain.AbstractWorkerMitmStrategy import \
     AbstractWorkerMitmStrategy
+import mapadroid.mitm_receiver.protos.Rpc_pb2 as pogoprotos
 
 logger = get_logger(LoggerEnums.worker)
 
@@ -20,15 +21,15 @@ class InitTypes(Enum):
 
 
 # Rural areas may not be populated with forts but wild or nearby mons...
-keys_to_check_fort: List[str] = ["forts", "wild_pokemon", "nearby_pokemon"]
+KEY_TO_CHECK_FORTS: List[str] = ["forts", "wild_pokemon", "nearby_pokemon"]
 
 
 class WorkerInitStrategy(AbstractWorkerMitmStrategy):
     async def _check_for_data_content(self, latest: Optional[LatestMitmDataEntry],
                                       proto_to_wait_for: ProtoIdentifier,
-                                      timestamp: int) -> Tuple[ReceivedType, Optional[object]]:
+                                      timestamp: int) -> Tuple[ReceivedType, Optional[Any]]:
         type_of_data_found: ReceivedType = ReceivedType.UNDEFINED
-        data_found: Optional[object] = None
+        data_found: Optional[Any] = None
         if not latest:
             return type_of_data_found, data_found
         # proto has previously been received, let's check the timestamp...
@@ -43,23 +44,26 @@ async def _check_for_data_content(self, latest: Optional[LatestMitmDataEntry],
             # TODO: latter indicates too high speeds for example
             return type_of_data_found, data_found
 
-        latest_proto_data: dict = latest.data
-        if latest_proto_data is None:
+        latest_proto_data: Union[List, Dict, bytes] = latest.data
+        if not latest_proto_data:
             return ReceivedType.UNDEFINED, data_found
-        if proto_to_wait_for == ProtoIdentifier.GMO:
+        elif proto_to_wait_for == ProtoIdentifier.GMO:
+            gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto.ParseFromString(
+                latest_proto_data)
             area_settings: Optional[SettingsAreaInitMitm] = await self._mapping_manager.routemanager_get_settings(
                 self._area_id)
             init_type: InitTypes = InitTypes(area_settings.init_type)
             if ((init_type == InitTypes.MONS
-                 and await self._gmo_contains_wild_mons_closeby(latest_proto_data))
+                 and await self._gmo_contains_wild_mons_closeby(gmo))
                     or (init_type == InitTypes.FORTS
-                        and self._gmo_cells_contain_multiple_of_key(latest_proto_data, keys_to_check_fort))):
-                data_found = latest_proto_data
+                        and self._gmo_cells_contain_multiple_of_key(gmo, KEY_TO_CHECK_FORTS))):
+                data_found = gmo
                 type_of_data_found = ReceivedType.GMO
             else:
                 logger.debug("Data looked for not in GMO")
         elif proto_to_wait_for == ProtoIdentifier.ENCOUNTER:
-            data_found = latest_proto_data
+            data_found: pogoprotos.EncounterOutProto = pogoprotos.EncounterOutProto.ParseFromString(
+                latest_proto_data)
             type_of_data_found = ReceivedType.MON
 
         return type_of_data_found, data_found