Feature/bicycle stand importer fix name and more

mobility_data/importers/bicycle_stands.py

@@ -56,7 +56,6 @@ class BicyleStand(MobileUnitDataBase):
 
  def __init__(self):
  super().__init__()
- self.prefix_name = {}
  self.related_unit = None
  self.extra = {f: None for f in self.EXTRA_FIELDS}
 
@@ -110,7 +109,7 @@ def set_geojson_feature(self, feature):
  else:
  self.name["sv"] = name
  self.name["en"] = name
- self.prefix_name = {k: f"{NAME_PREFIX[k]} {v}" for k, v in self.name.items()}
+ self.name = {k: f"{NAME_PREFIX[k]} {v}" for k, v in self.name.items()}
  addr = feature["osoite"].as_string().split(",")
  # Some addresses are in format:"Pyhän Henrikin aukio, Kupittaankatu 8, 20520 Turku"
  # Then remove the prefix part.
@@ -148,16 +147,18 @@ def set_gml_feature(self, feature):
  self.extra["model"] = model_elem.as_string()
  else:
  self.extra["model"] = None
- num_stands_elem = feature["Lukumaara"]
- if num_stands_elem is not None:
- num = num_stands_elem.as_int()
- # for bicycle stands that are Not maintained by Turku
- # the number of stands is set to 0 in the input data
- # but in reality there is no data so None is set.
- if num == 0 and not self.extra["maintained_by_turku"]:
- self.extra["number_of_stands"] = None
- else:
- self.extra["number_of_stands"] = num
+
+ # For some reason the field "lukumaara" has been removed
+ # num_stands_elem = feature["lukumaara"]
+ # if num_stands_elem is not None:
+ # num = num_stands_elem.as_int()
+ # # for bicycle stands that are Not maintained by Turku
+ # # the number of stands is set to 0 in the input data
+ # # but in reality there is no data so None is set.
+ # if num == 0 and not self.extra["maintained_by_turku"]:
+ # self.extra["number_of_stands"] = None
+ # else:
+ # self.extra["number_of_stands"] = num
 
  num_places_elem = feature["Pyorapaikkojen_lukumaara"].as_string()
  if num_places_elem:
@@ -186,8 +187,8 @@ def set_gml_feature(self, feature):
  full_names = get_closest_address_full_name(self.geometry)
  self.name[FI_KEY] = full_names[FI_KEY]
  self.name[SV_KEY] = full_names[SV_KEY]
- self.name[EN_KEY] = full_names[EN_KEY]
- self.prefix_name = {k: f"{NAME_PREFIX[k]} {v}" for k, v in self.name.items()}
+ self.name[EN_KEY] = full_names[FI_KEY]
+ self.name = {k: f"{NAME_PREFIX[k]} {v}" for k, v in self.name.items()}
 
 
 def get_data_sources():
@@ -219,7 +220,12 @@ def get_bicycle_stand_objects():
  for feature in data_source[1][0]:
  bicycle_stand = BicyleStand()
  if data_source[0] == "gml":
- bicycle_stand.set_gml_feature(feature)
+ try:
+ bicycle_stand.set_gml_feature(feature)
+ except IndexError as err:
+ # Handle IndexError gracefully, as the field names can change in the source data
+ # and it would prevent importing the GEOJSON data.
+ logger.warning(f"Discarding {feature}, reason: {err}")
  elif data_source[0] == "geojson":
  bicycle_stand.set_geojson_feature(feature)
  if (

mobility_data/importers/utils.py

@@ -5,6 +5,7 @@
 import tempfile
 import zipfile
 from enum import Enum
+from functools import lru_cache
 
 import requests
 import yaml
@@ -171,6 +172,7 @@ def get_postal_code(point):
  return postal_code_area.postal_code
 
 
+@lru_cache()
 def get_street_name_translations(name, municipality):
  """
  Returns a dict where the key is the language and the value is
@@ -197,6 +199,7 @@ def get_street_name_translations(name, municipality):
  return names
 
 
+@lru_cache()
 def get_municipality_name(point):
  """
  Returns the string name of the municipality in which the point

mobility_data/tests/test_import_bicycle_stands.py

@@ -47,11 +47,15 @@ def test_geojson_import(
  assert num_created == 3
  assert num_deleted == 0
  assert MobileUnit.objects.all().count() == 3
- kupittaan_maauimala = MobileUnit.objects.get(name="Kupittaan maauimala")
+ kupittaan_maauimala = MobileUnit.objects.get(
+ name="Pyöräpysäköinti Kupittaan maauimala"
+ )
  assert kupittaan_maauimala
- kupittaan_palloiluhalli = MobileUnit.objects.get(name="Kupittaan palloiluhalli")
+ kupittaan_palloiluhalli = MobileUnit.objects.get(
+ name="Pyöräpysäköinti Kupittaan palloiluhalli"
+ )
  assert kupittaan_palloiluhalli
- turun_amk = MobileUnit.objects.get(name="Turun AMK")
+ turun_amk = MobileUnit.objects.get(name="Pyöräpysäköinti Turun AMK")
  assert turun_amk
  assert kupittaan_maauimala.extra["hull_lockable"] is False
  assert kupittaan_maauimala.extra["covered"] is False
@@ -100,28 +104,29 @@ def test_gml_importer(
  stand_covered_hull_lockable = MobileUnit.objects.all()[1]
  # <GIS:Id>319490982</GIS:Id> in fixture xml
  stand_external = MobileUnit.objects.all()[2]
- assert stand_normal.name_fi == "Linnanpuisto"
- assert stand_normal.name_sv == "Slottsparken"
+ assert stand_normal.name_fi == "Pyöräpysäköinti Linnanpuisto"
+ assert stand_normal.name_sv == "Cykelparkering Slottsparken"
+ assert stand_normal.name_en == "Bicycle parking Linnanpuisto"
  assert stand_normal.municipality.name == "Turku"
  extra = stand_normal.extra
  assert extra["model"] == "Normaali"
  assert extra["maintained_by_turku"] is True
  assert extra["covered"] is False
  assert extra["hull_lockable"] is False
  assert extra["number_of_places"] == 24
- assert extra["number_of_stands"] == 2
- assert stand_covered_hull_lockable.name == "Pitkäpellonkatu 7"
- assert stand_covered_hull_lockable.name_sv == "Långåkersgatan 7"
+ assert extra["number_of_stands"] is None
+ assert stand_covered_hull_lockable.name == "Pyöräpysäköinti Pitkäpellonkatu 7"
+ assert stand_covered_hull_lockable.name_sv == "Cykelparkering Långåkersgatan 7"
  extra = stand_covered_hull_lockable.extra
  assert extra["maintained_by_turku"] is True
  assert extra["covered"] is True
  assert extra["hull_lockable"] is True
  assert extra["number_of_places"] == 18
- assert extra["number_of_stands"] == 1
+ assert extra["number_of_stands"] is None
  # external stand has no street name, so the closest street name
  # and address number is assigned as name and that is "Kupittaankatu 8".
- assert stand_external.name == "Kupittaankatu 8"
- assert stand_external.name_sv == "Kuppisgatan 8"
+ assert stand_external.name == "Pyöräpysäköinti Kupittaankatu 8"
+ assert stand_external.name_sv == "Cykelparkering Kuppisgatan 8"
  extra = stand_external.extra
  assert extra["maintained_by_turku"] is False
  # As there are no info for stand that are not maintained by turku