Merge pull request #85 from datakind/map_automation

Add automatic retrieval of an osm map and simplify user flow

dkroutingtool/Makefile

@@ -81,4 +81,7 @@ olddevelop:
 		dashboard:latest
 
 develop:
-	docker compose up
+	docker compose up &
+
+cleanup:
+	docker compose down

dkroutingtool/local_data/custom_header.yaml

@@ -5,9 +5,9 @@ lat_orig: lat # Latitude of the node, in the most popular typical projection (EP
 long_orig: lon # Longitude of the node, in the most popular typical projection (EPSG:4326)
 name: id # Name of the node, displayed on the map and in solutions
 zone: Zone # Optimization zone, a hard boundary for trips (2 nodes in different zones won't ever be on the same trip, or even considered as part of the same optimization)
+buckets: Demand # Load of a node, a configurable value can fill in unknown loads in config.json. A value is considered unknown if set to 0 because a location with 0 demand should not exist anyway.
 
 #Optional
-buckets: Demand # Load of a node, a configurable value can fill in unknown loads in config.json
 closed: Closed # 0 or 1, depending on whether the node should be considered 
 additional_info: Name # To be displayed on a map
 time_windows: Time Windows # Indicates a range of time where the node can be visited, e.g. 5:30AM-6:05PM

dkroutingtool/src/py/server.py

@@ -1,3 +1,6 @@
+import traceback
+import io
+from contextlib import redirect_stderr
 import main_application
 import fastapi
 import uvicorn
@@ -12,6 +15,8 @@
 
 app = fastapi.FastAPI()
 
+stateful_info = dict()
+
 def find_most_recent_output(session_id):
     most_recent = sorted(glob.glob(f'/WORKING_DATA_DIR/data{session_id}/output_data/*'))[-1]
     return most_recent
@@ -61,10 +66,20 @@ def get_solution(session_id: str=''):
     main_application.args.cloud = False
     main_application.args.manual_mapping_mode = False
     main_application.args.manual_input_path = None
+    #temp_output = io.StringIO()
+    try:
+        main_application.main(user_directory=f'data{session_id}')
+        stateful_info[f'{session_id}_last_output'] = 'Success'
+    except Exception:
+        error = traceback.format_exc()
+        print(error)
+        stateful_info[f'{session_id}_last_output'] = error
 
-    main_application.main(user_directory=f'data{session_id}')
-    return {'message': 'Done'}
+    return {'message': f"{stateful_info[f'{session_id}_last_output']}"}
 
+@app.get('/get_map_info')
+def get_map_info():
+    return {'message': stateful_info.get('bounding_box')}
 
 @app.post('/adjust_solution')
 def get_solution(files: List[UploadFile] = File(...), session_id: str=''):
@@ -96,7 +111,9 @@ def download(session_id: str=''):
 
 
 @app.get('/request_map/')
-def request_map(minlat, minlon, maxlat, maxlon):    
+def request_map(minlat, minlon, maxlat, maxlon):  
+    stateful_info['bounding_box'] = [minlat, minlon, maxlat, maxlon]
+
     request_template = f'''
     [out:xml]
     [bbox:{minlon},{minlat}, {maxlon}, {maxlat}];

dkroutingtool/src/py/ui/dashboard.py

@@ -1,3 +1,4 @@
+import json
 import base64
 import requests
 import datetime
@@ -11,6 +12,10 @@
 from streamlit.runtime import get_instance
 from streamlit.runtime.scriptrunner import get_script_run_ctx
 
+import numpy as np
+import pandas as pd
+import yaml
+
 st.set_page_config(page_title='Container-based Action Routing Tool (CART)', layout="wide")
 
 runtime = get_instance()
@@ -34,15 +39,26 @@ def download_solution(solution_path, map_path):
         solution = solution_txt.read().replace('\n', '  \n')
 
     with open(f'solution_files_{timestamp}/{map_path}', 'r') as map_html:
-        map = map_html.read()
+        solutionmap = map_html.read()
 
-    return solution, map, solution_zip
+    return solution, solutionmap, solution_zip
 
 def request_solution():
     response = requests.get(f'{host_url}/get_solution/?session_id={session_id}')
-    solution, map, solution_zip = download_solution(solution_path='solution.txt', map_path='/maps/route_map.html')
+    message = response.json()['message']
+    if message != "Success":
+        st.error(message.replace('\n', '  \n'))
+        st.error("""Any outputs below are not valid, please inspect the error above.  \n
+                 Guidelines: 
+                 1) Typically an IndexError indicates that a location mentioned in config.json doesn't exist properly in the data, it might be a typo.
+                 2) If it indicates that no solution was found, config.json may be changed to include more time or vehicles
+                 3) If it's a KeyError followed by a vehicle name, make sure config.json has vehicles matching the list of available vehicles at the top of the page.""")
+
+    #with st.expander('See optimization logs'):
+
+    solution, solutionmap, solution_zip = download_solution(solution_path='solution.txt', map_path='/maps/route_map.html')
 
-    return solution, map, solution_zip
+    return solution, solutionmap, solution_zip
 
 def request_map(bounding_box):
     response = requests.get(f'{host_url}/request_map/?minlat={bounding_box[0]}&minlon={bounding_box[1]}&maxlat={bounding_box[2]}&maxlon={bounding_box[3]}')
@@ -52,6 +68,9 @@ def request_map(bounding_box):
 def update_vehicle_or_map():
     response = requests.post(f'{host_url}/update_vehicle_or_map/?session_id={session_id}')
 
+def bound_check(new, old):
+    return old[0] <= new[0] and old[1] <= new[1] and old[2] >= new[2] and old[3] >= new[3] 
+
 def adjust(adjusted_file):
     headers = {
     'accept': 'application/json'
@@ -65,8 +84,8 @@ def adjust(adjusted_file):
     else: 
         message = 'Error, verify the adjusted routes file or raise an issue'
 
-    solution, map, solution_zip = download_solution(solution_path='manual_edits/manual_solution.txt', map_path='maps/trip_data.html')
-    return message, solution, map, solution_zip
+    solution, solutionmap, solution_zip = download_solution(solution_path='manual_edits/manual_solution.txt', map_path='maps/trip_data.html')
+    return message, solution, solutionmap, solution_zip
 
 def upload_data(files_from_streamlit):
     global session_id
@@ -96,61 +115,157 @@ def main():
 
     vehicles_text = st.empty()
     vehicles_text.text('Available vehicle profiles: '+ requests.get(f'{host_url}/available_vehicles').json()['message'])
+    recalculate_map = st.toggle(label='Calculate area to download from OpenStreetMaps automatically based on the locations to visit', value=True)
+    if not recalculate_map:
+        st.write('If required, draw a rectangle over the area you want to use for routing. Download it again only if you updated the OpenStreetMap data. Please select an area as small as possible.')
+        m = folium.Map(location=[-11.9858, -77.019], zoom_start=5)
+        Draw(export=False).add_to(m)
+        map_output = st_folium(m, width=700, height=500)
+        if map_output['last_active_drawing'] is not None:
+            coords = map_output['last_active_drawing']['geometry']['coordinates']
+            lats = [coords[0][i][0] for i in range(5)] # 5 because we expect a rectangle including its center point
+            lons = [coords[0][i][1] for i in range(5)]
+            bounding_box = [min(lats), min(lons), max(lats), max(lons)] # did I invert lats and lons here?
+            area = abs(bounding_box[2] - bounding_box[0]) * abs(bounding_box[3] - bounding_box[1]) 
+            st.write(f"Bounding box: {bounding_box}, area: {round(area,5)} Cartesian square units")
+            if area > 0.05:
+                st.write(f'Please choose a smaller area. We typically allow areas below 0.05')
+            else:
+                map_requested = st.button('Click here to download the area. You do not need to download it again if you try out multiple solutions below')
+                if map_requested:
+                    with st.spinner('Downloading the road network. This may take a few minutes, please wait...'):
+                        request_map(bounding_box)
+                    st.write('Road network ready for routing')
+
+    uploaded_files = st.file_uploader('Upload all required files (config.json, customer_data.xlsx, extra_points.csv, custom_header.yaml). You can refer to the [files here as an example](https://github.com/datakind/dk-routing/tree/main/dkroutingtool/local_data)', accept_multiple_files=True)
+    mandatory_files = ['config.json', 'custom_header.yaml', 'customer_data.xlsx', 'extra_points.csv']
+
+    for uploaded in uploaded_files:
+        uploaded.name = uploaded.name.lower()
+
+    present = []
+    for uploaded in uploaded_files:
+        present.append(uploaded.name)
+    missing_files = set(mandatory_files).difference(set(present))
 
-    st.write('If required, draw a rectangle over the area you want to use for routing. Download it again only if you updated the OpenStreetMap data. Please select an area as small as possible.')
-    m = folium.Map(location=[-11.9858, -77.019], zoom_start=5)
-    Draw(export=False).add_to(m)
-    map_output = st_folium(m, width=700, height=500)
-    if map_output['last_active_drawing'] is not None:
-        coords = map_output['last_active_drawing']['geometry']['coordinates']
-        lats = [coords[0][i][0] for i in range(5)] # 5 because we expect a rectangle including its center point
-        lons = [coords[0][i][1] for i in range(5)]
-        bounding_box = [min(lats), min(lons), max(lats), max(lons)]
-        area = abs(bounding_box[2] - bounding_box[0]) * abs(bounding_box[3] - bounding_box[1]) 
-        st.write(f"Bounding box: {bounding_box}, area: {round(area,5)} square units")
-        if area > 0.04:
-            st.write(f'Please choose a smaller area. We allow areas below 0.04')
-        else:
-            map_requested = st.button('Click here to download the area. You do not need to download it again if you try out multiple solutions below')
-            if map_requested:
-                with st.spinner('Downloading the road network. This may take a few minutes, please wait...'):
-                    request_map(bounding_box)
-                st.write('Road network ready for routing')
+    if len(missing_files) > 0:
+        st.error(f'Missing files: {list(missing_files)}')
 
-    uploaded_files = st.file_uploader('Upload all required files (config, locations, extra points)', accept_multiple_files=True)
-    if len(uploaded_files) > 0:
+    lat_lon_columns = []
+
+    extra_configuration = False
+
+    solution_requested = False
+
+    if len(uploaded_files) > 0 and len(missing_files) == 0:
+
+        # validation steps
+        for uploaded in uploaded_files:
+            if uploaded.name == 'config.json':
+                try:
+                    loaded = json.load(uploaded)
+                    uploaded.seek(0)
+                except json.JSONDecodeError:
+                    st.error('The file config.json is not valid JSON. Please validate the syntax in a text editor')
+            if uploaded.name.endswith('lua') or uploaded.name.endswith('osm.pbf') or uploaded.name.endswith('build_parameters.yml'):
+                extra_configuration = True
+
+            if uploaded.name == 'customer_data.xlsx':
+                    customers = pd.read_excel(uploaded)
+                    uploaded.seek(0)
+            if uploaded.name == 'custom_header.yaml':
+                    headers = yaml.load(uploaded, Loader=yaml.CLoader)
+                    uploaded.seek(0)
+
+            if uploaded.name == 'extra_points.csv':
+                extra = pd.read_csv(uploaded)
+                uploaded.seek(0)
+
+        mandatory_columns = ['lat_orig', 'long_orig', 'name', 'zone', 'buckets']
+        optional_columns = ['closed', 'additional_info', 'time_windows']
+        for column in mandatory_columns:
+            to_check = headers.get(column)
+            if to_check in customers.columns:
+                unknown_values = customers[to_check].isna().sum()
+                if unknown_values > 0:
+                    added = ''
+                    if column == 'buckets':
+                        added = ' If the number of containers is unknown for a particular customer, please enter 0 as the value and the software will assume a default value.'
+                    st.error(f'{to_check} has {unknown_values} invalid value(s) (blank, missing, etc.), please verify customer_data.xlsx before proceeding.{added}')
+            else:
+                st.error(f'{to_check} is missing in customer_data.xlsx and it is a mandatory column, please add it before proceeding or specify the right column in custom_header.yaml.')
+        for column in optional_columns:
+            to_check = headers.get(column)
+            if to_check in customers.columns:
+                unknown_values = customers[to_check].isna().sum()
+                if unknown_values > 0:
+                    st.write(f":grey_question: {to_check} has {unknown_values} invalid value(s) (blank, missing, etc.), please make this is fine for your use case")
+            else:
+                st.write(f":grey_question: {to_check} is not found in customer_data.xlsx, please make sure it is not needed for your use case")
+
+        # automatic area selection
+        if recalculate_map:
+            lat_lon_columns.append(headers['lat_orig'])
+            lat_lon_columns.append(headers['long_orig'])
+            extra_coordinates = extra[['GPS (Latitude)','GPS (Longitude)']]
+
+            all_coords = np.concatenate([customers[lat_lon_columns].values, extra_coordinates.values])
+            area_buffer = 0.01 # adding a buffer for the road network, 0.1 is about 11 km long at the equator
+            minima = all_coords.min(axis=0)-area_buffer
+            maxima = all_coords.max(axis=0)+area_buffer
+            bounding_box = [minima[1], minima[0], maxima[1], maxima[0]] 
+            area = abs(bounding_box[2] - bounding_box[0]) * abs(bounding_box[3] - bounding_box[1]) 
+
         response = upload_data(uploaded_files)
         st.write(response)
-        vehicle_or_map_update_requested = st.button('If you uploaded modified *.lua, build_parameters.yml, or *.osm.pbf files, click here to update the network')
-        if vehicle_or_map_update_requested:
-            with st.spinner('Rebuilding based on updated vehicles/maps. This may take a few minutes, please wait...'):
-                update_vehicle_or_map()
-            vehicles_text.text('Available vehicle profiles: '+ requests.get(f'{host_url}/available_vehicles').json()['message'])
+        if extra_configuration:
+            vehicle_or_map_update_requested = st.button('If you uploaded modified *.lua, build_parameters.yml, or *.osm.pbf files, click here to update the network')
+            if vehicle_or_map_update_requested:
+                with st.spinner('Rebuilding based on updated vehicles/maps. This may take a few minutes, please wait...'):
+                    update_vehicle_or_map()
+                vehicles_text.text('Available vehicle profiles: '+ requests.get(f'{host_url}/available_vehicles').json()['message'])
 
-    st.write('Calculating a solution will take up to twice the amount of time specified by the config file')
-
-    solution_requested = st.button('Click here to calculate routes')
+        if recalculate_map:
+            response = requests.get(f'{host_url}/get_map_info/')
+            old_bounding_box = [0,0,0,0]
+            if response.json()["message"] is not None:                
+                old_bounding_box = tuple(map(np.float64, response.json()['message']))
+
+            if bound_check(tuple(bounding_box), old_bounding_box):
+                st.write(':heavy_check_mark: The currently available map covers the desired area, no need to redownload it unless you edited OSM since the last download')
+            else:
+                st.error(f"It would be recommended to download the area as you have locations in your input data outside the currently downloaded area. The size is {round(area,2)} in Cartesian square units, be mindful that values above 0.2 may lead to the download taking many minutes")
+
+            map_requested_auto = st.button(f'Click here to download the area. You do not need to download it again if you try out multiple scenarios with the same customer_data.xlsx file')
+
+            if map_requested_auto:
+                with st.spinner('Downloading the road network. Please wait...'):
+                    request_map(bounding_box)
+                #st.write(':heavy_check_mark: Road network ready for routing')
+                st.rerun() 
+        st.write('Calculating a solution will take up to twice the amount of time specified by the config file')
+        solution_requested = st.button('Click here to calculate routes')
 
     if solution_requested:
         with st.spinner('Computing routes, please wait...'):
-            solution, map, solution_zip = request_solution()
+            solution, solutionmap, solution_zip = request_solution()
         #this button reloads the page, let's avoid it
         #st.download_button('Download solution files', solution_zip, file_name='solution.zip', 
         #                   mime='application/octet-stream', help='Downloads all the files generated by the tool')
         b64 = base64.b64encode(solution_zip).decode()
         st.markdown(f'<a href="data:application/octet-stream;base64,{b64}" download="solution.zip">Download solution files</a>', unsafe_allow_html=True)
-        components.html(map, height = 800)
+        components.html(solutionmap, height = 800)
         st.write(solution)
 
     st.subheader('Optional route adjustments')
     uploaded_files = st.file_uploader('If adjustments are made in the manual_edits spreadsheet, upload it here to get adjusted solutions', accept_multiple_files=True)
     if len(uploaded_files) > 0:
         with st.spinner('Adjusting routes, please wait...'):
-            response, solution, map, solution_zip = adjust(uploaded_files)
+            response, solution, solutionmap, solution_zip = adjust(uploaded_files)
         st.write(response)
         b64 = base64.b64encode(solution_zip).decode()
         st.markdown(f'<a href="data:application/octet-stream;base64,{b64}" download="solution.zip">Download solution files</a>', unsafe_allow_html=True)
-        components.html(map, height = 800)
+        components.html(solutionmap, height = 800)
         st.write(solution)
 
 if __name__ == '__main__':

dkroutingtool/src/py/ui/requirements.txt

@@ -1,3 +1,5 @@
-streamlit==1.23.1
+streamlit==1.41.0
 streamlit-folium==0.18.0
-folium==0.16.0
+folium==0.16.0
+openpyxl==3.1.5
+pyyaml==6.0.2