Merge pull request #395 from hotosm/feature/galaxy

Feature/galaxy

.github/workflows/config.yml

@@ -0,0 +1,34 @@
+name: Unit Test
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  deploy:
+    runs-on:
+      ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8.13
+
+      - name: Install gdal 
+        run: |
+          sudo apt-add-repository ppa:ubuntugis/ubuntugis-unstable
+          sudo apt-get update
+          sudo apt-get install libgdal-dev osmctools gdal-bin libspatialindex-dev
+         
+      - name: Install Dependencies
+        run: |
+          export CPLUS_INCLUDE_PATH=/usr/include/gdal
+          export C_INCLUDE_PATH=/usr/include/gdal
+          pip install -r requirements-dev.txt
+      - name: Run Tests
+        run: |
+          make test

.gitignore

@@ -83,6 +83,6 @@ celerybeat-schedule
 core/settings/secret.py
 .env
 node_modules
-ui/static/ui/js/bundle.js
+ui/static/ui/js/*
 *.js.map
 secrets

api/views.py

@@ -52,8 +52,11 @@
 renderer_classes = (JSONRenderer, HOTExportApiRenderer)
 
 DIR = os.path.dirname(os.path.abspath(__file__))
-idx = index.Rtree(os.path.join(DIR,'reverse_geocode'))
-
+try:
+    idx = index.Rtree(os.path.join(DIR,'reverse_geocode'))
+except:
+    pass
+    # raise ImportError("Can not read indexes")
 def bbox_to_geom(s):
     try:
         return GEOSGeometry(Polygon.from_bbox(s.split(',')), srid=4326)

core/settings/project.py

@@ -66,6 +66,10 @@
 
 # url to overpass api endpoint
 OVERPASS_API_URL = os.getenv('OVERPASS_API_URL', 'http://overpass-api.de/api/')
+
+#url to galaxy api endpoint
+GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'https://galaxy-api.hotosm.org/v1/raw-data/current-snapshot/')
+
 GENERATE_MWM = os.getenv('GENERATE_MWM','/usr/local/bin/generate_mwm.sh')
 GENERATOR_TOOL = os.getenv('GENERATOR_TOOL','/usr/local/bin/generator_tool')
 PLANET_FILE = os.getenv('PLANET_FILE','')
@@ -214,7 +218,6 @@
 
 MATOMO_URL = os.getenv('MATOMO_URL')
 MATOMO_SITEID = os.getenv('MATOMO_SITEID')
-
 HDX_URL_PREFIX = Configuration.create(
     hdx_site=os.getenv('HDX_SITE', 'demo'),
     hdx_key=os.getenv('HDX_API_KEY'),

doc/setup-development.md

@@ -68,6 +68,7 @@ python manage.py migrate
 ```bash
 cd ui/
 yarn install
+yarn run dist # for production
 yarn start  # will watch for changes and re-compile as necessary
 ```
 
@@ -110,6 +111,9 @@ Most of these environment variables have reasonable default settings.
 * `OSMAND_MAP_CREATOR_DIR` path to directory where OsmAndMapCreator is installed
 * `GARMIN_CONFIG`, `GARMIN_MKGMAP` absolute paths to garmin JARs
 * `OVERPASS_API_URL` url of Overpass api endpoint
+
+* `GALAXY_API_URL` url of Galaxy api endpoint
+
 * `DATABASE_URL`  Database URL. Defaults to `postgres:///exports`
 * `DEBUG`  Whether to enable debug mode. Defaults to `False` (production).
 * `DJANGO_ENV`  Django environment. Set to `development` to enable development tools and email logging to console.

hdx_exports/hdx_export_set.py

@@ -87,13 +87,15 @@ def hdx_note(self, theme):
 
     def datasets(self,is_private,subnational,data_update_frequency,locations,files,public_dir):
         HDX_FORMATS = {
+            'GeoJSON':'geojson',
             'shp': 'SHP',
             'geopackage': 'Geopackage',
             'garmin_img': 'Garmin IMG',
             'kml': 'KML'
         }
 
         HDX_DESCRIPTIONS = {
+            'GeoJSON': 'geojson',
             'shp':'ESRI Shapefile',
             'geopackage':'Geopackage, SQLite compatible',
             'garmin_img':'.IMG for Garmin GPS Devices (All OSM layers for area)',
@@ -134,14 +136,24 @@ def datasets(self,is_private,subnational,data_update_frequency,locations,files,p
 
             resources = []
             for f in files:
-                if 'theme' not in f.extra or f.extra['theme'] == theme.name:
-                    file_name = os.path.basename(f.parts[0]) # only one part: the zip file
-                    resources.append({
-                       'name': file_name, 
-                       'format': HDX_FORMATS[f.output_name],
-                       'description': HDX_DESCRIPTIONS[f.output_name],
-                       'url': os.path.join(public_dir,file_name)
-                    })
+                if isinstance(f, dict): # it is coming from galaxy
+                    if f['theme'] == theme.name:
+                        file_name = f['file_name'] # only one part: the zip file
+                        resources.append({
+                        'name': file_name, 
+                        'format': HDX_FORMATS[f['output_name']],
+                        'description': HDX_DESCRIPTIONS[f['output_name']],
+                        'url': f['download_url']
+                        }) 
+                else: 
+                    if 'theme' not in f.extra or f.extra['theme'] == theme.name:
+                        file_name = os.path.basename(f.parts[0]) # only one part: the zip file
+                        resources.append({
+                        'name': file_name, 
+                        'format': HDX_FORMATS[f.output_name],
+                        'description': HDX_DESCRIPTIONS[f.output_name],
+                        'url': os.path.join(public_dir,file_name)
+                        })
             # stable sort, but put shapefiles first for Geopreview to pick up correctly
             resources.sort(key=lambda x: 0 if x['format'] == 'zipped shapefile' else 1)
             dataset.add_update_resources(resources)

jobs/models.py

@@ -83,7 +83,7 @@ def validate_export_formats(value):
         )
 
     for format_name in value:
-        if format_name not in ['shp','geopackage','garmin_img','kml','mwm','osmand_obf','osm_pbf','osm_xml','bundle','mbtiles','full_pbf']:
+        if format_name not in ['shp','geojson','geopackage','garmin_img','kml','mwm','osmand_obf','osm_pbf','osm_xml','bundle','mbtiles','full_pbf']:
             raise ValidationError(
                 "Bad format name: %(format_name)s",
                 params={'format_name': format_name},

requirements.txt

@@ -25,3 +25,4 @@ boto3~=1.9.238 # needed for cloudwatch
 rasterio~=1.0.25
 osm-export-tool==0.0.25
 rtree==0.9.1
+validators

tasks/models.py

@@ -14,6 +14,9 @@
 from django.contrib.gis.admin import GeoModelAdmin
 from django.utils.safestring import mark_safe
 from django.core.urlresolvers import reverse
+import validators
+import requests
+
 
 class ExportRun(models.Model):
     """
@@ -87,17 +90,37 @@ def duration(self):
     @property
     def download_urls(self):
         def fdownload(fname):
-            try:
-                filesize_bytes = os.path.getsize(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid), fname).encode('utf-8'))
-            except Exception:
-                filesize_bytes = 0
-
-            download_url = os.path.join(settings.EXPORT_MEDIA_ROOT,str(self.run.uid), fname)
-            return {
+            valid=validators.url(fname)
+            if valid==True:
+                download_url = fname
+                absolute_download_url=download_url
+                try :
+                    value = download_url.split('/')
+                    name=value[-1]
+                    split_name=name.split('_')
+                    download_name=f"{split_name[0]}_{split_name[-1]}"  # getting human redable name ignoring unique id
+                    fname=download_name
+                except:
+                    fname=f"""{self.run.job.name}_{self.name}.zip"""
+                try:
+                    with open(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid),f"{name}_size.txt")) as f :
+                        size=f.readline()
+                    filesize_bytes=int(size)
+                except: 
+                    filesize_bytes=0
+
+            else:
+                try:
+                    filesize_bytes = os.path.getsize(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid), fname).encode('utf-8'))
+                except Exception:
+                    filesize_bytes = 0
+                download_url = os.path.join(settings.EXPORT_MEDIA_ROOT,str(self.run.uid), fname)
+                absolute_download_url=settings.HOSTNAME + download_url
+            return { 
                 "filename":fname,
                 "filesize_bytes": filesize_bytes,
                 "download_url":download_url,
-                "absolute_download_url":settings.HOSTNAME + download_url
+                "absolute_download_url":absolute_download_url
             }
         return map(fdownload, self.filenames)