diff --git a/.gitignore b/.gitignore index 4c43438..a88987d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,156 @@ bower_components/ node_modules/ + +# Retrieved from https://github.com/github/gitignore/blob/752b7678db497edb377fa982ded0c70035393794/Python.gitignore +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ dist/ -*.pyc +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/README.md b/README.md index 6895463..1040f28 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # GeoDropdown GeoDropdown makes use of geowebservice to create a multiple drodpdown lists system, which guarantees that users will provides well formatted locations, according to ISO-3166. -Live demo: [demo.html](demo.html) - # Hierarchical levels There are 7 hierarchical levels, which corresponds to 7 dropdown list: @@ -73,17 +71,34 @@ Portugal (wich is a country) geonameid is 2264397, so you can use the link /geod [{"geonameid": 2262961, "fcode": "ADM1", "name": "Distrito de Set\u00c3\u00babal", "adm1": 19}, {"geonameid": 2263478, "fcode": "ADM1", "name": "Distrito de Santar\u00c3\u00a9m", "adm1": 18}, {"geonameid": 2264507, "fcode": "ADM1", "name": "Distrito de Portalegre", "adm1": 16}, {"geonameid": 2267056, "fcode": "ADM1", "name": "Distrito de Lisboa", "adm1": 14}, {"geonameid": 2267094, "fcode": "ADM1", "name": "Distrito de Leiria", "adm1": 13}, {"geonameid": 2268337, "fcode": "ADM1", "name": "Distrito de Faro", "adm1": 9}, {"geonameid": 2268404, "fcode": "ADM1", "name": "Distrito de \u00c3\u2030vora", "adm1": 8}, {"geonameid": 2269513, "fcode": "ADM1", "name": "Distrito de Castelo Branco", "adm1": 6}, {"geonameid": 2270984, "fcode": "ADM1", "name": "Distrito de Beja", "adm1": 3}, {"geonameid": 2593105, "fcode": "ADM1", "name": "Madeira", "adm1": 10}, {"geonameid": 2732264, "fcode": "ADM1", "name": "Distrito de Viseu", "adm1": 22}, {"geonameid": 2732437, "fcode": "ADM1", "name": "Distrito de Vila Real", "adm1": 21}, {"geonameid": 2732772, "fcode": "ADM1", "name": "Distrito de Viana do Castelo", "adm1": 20}, {"geonameid": 2735941, "fcode": "ADM1", "name": "Distrito do Porto", "adm1": 17}, {"geonameid": 2738782, "fcode": "ADM1", "name": "Distrito da Guarda", "adm1": 11}, {"geonameid": 2740636, "fcode": "ADM1", "name": "Distrito de Coimbra", "adm1": 7}, {"geonameid": 2742026, "fcode": "ADM1", "name": "Distrito de Bragan\u00c3\u00a7a", "adm1": 5}, {"geonameid": 2742031, "fcode": "ADM1", "name": "Distrito de Braga", "adm1": 4}, {"geonameid": 2742610, "fcode": "ADM1", "name": "Distrito de Aveiro", "adm1": 2}] ``` -# Server Side: -# How to make it run -There is a docker file to run this application. Therefore, if you want to run geodropdown, just go to geowebservice folder and run the following commands: +# Setup demo +## Front end: +```sh +npm install -g grunt bower serve + +# download external dependencies +bower install + +# build dist files +grunt + +# start a dev server to test demo.html +serve +``` + +## Server Side: +### How to make it run +There is a docker-compose file to run this application. Therefore, if you want to run geodropdown, just go to geowebservice folder and run the following commands: ``` -sh build.sh -docker-compose up +cd geowebservice +docker-compose build +./start.sh ``` -Now, you need to have data related to all locations around the world +### How to load data to the Web Service databases + +**Note:** Data is automatically loaded with the start.sh script. You can ignore this section. -# How to load data to the Web Service databases To load all the required data, you will need two files from geonames: allCountries.txt and countryInfo.txt. You can find them here: [files link](http://download.geonames.org/export/dump/) @@ -91,6 +106,9 @@ The next step is to give the location of those files to the webservice. So you n Now you are prepared to load all locations, simply execute python services.py in console. +## Test it +Go to [http://localhost:3000/demo](http://localhost:3000/demo) + # Authors: - Leonardo Coelho - @@ -98,6 +116,7 @@ Now you are prepared to load all locations, simply execute python services.py in # Contributors - Renato Pinho - Luis A. Bastião Silva - +- André Pedrosa - # Mantainers diff --git a/geowebservice/.dockerignore b/geowebservice/.dockerignore new file mode 100644 index 0000000..bb2655f --- /dev/null +++ b/geowebservice/.dockerignore @@ -0,0 +1,4 @@ +Dockerfile +docker-compose.yml +venv +db.sqlite3 diff --git a/geowebservice/Dockerfile b/geowebservice/Dockerfile index b1a41a3..06de7a0 100644 --- a/geowebservice/Dockerfile +++ b/geowebservice/Dockerfile @@ -1,20 +1,12 @@ -FROM debian:jessie -MAINTAINER Matt Bentley +FROM python:3.10 -RUN (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git python python-dev python-setuptools nginx sqlite3 supervisor) -RUN (easy_install pip &&\ - pip install uwsgi) +WORKDIR /app -ADD requirements.txt /opt/django/app/requirements.txt -RUN pip install -r /opt/django/app/requirements.txt -ADD . /opt/django/ +COPY requirements.txt . +RUN pip install -r requirements.txt -RUN (echo "daemon off;" >> /etc/nginx/nginx.conf &&\ - rm /etc/nginx/sites-enabled/default &&\ - ln -s /opt/django/django.conf /etc/nginx/sites-enabled/ &&\ - ln -s /opt/django/supervisord.conf /etc/supervisor/conf.d/) +COPY . . -RUN chmod +x /opt/django/run.sh -VOLUME ["/opt/django/app"] -EXPOSE 80 -CMD ["/opt/django/run.sh"] \ No newline at end of file +EXPOSE 8000 + +ENTRYPOINT ["gunicorn", "-b 0.0.0.0:8000", "geowebservice.wsgi"] diff --git a/geowebservice/all.json b/geowebservice/all.json deleted file mode 100644 index e69de29..0000000 diff --git a/geowebservice/build.sh b/geowebservice/build.sh deleted file mode 100644 index 0b1e8e3..0000000 --- a/geowebservice/build.sh +++ /dev/null @@ -1 +0,0 @@ - docker build --rm=true -t bioinformaticsua/geodropdownservice . \ No newline at end of file diff --git a/geowebservice/django.conf b/geowebservice/django.conf deleted file mode 100644 index 1cfbc12..0000000 --- a/geowebservice/django.conf +++ /dev/null @@ -1,14 +0,0 @@ -upstream django { - server unix:/opt/django/app.sock; - } - -server { - listen 80 default_server; - charset utf-8; - client_max_body_size 75M; - - location / { - uwsgi_pass django; - include /opt/django/uwsgi_params; # the uwsgi_params file you installed - } -} diff --git a/geowebservice/docker-compose.yml b/geowebservice/docker-compose.yml index f3e5885..229a65b 100644 --- a/geowebservice/docker-compose.yml +++ b/geowebservice/docker-compose.yml @@ -1,17 +1,18 @@ -solr: - image: solr:5.5.3 - hostname: solr - ports: - - "8887:8983" - +version: "3" +services: + solr: + image: solr:5.5.3 + volumes: + - geodata:/opt/solr/server/solr -geodropdownservice: - image: bioinformaticsua/geodropdownservice + geodropdownservice: + build: + context: . ports: - - "8886:80" - volumes: - - .:/opt/django/app - links: - - solr:solr - + - "8886:8000" + depends_on: + - solr + +volumes: + geodata: diff --git a/geowebservice/geodatabase/admin.py b/geowebservice/geodatabase/admin.py deleted file mode 100644 index 6f9e412..0000000 --- a/geowebservice/geodatabase/admin.py +++ /dev/null @@ -1,11 +0,0 @@ -from geodatabase.models import Geoname,Countryinfo -from django.contrib import admin - -class GeonameAdmin(admin.ModelAdmin): - list_display = ('geonameid', 'asciiname', 'fcode', 'country', 'admin1', 'admin2', 'admin3', 'admin4') - -class CountryinfoAdmin(admin.ModelAdmin): - list_display = ('name', 'continent', 'geonameId') - -admin.site.register(Geoname,GeonameAdmin) -admin.site.register(Countryinfo,CountryinfoAdmin) diff --git a/geowebservice/geodatabase/management/commands/load_data.py b/geowebservice/geodatabase/management/commands/load_data.py index e376030..1555099 100644 --- a/geowebservice/geodatabase/management/commands/load_data.py +++ b/geowebservice/geodatabase/management/commands/load_data.py @@ -1,35 +1,25 @@ # -*- coding: utf-8 -*- -from __future__ import print_function -import pysolr - -from django.core.management.base import BaseCommand, CommandError +import logging -import ast -from django.conf import settings +from django.core.management.base import BaseCommand from geodatabase.services import ServiceSolr -import logging logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) + class Command(BaseCommand): help = 'Closes the specified poll for voting' def add_arguments(self, parser): - parser.add_argument('all', nargs='+', type=str) - parser.add_argument('countries', nargs='+', type=str) + parser.add_argument('countries', type=str) + parser.add_argument('all', type=str) def handle(self, *args, **options): - all_locations = '' - countries_names = '' - try: - all_locations = options['all'][0] - countries_names = options['countries'][0] - except: - all_locations = '/home/leonardo/Área de Trabalho/geonames_files/allCountries.txt' - countries_names = '/home/leonardo/Área de Trabalho/geonames_files/country.csv' - + countries_names = options['countries'] + all_locations = options['all'] + s = ServiceSolr() - s.load_contry_info(countries_names) - s.load_initial_data(all_locations) \ No newline at end of file + s.load_country_info(countries_names) + s.load_initial_data(all_locations) diff --git a/geowebservice/geodatabase/models.py b/geowebservice/geodatabase/models.py deleted file mode 100644 index 51eb732..0000000 --- a/geowebservice/geodatabase/models.py +++ /dev/null @@ -1,64 +0,0 @@ -# This is an auto-generated Django model module. -# You'll have to do the following manually to clean this up: -# * Rearrange models' order -# * Make sure each model has one field with primary_key=True -# Feel free to rename the models, but don't rename db_table values or field names. -# -# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]' -# into your database. - -from django.db import models - - - -class Countryinfo(models.Model): - iso_alpha2 = models.CharField(max_length=765, blank=True) - iso_alpha3 = models.CharField(max_length=765, blank=True) - iso_numeric = models.IntegerField(null=True, blank=True) - fips_code = models.CharField(max_length=900, blank=True) - name = models.CharField(max_length=6000, blank=True) - capital = models.CharField(max_length=6000, blank=True) - areainsqkm = models.FloatField(null=True, blank=True) - population = models.IntegerField(null=True, blank=True) - continent = models.CharField(max_length=765, blank=True) - currency = models.CharField(max_length=765, blank=True) - currencyName = models.CharField(max_length=765, db_column='currencyName', blank=True) - geonameId = models.IntegerField(primary_key=True,db_column='geonameId', blank=True) - class Meta: - db_table = u'countryInfo' - - def __unicode__(self): - return str(self.geonameId)+','+self.name+','+self.continent - -class Geoname(models.Model): - geonameid = models.IntegerField(primary_key=True,) - name = models.CharField(max_length=600, blank=True) - asciiname = models.CharField(max_length=600, blank=True) - alternatenames = models.CharField(max_length=12000, blank=True) - latitude = models.DecimalField(null=True, max_digits=12, decimal_places=7, blank=True) - longitude = models.DecimalField(null=True, max_digits=12, decimal_places=7, blank=True) - fclass = models.CharField(max_length=3, blank=True) - fcode = models.CharField(max_length=30, blank=True) - country = models.CharField(max_length=6, blank=True) - cc2 = models.CharField(max_length=180, blank=True) - admin1 = models.CharField(max_length=60, blank=True) - admin2 = models.CharField(max_length=240, blank=True) - admin3 = models.CharField(max_length=60, blank=True) - admin4 = models.CharField(max_length=60, blank=True) - - class Meta: - db_table = u'geoname' - - def __unicode__(self): - selfName = str(self.geonameid)+'\t'+self.asciiname+'\t'+self.fcode - if self.country: - selfName += '\t'+str(self.country) - if self.admin1: - selfName += '\t'+str(self.admin1) - if self.admin2: - selfName += '\t'+str(self.admin2) - if self.admin3: - selfName += '\t'+str(self.admin3) - if self.admin4: - selfName += '\t'+str(self.admin4) - return selfName \ No newline at end of file diff --git a/geowebservice/geodatabase/services.py b/geowebservice/geodatabase/services.py index e42419c..738d354 100644 --- a/geowebservice/geodatabase/services.py +++ b/geowebservice/geodatabase/services.py @@ -16,137 +16,77 @@ """A module to index .. moduleauthor:: Luís A. Bastião Silva """ +import csv +import logging -from __future__ import print_function import pysolr - - -import ast from django.conf import settings -import csv -import random - -import logging - -from utils import * logger = logging.getLogger() - logging.basicConfig(level=logging.DEBUG) -"""convert input to unicode""" -def make_unicode(input): - if type(input) != unicode: - input = input.decode('utf-8') - return input - else: - return input - -""" -Search service, that contains all the information about contries. -It also has methods to load the initial data and to update them -""" class ServiceSolr(object): + """ + Search service, that contains all the information about contries. + It also has methods to load the initial data and to update them + """ + CONNECTION_TIMEOUT_DEFAULT = 99000000 - - def __fetch_initial_settings(self): - try: - self.SOLR_HOST = settings.SOLR_HOST - self.SOLR_PORT = settings.SOLR_PORT - self.SOLR_PATH = settings.SOLR_PATH - except: - # TODO: - # Maybe should load default settings here? - logger.error("It is not running in Django enviroment") - raise - def __init__(self, host="solr", port="8983", path="/solr", timeout=CONNECTION_TIMEOUT_DEFAULT, core="geonames"): + + def __init__(self): # Setup a Solr instance. The timeout is optional. logger.info("Initial Solr Service") - try: - self.__fetch_initial_settings() - except: - self.SOLR_HOST = "localhost" - self.SOLR_PORT = "8983" - self.SOLR_PATH = "/solr" - - self.solr = pysolr.Solr('http://' +self.SOLR_HOST+ ':'+ self.SOLR_PORT+self.SOLR_PATH+'/', timeout=timeout) - logger.info("Connected to Solr") - - def load_contry_info(self, countryFile="country.csv"): - self.contryInfo = {} - - import csv - ifile = open(countryFile, "rU") - #reader = csv.reader(ifile, delimiter='\t') - reader = UnicodeReader(ifile, delimiter='\t', ) - for row in reader: - - #a.name = row[4] - # a.continent = row[8] - # a.geonameId = row[11] - self.contryInfo[row[11]] = {'continent': row[8],'name': row[4]} - - - + + self.solr = pysolr.Solr( + f"http://{settings.SOLR_HOST}:{settings.SOLR_PORT}/solr/{settings.SOLR_CORE}/", + timeout=self.CONNECTION_TIMEOUT_DEFAULT, + ) + logger.info("Connected to Solr") + + def load_country_info(self, countryFile): + logger.info("Loading country info") + + with open(countryFile, "rU") as ifile: + reader = csv.reader(ifile, delimiter="\t") + + self.countryInfo = {row[11]: {'continent': row[8], 'name': row[4]} for row in reader} + def load_initial_data(self, allCountriesFile): logger.info("Load initial data to Solr") - + with open(allCountriesFile) as csvfile: - #spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') - spamreader = UnicodeReader(csvfile, delimiter='\t', quotechar='|') + spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') list_docs_to_commit = [] - i = 0 + i = 0 for row in spamreader: - if not (row[1]=='Earth' or row[7] == 'CONT' or row[7] == 'PCLI' or row[7] == 'ISLS' or row[7] == 'ADM1' or row[7] == 'ADM2' or row[7] == 'ADM3' or row[7] == 'ADM4'): + if not (row[1] == 'Earth' or row[7] == 'CONT' or row[7] == 'PCLI' or row[7] == 'ISLS' or row[7] == 'ADM1' or row[7] == 'ADM2' or row[7] == 'ADM3' or row[7] == 'ADM4'): continue i = i + 1 - d = {} - d['id'] = i - d['geonameId_t'] = row[0] - print(row[0]) - d['name_t'] = row[1] - d['asciiname_t'] = row[2] - d['alternatenames_t'] = row[3] - d['latitude_f'] = row[4] - d['longitude_f'] = row[5] - d['fclass_t'] = row[6] - d['fcode_t'] = row[7] - try: - d['country_t'] = row[8] - except: - pass - d['cc2_t'] = row[9] - d['admin1_t'] = row[10] - d['admin2_t'] = row[11] - d['admin3_t'] = row[12] - d['admin4_t'] = row[13] - d['population_t'] = row[14] - d['elevation_t'] = row[15] - d['gtopo30_t'] = row[16] - d['timezone_t'] = row[17] - - if row[0] in self.contryInfo: - d['continent_t'] = self.contryInfo[row[0]]['continent'] - d['name_t'] = self.contryInfo[row[0]]['name'] - - #print(d) - d = dict(d.items()) + d = {'id': i, 'geonameId_t': row[0], 'name_t': row[1], 'asciiname_t': row[2], + 'alternatenames_t': row[3], 'latitude_f': row[4], 'longitude_f': row[5], 'fclass_t': row[6], + 'fcode_t': row[7], 'country_t': row[8], 'cc2_t': row[9], 'admin1_t': row[10], 'admin2_t': row[11], + 'admin3_t': row[12], + 'admin4_t': row[13], 'population_t': row[14], 'elevation_t': row[15], 'gtopo30_t': row[16], + 'timezone_t': row[17]} + + if row[0] in self.countryInfo: + d['continent_t'] = self.countryInfo[row[0]]['continent'] + d['name_t'] = self.countryInfo[row[0]]['name'] + list_docs_to_commit.append(d) - if (i%60000==0): - xml_answer = self.solr.add(list_docs_to_commit) + if i % 60000 == 0: + self.solr.add(list_docs_to_commit) list_docs_to_commit = [] - print(i) - - xml_answer = self.solr.add(list_docs_to_commit) - list_docs_to_commit = [] + print(i) + + self.solr.add(list_docs_to_commit) self.solr.optimize() - - """Fetch the result by id""" + def geonameId(self, geonameId, start=0, rows=100, fl='', sort='', facet="off"): - print(self.solr) - results = self.solr.search("geonameId_t:"+geonameId,**{ + """Fetch the result by id""" + results = self.solr.search("geonameId_t:" + geonameId, **{ 'facet': facet, 'rows': rows, 'start': start, @@ -154,12 +94,12 @@ def geonameId(self, geonameId, start=0, rows=100, fl='', sort='', facet="off"): 'sort': sort }) return results - - - """Generic search - """ + def search(self, query, start=0, rows=500, fl='', sort='', facet="off"): - results = self.solr.search(query,**{ + """ + Generic search + """ + results = self.solr.search(query, **{ 'facet': facet, 'rows': rows, 'start': start, @@ -167,20 +107,3 @@ def search(self, query, start=0, rows=500, fl='', sort='', facet="off"): 'sort': sort }) return results - - -def main(): - s = ServiceSolr() - allCountriesFile = '/opt/django/app/allCountries.txt' - allCountriesFile = '../allCountries.txt' - countryFile = '/opt/django/app/country.csv' - countryFile = '../country.csv' - print(allCountriesFile) - s.load_contry_info(countryFile) - s.load_initial_data(allCountriesFile) - - #results = s.search("3039162") - #d = results.docs[0] - #print(d) -if __name__ == "__main__": - main() diff --git a/geowebservice/geodatabase/tests.py b/geowebservice/geodatabase/tests.py deleted file mode 100644 index 501deb7..0000000 --- a/geowebservice/geodatabase/tests.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -This file demonstrates writing tests using the unittest module. These will pass -when you run "manage.py test". - -Replace this with more appropriate tests for your application. -""" - -from django.test import TestCase - - -class SimpleTest(TestCase): - def test_basic_addition(self): - """ - Tests that 1 + 1 always equals 2. - """ - self.assertEqual(1 + 1, 2) diff --git a/geowebservice/geodatabase/utils.py b/geowebservice/geodatabase/utils.py deleted file mode 100644 index e962ef2..0000000 --- a/geowebservice/geodatabase/utils.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (C) 2015 Universidade de Aveiro, DETI/IEETA, Bioinformatics Group - http://bioinformatics.ua.pt/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# From http://stackoverflow.com/questions/17245415/read-and-write-csv-files-including-unicode-with-python-2-7 -import csv,codecs,cStringIO - -class UTF8Recoder: - def __init__(self, f, encoding): - self.reader = codecs.getreader(encoding)(f) - def __iter__(self): - return self - def next(self): - return self.reader.next().encode("utf-8") - -class UnicodeReader: - def __init__(self, f, dialect=csv.excel, encoding="utf-8-sig", **kwds): - f = UTF8Recoder(f, encoding) - self.reader = csv.reader(f, dialect=dialect, **kwds) - def next(self): - '''next() -> unicode - This function reads and returns the next line as a Unicode string. - ''' - row = self.reader.next() - return [unicode(s, "utf-8") for s in row] - def __iter__(self): - return self - -class UnicodeWriter: - def __init__(self, f, dialect=csv.excel, encoding="utf-8-sig", **kwds): - self.queue = cStringIO.StringIO() - self.writer = csv.writer(self.queue, dialect=dialect, **kwds) - self.stream = f - self.encoder = codecs.getincrementalencoder(encoding)() - def writerow(self, row): - '''writerow(unicode) -> None - This function takes a Unicode string and encodes it to the output. - ''' - self.writer.writerow([s.encode("utf-8") for s in row]) - data = self.queue.getvalue() - data = data.decode("utf-8") - data = self.encoder.encode(data) - self.stream.write(data) - self.queue.truncate(0) - - def writerows(self, rows): - for row in rows: - self.writerow(row) diff --git a/geowebservice/geodatabase/views.py b/geowebservice/geodatabase/views.py index 0333f98..9c371b5 100644 --- a/geowebservice/geodatabase/views.py +++ b/geowebservice/geodatabase/views.py @@ -15,150 +15,143 @@ # along with this program. If not, see . # Create your views here. import json -from django.http import HttpResponse -from geodatabase.models import Geoname,Countryinfo -from django.http import HttpResponseNotAllowed, HttpResponseBadRequest, HttpResponseNotFound, HttpResponse, HttpResponseForbidden +import logging -from geodatabase.services import * +from django.http import HttpResponseBadRequest, HttpResponse +from .services import ServiceSolr -import logging logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) -# TODO: a refactor on that method need to be done. + +# TODO: a refactor on that method need to be done. # The business logic should not leave here! -"""This method handles with all request for all hierarchies. Thus, depending on the id, -it will analyse what type of location that it handles and then returns -the proper locations according to the hierarchy. -""" + def detail(request, geonameid): - + """ + This method handles with all request for all hierarchies. Thus, depending on the id, + it will analyse what type of location that it handles and then returns + the proper locations according to the hierarchy. + """ + solr = ServiceSolr() + # TODO: validate geonameid properly. - if geonameid!="0": - solr = ServiceSolr() + if geonameid != "0": results = solr.geonameId(geonameid) - + # Check it, if no results in the geonames database, it returns an error - if (len(results)==0): + if len(results) == 0: return HttpResponseBadRequest() - + # Fetch one, and only one document # d variable corresponding now to the geoNameId that is passed as parameter d = results.docs[0] - + # Fetch the data to variables, only to facilitate the access and to become the code easy to read. fcode = d['fcode_t'] name = d['name_t'] - + try: country = d['country_t'] except: pass - + try: admin1 = d['admin1_t'] except: pass - + try: admin2 = d['admin2_t'] except: pass - + try: admin3 = d['admin3_t'] except: pass - + try: admin4 = d['admin4_t'] except: pass else: - name = "Mundus" + name = "Mundus" response_data = [] if name == 'Earth': response_object = solr.search("fcode_t:CONT") - response_data = buildJson(response_object.docs,response_data) - + response_data = buildJson(response_object.docs, response_data) + elif name == 'Mundus': - solr = ServiceSolr() - response_object = solr.search("continent_t:EU OR continent_t:AF OR continent_t:OC OR continent_t:SA OR continent_t:NA OR continent_t:AS") - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search( + "continent_t:EU OR continent_t:AF OR continent_t:OC OR continent_t:SA OR continent_t:NA OR continent_t:AS") + response_data = buildJson(response_object.docs, response_data) elif fcode == 'CONT': if name == 'Europe': tmp = solr.search("continent_t:EU") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'Africa': - tmp = solr.search("continent_t:AF") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'Oceania': tmp = solr.search("continent_t:OC") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'South America': tmp = solr.search("continent_t:SA") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'North America': tmp = solr.search("continent_t:NA") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'Asia': tmp = solr.search("continent_t:AS") - response_data = buildJson(tmp.docs,response_data) + response_data = buildJson(tmp.docs, response_data) elif fcode == 'PCLI': - #response_object = Geoname.objects.filter(country=location[0].country,fcode='ADM1') - response_object = solr.search("country_t:"+country+" AND (fcode_t:ADM1 OR fcode_t:ISLS)") - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search("country_t:" + country + " AND (fcode_t:ADM1 OR fcode_t:ISLS)") + response_data = buildJson(response_object.docs, response_data) elif fcode == 'ADM1' or fcode == 'ISLS': - response_object = solr.search("country_t:"+country+" AND admin1_t:"+admin1+" AND fcode_t:ADM2") - #response_object = Geoname.objects.filter(country=location[0].country,admin1=location[0].admin1,fcode='ADM2') - response_data=buildJson(response_object.docs,response_data) + response_object = solr.search("country_t:" + country + " AND admin1_t:" + admin1 + " AND fcode_t:ADM2") + response_data = buildJson(response_object.docs, response_data) elif fcode == 'ADM2': - #response_object = Geoname.objects.filter(country=location[0].country,admin1=location[0].admin1,admin2=location[0].admin2,fcode='ADM3') - response_object = solr.search("country_t:"+country+" AND admin1_t:"+admin1+" AND admin2_t:"+admin2+" AND fcode_t:ADM3") - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search( + "country_t:" + country + " AND admin1_t:" + admin1 + " AND admin2_t:" + admin2 + " AND fcode_t:ADM3") + response_data = buildJson(response_object.docs, response_data) elif fcode == 'ADM3': - #response_object = Geoname.objects.filter(country=location[0].country,admin1=location[0].admin1,admin2=location[0].admin2,admin3=location[0].admin3,fcode='ADM4') - response_object = solr.search("country_t:"+country+" AND admin1_t:"+admin1+" AND admin2_t:"+admin2+" AND admin3_t:"+admin3+" AND fcode_t:ADM4") - - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search( + "country_t:" + country + " AND admin1_t:" + admin1 + " AND admin2_t:" + admin2 + " AND admin3_t:" + admin3 + " AND fcode_t:ADM4") + + response_data = buildJson(response_object.docs, response_data) elif fcode == 'ADM4': - #response_object = Geoname.objects.filter(country=location[0].country,admin1=location[0].admin1,admin2=location[0].admin2,admin3=location[0].admin3,admin4=location[0].admin4,fcode='ADM5') - response_object = solr.search("country_t:"+country+" AND admin1_t:"+admin1+" AND admin2_t:"+admin2+" AND admin3_t:"+admin3+" AND admin4_t:"+admin4+" AND fcode_t:ADM5") - - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search( + "country_t:" + country + " AND admin1_t:" + admin1 + " AND admin2_t:" + admin2 + " AND admin3_t:" + admin3 + " AND admin4_t:" + admin4 + " AND fcode_t:ADM5") + + response_data = buildJson(response_object.docs, response_data) - return HttpResponse(json.dumps(response_data),content_type="application/json") + return HttpResponse(json.dumps(response_data), content_type="application/json") -""" Builds the answer with multiples geonames entries""" -def buildJson(response_object,response_data): - for i in range(0,len(response_object)-1): - print(response_object[i]) - response_data = response_data+addEntry(response_object[i],response_data) +def buildJson(response_object, response_data): + """ Builds the answer with multiples geonames entries""" + for i in range(0, len(response_object) - 1): + response_data = response_data + addEntry(response_object[i]) return response_data -""" -Handles a GeoName entry -""" -def addEntry(geoname,response_data): - response = {} - print(geoname) - response['geonameid'] = int(geoname['geonameId_t']) - response['name'] = geoname['name_t'] - response['fcode'] = geoname['fcode_t'] +def addEntry(geoname): + """ + Handles a GeoName entry + """ + response = {'geonameid': int(geoname['geonameId_t']), 'name': geoname['name_t'], 'fcode': geoname['fcode_t']} try: response['country'] = geoname['country_t'] response['adm1'] = geoname['admin1_t'] @@ -167,34 +160,36 @@ def addEntry(geoname,response_data): response['adm4'] = geoname['admin3_t'] except: pass - + return [response] -""" -Get coordinates by location name and fcode -""" -def getCoordinates(request,location): - if location == '' or location == None: - return HttpResponse(json.dumps([]),content_type="application/json") + +def getCoordinates(request, location): + """ + Get coordinates by location name and fcode + """ + if location == '' or location is None: + return HttpResponse(json.dumps([]), content_type="application/json") location = location.strip().split(',') - levels = [ 'PCLI', 'ADM1', 'ADM2', 'ADM3', 'ADM4', 'ADM5' ] - fcode = levels[len(location)-1] + levels = ['PCLI', 'ADM1', 'ADM2', 'ADM3', 'ADM4', 'ADM5'] + fcode = levels[len(location) - 1] name = location[0] - + try: solr = ServiceSolr() - response_object = solr.search("fcode_t:"+fcode+" AND name_t:"+name) + response_object = solr.search("fcode_t:" + fcode + " AND name_t:" + name) if fcode == 'ADM1' and response_object.hits == 0: - response_object = solr.search("fcode_t:ISLS AND name_t:"+name) + response_object = solr.search("fcode_t:ISLS AND name_t:" + name) response_object = response_object.docs[0] - response_data = buildCoordinates(response_object,[]) - return HttpResponse(json.dumps(response_data),content_type="application/json") + response_data = buildCoordinates(response_object) + return HttpResponse(json.dumps(response_data), content_type="application/json") except: - return HttpResponse(json.dumps([]),content_type="application/json") + return HttpResponse(json.dumps([]), content_type="application/json") + -""" -Build a list with latitude and longitude of the location -""" -def buildCoordinates(response_object,response_data): - return [ response_object['latitude_f'], response_object['longitude_f'] ] \ No newline at end of file +def buildCoordinates(response_object): + """ + Build a list with latitude and longitude of the location + """ + return [response_object['latitude_f'], response_object['longitude_f']] diff --git a/geowebservice/geowebservice/asgi.py b/geowebservice/geowebservice/asgi.py new file mode 100644 index 0000000..ceb3cca --- /dev/null +++ b/geowebservice/geowebservice/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for geowebservice project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'geowebservice.settings') + +application = get_asgi_application() diff --git a/geowebservice/geowebservice/settings.py b/geowebservice/geowebservice/settings.py index b13b207..6fc6310 100644 --- a/geowebservice/geowebservice/settings.py +++ b/geowebservice/geowebservice/settings.py @@ -1,149 +1,111 @@ # Django settings for geowebservice project. +import os +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / "subdir". +BASE_DIR = Path(__file__).resolve().parent.parent + +# Make this unique, and don"t share it with anybody. +SECRET_KEY = "%suk*!m42-@mw+)))6%g6u=dpwseyxew&zij619qovu&=#4yk#" + DEBUG = True -TEMPLATE_DEBUG = DEBUG -ADMINS = ( - # ('Your Name', 'your_email@example.com'), -) +ALLOWED_HOSTS = ["*"] -MANAGERS = ADMINS +INSTALLED_APPS = ( + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "corsheaders", + "geodatabase", +) +MIDDLEWARE = ( + "corsheaders.middleware.CorsMiddleware", + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +) +ROOT_URLCONF = "geowebservice.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'geonames', - 'USER': 'root', - 'PASSWORD': 'root', - 'HOST': 'hs', - 'PORT': '3306', - } -} +WSGI_APPLICATION = "geowebservice.wsgi.application" +# Database +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases DATABASES = { - 'default' : { - 'ENGINE' :'django.db.backends.sqlite3', - 'NAME' : 'sqlite3.db', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", } } -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', - # 'django.template.loaders.eggs.Loader', -) - -# Hosts/domain names that are valid for this site; required if DEBUG is False -# See https://docs.djangoproject.com/en/1.4/ref/settings/#allowed-hosts -ALLOWED_HOSTS = ["*"] - -# Local time zone for this installation. Choices can be found here: -# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name -# although not all choices may be available on all operating systems. -# In a Windows environment this must be set to your system time zone. -TIME_ZONE = 'America/Chicago' +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] -# Language code for this installation. All choices can be found here: -# http://www.i18nguy.com/unicode/language-identifiers.html -LANGUAGE_CODE = 'en-us' +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ +LANGUAGE_CODE = "en-us" -SITE_ID = 1 +TIME_ZONE = "America/Chicago" -# If you set this to False, Django will make some optimizations so as not -# to load the internationalization machinery. USE_I18N = True -# If you set this to False, Django will not format dates, numbers and -# calendars according to the current locale. USE_L10N = True -# If you set this to False, Django will not use timezone-aware datetimes. USE_TZ = True -# Absolute filesystem path to the directory that will hold user-uploaded files. -# Example: "/home/media/media.lawrence.com/media/" -MEDIA_ROOT = '' - -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash. -# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" -MEDIA_URL = '' - -# Absolute path to the directory static files should be collected to. -# Don't put anything in this directory yourself; store your static files -# in apps' "static/" subdirectories and in STATICFILES_DIRS. -# Example: "/home/media/media.lawrence.com/static/" -STATIC_ROOT = '' - -# URL prefix for static files. -# Example: "http://media.lawrence.com/static/" -STATIC_URL = '/static/' - -# Additional locations of static files -STATICFILES_DIRS = ( - # Put strings here, like "/home/html/static" or "C:/www/django/static". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. -) - -# List of finder classes that know how to find static files in -# various locations. -STATICFILES_FINDERS = ( - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', -# 'django.contrib.staticfiles.finders.DefaultStorageFinder', -) - -# Make this unique, and don't share it with anybody. -SECRET_KEY = '%suk*!m42-@mw+)))6%g6u=dpwseyxew&zij619qovu&=#4yk#' +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ +STATIC_URL = "/static/" -# List of callables that know how to import templates from various sources. -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', -# 'django.template.loaders.eggs.Loader', -) - -MIDDLEWARE_CLASSES = ( - 'corsheaders.middleware.CorsMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - # Uncomment the next line for simple clickjacking protection: - # 'django.middleware.clickjacking.XFrameOptionsMiddleware', -) +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -ROOT_URLCONF = 'geowebservice.urls' -# Python dotted path to the WSGI application used by Django's runserver. -WSGI_APPLICATION = 'geowebservice.wsgi.application' - -TEMPLATE_DIRS = ( - # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. +ADMINS = ( + # ("Your Name", "your_email@example.com"), ) -INSTALLED_APPS = ( +MANAGERS = ADMINS - 'django.contrib.admin', - - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.messages', - 'django.contrib.staticfiles', - # Uncomment the next line to enable the admin: - 'corsheaders', - 'geodatabase', - # Uncomment the next line to enable admin documentation: - # 'django.contrib.admindocs', -) # A sample logging configuration. The only tangible logging # performed by this configuration is to send an email to @@ -151,33 +113,32 @@ # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse' + "version": 1, + "disable_existing_loggers": False, + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse" } }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'filters': ['require_debug_false'], - 'class': 'django.utils.log.AdminEmailHandler' + "handlers": { + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false"], + "class": "django.utils.log.AdminEmailHandler" } }, - 'loggers': { - 'django.request': { - 'handlers': ['mail_admins'], - 'level': 'ERROR', - 'propagate': True, + "loggers": { + "django.request": { + "handlers": ["mail_admins"], + "level": "ERROR", + "propagate": True, }, } } -SOLR_HOST = "localhost" -SOLR_PORT = "8983" -SOLR_PATH = "/solr" - +SOLR_HOST = os.environ.get("SOLR_HOST", "solr") +SOLR_PORT = os.environ.get("SOLR_PORT", "8983") +SOLR_CORE = os.environ.get("SOLR_CORE", "geonames") -CORS_ORIGIN_ALLOW_ALL = True \ No newline at end of file +CORS_ORIGIN_ALLOW_ALL = True diff --git a/geowebservice/geowebservice/urls.py b/geowebservice/geowebservice/urls.py index 676a8f6..bba6a23 100644 --- a/geowebservice/geowebservice/urls.py +++ b/geowebservice/geowebservice/urls.py @@ -1,24 +1,10 @@ -from django.conf.urls import patterns, include, url - -# Uncomment the next two lines to enable the admin: -from django.contrib import admin -admin.autodiscover() - -urlpatterns = patterns('', - # Examples: - # url(r'^$', '{{ project_name }}.views.home', name='home'), - # url(r'^{{ project_name }}/', include('{{ project_name }}.foo.urls')), - # url(r'^geodatabase/$', 'geodatabase.views.index'), +from django.urls import re_path +from geodatabase import views +urlpatterns = [ # get child locations by geonameid - url(r'^geodatabase/(?P\d+)/$', 'geodatabase.views.detail'), + re_path(r"^geodropdown/geodatabase/(?P\d+)/$", views.detail), # get coordinates by name and fcode - url(r'^geodatabase/(?P[a-zA-Z, ]+)/$', 'geodatabase.views.getCoordinates'), - - # Uncomment the admin/doc line below to enable admin documentation: - # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - - # Uncomment the next line to enable the admin: - url(r'^admin/', include(admin.site.urls)), -) \ No newline at end of file + re_path(r"^geodropdown/geodatabase/(?P[a-zA-Z, ]+)/$", views.getCoordinates), +] diff --git a/geowebservice/geowebservice/wsgi.py b/geowebservice/geowebservice/wsgi.py index f20f185..3da33e6 100644 --- a/geowebservice/geowebservice/wsgi.py +++ b/geowebservice/geowebservice/wsgi.py @@ -1,28 +1,16 @@ """ WSGI config for geowebservice project. -This module contains the WSGI application used by Django's development server -and any production WSGI deployments. It should expose a module-level variable -named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover -this application via the ``WSGI_APPLICATION`` setting. - -Usually you will have the standard Django WSGI application here, but it also -might make sense to replace the whole Django WSGI application with a custom one -that later delegates to the Django one. For example, you could introduce WSGI -middleware here, or combine a Django application with an application of another -framework. +It exposes the WSGI callable as a module-level variable named ``application``. +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ """ -import os -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geowebservice.settings") +import os -# This application object is used by any WSGI server configured to use this -# file. This includes Django's development server, if the WSGI_APPLICATION -# setting points here. from django.core.wsgi import get_wsgi_application -application = get_wsgi_application() -# Apply WSGI middleware here. -# from helloworld.wsgi import HelloWorldApplication -# application = HelloWorldApplication(application) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'geowebservice.settings') + +application = get_wsgi_application() diff --git a/geowebservice/import_countryInfo.py b/geowebservice/import_countryInfo.py deleted file mode 100644 index 5052671..0000000 --- a/geowebservice/import_countryInfo.py +++ /dev/null @@ -1,26 +0,0 @@ - - - -import csv - -from geodatabase.models import Countryinfo - -import csv -ifile = open("country.csv", "rU") -reader = csv.reader(ifile, delimiter='\t') -for row in reader: - print row - a = Countryinfo() - a.iso_alpha2 = row[0] - a.iso_alpha3 = row[1] - a.iso_numeric = row[2] - a.fips_code = row[3] - a.name = row[4] - a.capital = row[5] - a.areainsqkm = row[6] - a.population = row[7] - a.continent = row[8] - a.currency = row[9] - a.currencyName = row[10] - a.geonameId = row[11] - a.save() \ No newline at end of file diff --git a/geowebservice/load_data.py b/geowebservice/load_data.py deleted file mode 100644 index 7157665..0000000 --- a/geowebservice/load_data.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - - -from geodatabase.models import Geoname -from django.db import transaction - -import csv - -i = 0 -def make_unicode(input): - if type(input) != unicode: - input = input.decode('utf-8') - return input - else: - return input -with open('/Users/bastiao/Downloads/allCountries.txt') as csvfile: - spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') - l = [] - - for row in spamreader: - i = i + 1 - - g = Geoname() - try: - g.geonameid = int(row[0]) - except: - pass - try: - g.name = make_unicode(row[1]) - except: - pass - try: - - g.asciiname = make_unicode(row[2]) - #g.alternatenames = make_unicode(row[3]) - except: - pass - - g.latitude = float(row[4]) - g.longitude = float(row[5]) - g.fclass = row[6] - g.country = row[7] - g.cc2 = row[8] - g.admin1 = row[9] - g.admin2 = row[10] - g.admin3 = row[11] - g.admin4 = row[12] - try: - g.population = int(row[13]) - g.elevation = int(row[14]) - g.gtopo30 = int(row[15]) - g.timezone = row[16] - except: - pass - #g.moddate = Date(row[17]) - if (i%6000==0): - print i - Geoname.objects.bulk_create(l) - l = [] - else: - l.append(g) - Geoname.objects.bulk_create(l) - - \ No newline at end of file diff --git a/geowebservice/requirements.txt b/geowebservice/requirements.txt index 6df1f1d..e800a3c 100644 --- a/geowebservice/requirements.txt +++ b/geowebservice/requirements.txt @@ -1,3 +1,4 @@ -Django==1.4.5 -pysolr==3.0.6 -django-cors-headers==0.13 +Django==3.2.11 +django-cors-headers==3.11.0 +gunicorn==20.1.0 +pysolr==3.8.1 \ No newline at end of file diff --git a/geowebservice/run.sh b/geowebservice/run.sh deleted file mode 100644 index 0ccb684..0000000 --- a/geowebservice/run.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e - -MODULE=${MODULE:-website} - -sed -i "s#module=website.wsgi:application#module=${MODULE}.wsgi:application#g" /opt/django/uwsgi.ini - -if [ ! -f "/opt/django/app/manage.py" ] -then - echo "creating basic django project (module: ${MODULE})" - django-admin.py startproject ${MODULE} /opt/django/app/ -fi - -exec /usr/bin/supervisord diff --git a/geowebservice/sql/geonames_countryInfo.sql b/geowebservice/sql/geonames_countryInfo.sql deleted file mode 100644 index 7d2bbbd..0000000 --- a/geowebservice/sql/geonames_countryInfo.sql +++ /dev/null @@ -1,50 +0,0 @@ --- MySQL dump 10.13 Distrib 5.6.24, for linux-glibc2.5 (x86_64) --- --- Host: 127.0.0.1 Database: geonames --- ------------------------------------------------------ --- Server version 5.6.25-0ubuntu0.15.04.1 - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Table structure for table `countryInfo` --- - -DROP TABLE IF EXISTS `countryInfo`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `countryInfo` ( - `iso_alpha2` char(255) DEFAULT NULL, - `iso_alpha3` char(255) DEFAULT NULL, - `iso_numeric` int(11) DEFAULT NULL, - `fips_code` varchar(300) DEFAULT NULL, - `name` varchar(2000) DEFAULT NULL, - `capital` varchar(2000) DEFAULT NULL, - `areainsqkm` double DEFAULT NULL, - `population` int(11) DEFAULT NULL, - `continent` char(255) DEFAULT NULL, - `currency` char(255) DEFAULT NULL, - `currencyName` char(255) DEFAULT NULL, - `geonameId` int(11) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -/*!40101 SET character_set_client = @saved_cs_client */; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed on 2015-10-27 19:13:52 diff --git a/geowebservice/sql/geonames_geoname.sql b/geowebservice/sql/geonames_geoname.sql deleted file mode 100644 index c89aa64..0000000 --- a/geowebservice/sql/geonames_geoname.sql +++ /dev/null @@ -1,53 +0,0 @@ --- MySQL dump 10.13 Distrib 5.6.24, for linux-glibc2.5 (x86_64) --- --- Host: 127.0.0.1 Database: geonames --- ------------------------------------------------------ --- Server version 5.6.25-0ubuntu0.15.04.1 - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Table structure for table `geoname` --- - -DROP TABLE IF EXISTS `geoname`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `geoname` ( - `geonameid` int(11) NOT NULL, - `name` varchar(2000) CHARACTER SET utf8 DEFAULT NULL, - `asciiname` varchar(2000) CHARACTER SET utf8 DEFAULT NULL, - `alternatenames` varchar(4000) CHARACTER SET utf8 DEFAULT NULL, - `latitude` decimal(10,7) DEFAULT NULL, - `longitude` decimal(10,7) DEFAULT NULL, - `fclass` char(1) CHARACTER SET utf8 DEFAULT NULL, - `fcode` varchar(100) CHARACTER SET utf8 DEFAULT NULL, - `country` varchar(2) CHARACTER SET utf8 DEFAULT NULL, - `cc2` varchar(600) CHARACTER SET utf8 DEFAULT NULL, - `admin1` varchar(200) CHARACTER SET utf8 DEFAULT NULL, - `admin2` varchar(800) CHARACTER SET utf8 DEFAULT NULL, - `admin3` varchar(200) CHARACTER SET utf8 DEFAULT NULL, - `admin4` varchar(200) CHARACTER SET utf8 DEFAULT NULL, - UNIQUE KEY `idx_geoname_geonameid` (`geonameid`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed on 2015-10-27 19:13:52 diff --git a/geowebservice/sqlite3.db b/geowebservice/sqlite3.db deleted file mode 100644 index fe1eb2f..0000000 Binary files a/geowebservice/sqlite3.db and /dev/null differ diff --git a/geowebservice/start.sh b/geowebservice/start.sh old mode 100644 new mode 100755 index 6658c98..641cb65 --- a/geowebservice/start.sh +++ b/geowebservice/start.sh @@ -1 +1,28 @@ -docker-compose up -d --no-recreate +#!/bin/sh + +PREVIOUS_PWD=$(pwd) +cd $(dirname "$0") + +docker-compose up -d solr + +until docker-compose exec -T solr sh -c "solr status" | grep running ; do + sleep 2 +done + +( + set -e + + docker-compose exec -T solr sh -c "solr create_core -c geonames -d basic_configs" + + docker-compose up -d geodropdownservice + + docker-compose exec -T geodropdownservice sh -c """ + set -x + wget -P /tmp http://download.geonames.org/export/dump/allCountries.zip + unzip /tmp/allCountries.zip -d /tmp + python manage.py load_data country.csv /tmp/allCountries.txt + rm /tmp/allCountries.* + """ +) + +cd $PREVIOUS_PWD diff --git a/geowebservice/supervisord.conf b/geowebservice/supervisord.conf deleted file mode 100644 index 2a8d336..0000000 --- a/geowebservice/supervisord.conf +++ /dev/null @@ -1,8 +0,0 @@ -[supervisord] -nodaemon=true - -[program:uwsgi] -command = /usr/local/bin/uwsgi --ini /opt/django/uwsgi.ini --touch-reload /opt/django/reload - -[program:nginx] -command = /usr/sbin/nginx diff --git a/geowebservice/uwsgi.ini b/geowebservice/uwsgi.ini deleted file mode 100644 index a1f0c4b..0000000 --- a/geowebservice/uwsgi.ini +++ /dev/null @@ -1,31 +0,0 @@ -[uwsgi] -# this config will be loaded if nothing specific is specified -# load base config from below -ini = :base - -# %d is the dir this configuration file is in -socket = %dapp.sock -master = true -processes = 4 - -[dev] -ini = :base -# socket (uwsgi) is not the same as http, nor http-socket -socket = :8001 - - -[local] -ini = :base -http = :8000 -# set the virtual env to use -home=/Users/you/envs/env - - -[base] -# chdir to the folder of this config file, plus app/website -chdir = /opt/django -# load the module from wsgi.py, it is a python path from -# the directory above. -module=geowebservice.wsgi:application -# allow anyone to connect to the socket. This is very permissive -chmod-socket=666 diff --git a/geowebservice/uwsgi_params b/geowebservice/uwsgi_params deleted file mode 100644 index f539451..0000000 --- a/geowebservice/uwsgi_params +++ /dev/null @@ -1,16 +0,0 @@ - -uwsgi_param QUERY_STRING $query_string; -uwsgi_param REQUEST_METHOD $request_method; -uwsgi_param CONTENT_TYPE $content_type; -uwsgi_param CONTENT_LENGTH $content_length; - -uwsgi_param REQUEST_URI $request_uri; -uwsgi_param PATH_INFO $document_uri; -uwsgi_param DOCUMENT_ROOT $document_root; -uwsgi_param SERVER_PROTOCOL $server_protocol; -uwsgi_param HTTPS $https if_not_empty; - -uwsgi_param REMOTE_ADDR $remote_addr; -uwsgi_param REMOTE_PORT $remote_port; -uwsgi_param SERVER_PORT $server_port; -uwsgi_param SERVER_NAME $server_name; diff --git a/package.json b/package.json deleted file mode 100644 index ea216ff..0000000 --- a/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "GeoDropdown", - "version": "0.0.0", - "description": "dropdown lists location system", - "devDependencies": { - "grunt": "^0.4.5", - "grunt-contrib-copy": "^0.8.1", - "grunt-contrib-htmlmin": "^0.5.0", - "grunt-contrib-uglify": "^0.9.2" - } -}