Merge pull request #60 from retromeet/add_postgis_and_locations

Add postgis and locations

.env.template

@@ -13,3 +13,15 @@ HMAC_SECRET=
 # The secret key used for generating the jwt tokens
 # You can generate one using irb: `require "securerandom"; SecureRandom.hex(64)`
 JWT_SECRET=
+
+# We use photon (https://photon.komoot.io/) to get geolocation for the users.
+# You can configure which instance of photon to use and which languages are supported by that instance.
+# PHOTON_API_HOST=https://photon.komoot.io
+# The first language on the list will be the default in case the user's language is not supported.
+# PHOTON_SUPPORTED_LANGUAGES=en,fr,de
+
+
+# We use nominatim (https://nominatim.org/) as a fallback for geolocation.
+# We use it whenever we cannot use photon, such as when the language is not supported by photon
+# However, nominatim's API limits are low. So this allows us to configure our own nominatim host
+# NOMINATIM_API_HOST=https://nominatim.openstreetmap.org

.github/workflows/run-tests.yml

@@ -13,7 +13,7 @@ jobs:
     # Service containers to run with `runner-job`
     services:
       postgres:
-        image: postgres:16
+        image: postgis/postgis:16-3.5
         # Provide the password for postgres
         env:
           POSTGRES_PASSWORD: postgres
@@ -39,8 +39,10 @@ jobs:
       run: cp .env.github.test .env.test.local
     - name: Create needed user
       run: PGPASSWORD=postgres createuser -U postgres -h localhost postgres_password
-    - name: Create needed extension
+    - name: Create needed extensions
       run: PGPASSWORD=postgres psql -h localhost -U postgres -c "CREATE EXTENSION citext" retromeet_test
+      # We don't need to create the extension because it's already in the image
+      # && PGPASSWORD=postgres psql -h localhost -U postgres -c "CREATE EXTENSION postgis" retromeet_test
     - name: Setup the database
       run: APP_ENV=test bundle exec rake db:setup
     - name: Run tests

Gemfile

@@ -31,6 +31,8 @@ gem "zeitwerk" # Used for autoloading the code in the API
 
 gem "async-http" # Used for making requests towards other APIs
 
+gem "address_composer", github: "retromeet/address_composer", submodules: true
+
 group :development, :test do
   gem "dotenv" # Used to load environment variables from .env files
 

Gemfile.lock

@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/retromeet/address_composer.git
+  revision: 9170b3fe686ac651a53ec26f59e700baa1eb5bb5
+  submodules: true
+  specs:
+    address_composer (1.0.1)
+      mustache (~> 1.1)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -157,6 +165,7 @@ GEM
     multi_json (1.15.0)
     multi_xml (0.7.1)
       bigdecimal (~> 3.1)
+    mustache (1.1.1)
     mustermann (3.0.3)
       ruby2_keywords (~> 0.0.1)
     mustermann-grape (1.1.0)
@@ -285,6 +294,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  address_composer!
   async-http
   bcrypt
   database_cleaner-sequel

README.md

@@ -16,7 +16,7 @@ There's a [pronto](https://github.com/prontolabs/pronto) github action running o
 
 ### Setup
 
-RetroMeet requires Postgresql >= 16.0 (it might work with a lower version than that, but it is not guaranteed).
+RetroMeet requires Postgresql >= 16.0 (it might work with a lower version than that, but it is not guaranteed) and PostGIS >= 3.4 (again, might work with a lower version, but not guaranteed).
 
 First, we need to set up the database. RetroMeet uses [rodauth](https://github.com/jeremyevans/rodauth), the following instructions will create the needed users, database and extensions needed for roda.
 1. Create two users:
@@ -32,6 +32,10 @@ createdb -U postgres -O retromeet retromeet_dev
 ```sh
 psql -U postgres -c "CREATE EXTENSION citext" retromeet_dev
 ```
+1. Load the postgis extension:
+```sh
+psql -U postgres -c "CREATE EXTENSION postgis" retromeet_dev
+```
 1. Give the password user temporary rights to the schema:
 ```sh
 psql -U postgres -c "GRANT CREATE ON SCHEMA public TO retromeet_password" retromeet_dev
@@ -51,6 +55,7 @@ The same setup needs to be done for the test database, replacing `retromeet_dev`
 ```sh
 createdb -U postgres -O retromeet retromeet_test
 psql -U postgres -c "CREATE EXTENSION citext" retromeet_test
+psql -U postgres -c "CREATE EXTENSION postgis" retromeet_test
 psql -U postgres -c "GRANT CREATE ON SCHEMA public TO retromeet_password" retromeet_test
 RACK_ENV=test rake db:setup
 psql -U postgres -c "REVOKE CREATE ON SCHEMA public FROM retromeet_password" retromeet_test

app/api/authenticated/profile.rb

@@ -26,7 +26,7 @@ class Profile < Grape::API
         end
 
         desc "Updates the current user's profile with the given parameters. The return will only contain fields that could have been modified.",
-             success: { model: API::Entities::ProfileInfo, message: "The profile for the authenticated user" },
+             success: { status: 200, model: API::Entities::ProfileInfo, message: "The profile for the authenticated user" },
              failure: Authenticated::FAILURES,
              produces: Authenticated::PRODUCES,
              consumes: Authenticated::CONSUMES
@@ -55,12 +55,31 @@ class Profile < Grape::API
           coerce_empty_array_param_to_nil(declared_params, :genders)
           coerce_empty_array_param_to_nil(declared_params, :orientations)
           coerce_empty_array_param_to_nil(declared_params, :languages)
+          error!({ error: :AT_LEAST_ONE_PARAMETER_NEEDED, detail: "You need to provide at least one parameter to be changed, none given" }, :bad_request) if declared_params.empty?
 
           Persistence::Repository::Account.update_profile_info(account_id: rodauth.session[:account_id], **declared_params)
           profile_info = Persistence::Repository::Account.profile_info(account_id: rodauth.session[:account_id])
           status :ok
           Entities::ProfileInfo.represent(profile_info, only: declared_params.keys.map(&:to_sym))
         end
+
+        desc "Updates the current user's profile location with the given place.",
+             success: { status: 200, model: API::Entities::ProfileInfo, message: "The profile for the authenticated user" },
+             failure: Authenticated::FAILURES,
+             produces: Authenticated::PRODUCES,
+             consumes: Authenticated::CONSUMES
+        params do
+          requires :location, type: String, desc: "The place you're updating to. It should be one of the responses from /api/search/address"
+        end
+        post :location do
+          results = PhotonClient.search(query: params[:location])
+          error!({ error: :UNEXPECTED_RESULTS_SIZE, detail: "Expected to have exactly one location with the given name, had #{results.size} instead" }, :unprocessable_content) if results.size != 1
+
+          Persistence::Repository::Account.update_profile_location(account_id: rodauth.session[:account_id], location_result: results.first)
+          profile_info = Persistence::Repository::Account.profile_info(account_id: rodauth.session[:account_id])
+          status :ok
+          Entities::ProfileInfo.represent(profile_info, only: %i[location])
+        end
       end
     end
   end

app/api/authenticated/search.rb

@@ -6,7 +6,8 @@ module Authenticated
     class Search < Grape::API
       namespace :search do
         desc "Given an address, searches for a geolocation using one location provider. It takes the language of the current user in consideration.",
-             success: { model: API::Entities::ProfileInfo, message: "The profile for the authenticated user" },
+             success: { code: 200, model: API::Entities::LocationResult, message: "One or more locations" },
+             is_array: true,
              failure: Authenticated::FAILURES,
              produces: Authenticated::PRODUCES,
              consumes: Authenticated::CONSUMES
@@ -16,7 +17,7 @@ class Search < Grape::API
         end
         post :address do
           status :ok
-          NominatimClient.search(query: params[:query], limit: params[:limit])
+          Entities::LocationResult.represent(LocationServiceProxy.search(query: params[:query], limit: params[:limit]))
         end
       end
     end

app/api/base.rb

@@ -11,6 +11,12 @@ class Base < Grape::API
 
     helpers API::Helpers::Params
 
+    unless Environment.test?
+      rescue_from :all do |_e|
+        error!({ error: "Internal server error" }, 500)
+      end
+    end
+
     if Environment.development?
       add_swagger_documentation \
         mount_path: "/swagger_doc",

app/api/entities/location_result.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module API
+  module Entities
+    # Represents the profile info entity for the API
+    class LocationResult < Grape::Entity
+      expose :latitude, documentation: { type: Float }
+      expose :longitude, documentation: { type: Float }
+      expose :display_name, documentation: { type: String }
+    end
+  end
+end

app/api/entities/profile_info.rb

@@ -21,6 +21,7 @@ class ProfileInfo < BasicProfileInfo
       expose :wants_kids, documentation: { type: String }
       expose :religion, documentation: { type: String }
       expose :religion_importance, documentation: { type: String }
+      expose :location_display_name, documentation: { type: Hash }
     end
   end
 end

app/models/location_result.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Models
+  LocationResult = Data.define(:latitude, :longitude, :display_name, :osm_id, :country_code, :language) do
+    def initialize(latitude:, longitude:, display_name:, osm_id:, country_code:, language:)
+      country_code = country_code.downcase
+      super
+    end
+  end
+end

app/objects/location_service_proxy.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# This module automatically chooses the best Location Service based on the requested language
+module LocationServiceProxy
+  class << self
+    # @param query [String] A query to be sent to nominatim
+    # @param limit [Integer] The max results to return
+    # @param language [String] The language for the results
+    # @return [Array<Models::LocationResult>]
+    def search(query:, limit: nil, language: "en")
+      if PhotonClient.language_supported?(language)
+        limit = PhotonClient::MAX_SEARCH_RESULTS if limit.nil?
+        PhotonClient.search(query:, limit:, language:)
+      else
+        limit = NominatimClient::MAX_SEARCH_RESULTS if limit.nil?
+        NominatimClient.search(query:, limit:, language:)
+      end
+    end
+  end
+end

app/objects/nominatim_client.rb

@@ -9,33 +9,47 @@ class << self
     # @param query [String] A query to be sent to nominatim
     # @param limit [Integer] The max results to return
     # @param language [String] The language for the results
+    # @return [Array<Models::LocationResult>]
     def search(query:, limit: MAX_SEARCH_RESULTS, language: "en")
+      limit = MAX_SEARCH_RESULTS if limit > MAX_SEARCH_RESULTS
+
       params = {
         q: CGI.escape(query),
         format: :jsonv2,
         language:,
         limit:,
         layer: :address,
-        featureType: :settlement
+        featureType: :settlement,
+        addressdetails: 1
       }
       query_params = params.map { |k, v| "#{k}=#{v}" }.join("&")
-      Sync do
+      results = Sync do
         response = client.get("/search?#{query_params}", headers: base_headers)
-        # TODO: (renatolond, 2024-11-05) I'm not too sure of the result format, but it will do for now
         JSON.parse(response.read, symbolize_names: true)
-            .map do |loc|
-          loc.slice(:lat, :lon, :display_name)
-        end
       ensure
         response&.close
       end
+      results.map do |result|
+        components = result[:address].slice(*AddressComposer::AllComponents)
+        display_name = AddressComposer.compose(components)
+        display_name.chomp!
+        display_name.gsub!("\n", ", ")
+        Models::LocationResult.new(
+          latitude: result[:lat].to_f,
+          longitude: result[:lon].to_f,
+          display_name:,
+          osm_id: result[:osm_id],
+          country_code: components[:country_code],
+          language:
+        )
+      end
     end
 
     private
 
       # Returns the retromeet-core base host to be used for requests based off the environment variables
       # @return [Async::HTTP::Endpoint]
-      def nominatim_host = @nominatim_host ||= Async::HTTP::Endpoint.parse("https://nominatim.openstreetmap.org")
+      def nominatim_host = @nominatim_host ||= Async::HTTP::Endpoint.parse(ENV.fetch("NOMINATIM_API_HOST", "https://nominatim.openstreetmap.org"))
 
       # @return [Hash] Base headers to be used for requests
       def base_headers = @base_headers ||= { "Content-Type" => "application/json", "User-Agent": RetroMeet::Version.user_agent }.freeze

app/objects/photon_client.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# This modules contains functions to interact with the {https://photon.komoot.io/ Photon} backend.
+# The backend is configured through ENV variables and will use the default backend otherwise.
+module PhotonClient
+  MAX_SEARCH_RESULTS = 10
+
+  class << self
+    # @param query [String] A query to be sent to nominatim
+    # @param limit [Integer] The max results to return
+    # @param language [String] The language for the results, can only be one of +supported_languages+.
+    # @return [Array<Models::LocationResult>]
+    def search(query:, limit: MAX_SEARCH_RESULTS, language: "en")
+      language = normalize_language(language)
+      params = {
+        q: CGI.escape(query),
+        lang: language,
+        limit:
+      }
+      layers = "layer=state&layer=county&layer=city&layer=district"
+      query_params = params.map { |k, v| "#{k}=#{v}" }
+      query_params << layers
+      query_params = query_params.join("&")
+      results = Sync do
+        response = client.get("/api?#{query_params}", headers: base_headers)
+        JSON.parse(response.read, symbolize_names: true)
+      ensure
+        response&.close
+      end
+      results[:features].map do |place|
+        components = place[:properties].slice(*AddressComposer::AllComponents)
+        components[:country_code] = place[:properties][:countrycode]
+        components[place[:properties][:osm_value]] = place[:properties][:name]
+        longitude, latitude = place[:geometry][:coordinates]
+        display_name = AddressComposer.compose(components)
+        display_name.chomp!
+        display_name.gsub!("\n", ", ")
+        Models::LocationResult.new(
+          latitude:,
+          longitude:,
+          display_name:,
+          osm_id: place[:properties][:osm_id],
+          country_code: components[:country_code],
+          language:
+        )
+      end
+    end
+
+    # @param language (see .search)
+    # @return [Boolean]
+    def language_supported?(language) = supported_languages.include?(language)
+
+    private
+
+      # The public photon API only supports a few languages (en, fr, de)
+      # to support more languages another photon instance has to be used with a custom import
+      # This will normalize any language not supported by the default endpoint to one of the supported ones
+      # you can override the supported languages with the env variable defined below
+      #
+      # @param language (see .search)
+      # @return [String]
+      def normalize_language(language)
+        return language if language_supported?(language)
+
+        supported_languages.first
+      end
+
+      # @return [Array<String>]
+      def supported_languages
+        @supported_languages ||= ENV.fetch("PHOTON_SUPPORTED_LANGUAGES", "en,fr,de").split(",")
+      end
+
+      # Returns the photon host to be used for requests
+      # @return [Async::HTTP::Endpoint]
+      def photon_host = @photon_host ||= Async::HTTP::Endpoint.parse(ENV.fetch("PHOTON_API_HOST", "https://photon.komoot.io"))
+
+      # @return [Hash] Base headers to be used for requests
+      def base_headers = @base_headers ||= { "Content-Type" => "application/json", "User-Agent": RetroMeet::Version.user_agent }.freeze
+
+      # @return [Async::HTTP::Client]
+      def client = Async::HTTP::Client.new(photon_host)
+  end
+end

app/objects/retromeet/version.rb

@@ -39,7 +39,7 @@ def prerelease
       # Takes the +RETROMEET_VERSION_METADATA+ environment variable into consideration
       # @return [String,nil]
       def build_metadata
-        ENV.fetch("MASTODON_VERSION_METADATA", nil)
+        ENV.fetch("RETROMEET_VERSION_METADATA", nil)
       end
 
       # @return [Array<String>]