From f6bfbede044ade20fc7876c7c5b87fd4a57a3bcd Mon Sep 17 00:00:00 2001 From: Jack Reed Date: Thu, 11 Jan 2018 16:11:28 -0700 Subject: [PATCH 1/4] move all constants to a separate module --- app/models/geo_monitor/layer.rb | 2 +- app/models/geo_monitor/status.rb | 2 +- lib/geo_monitor.rb | 3 +++ lib/geo_monitor/bounding_box.rb | 6 +++--- lib/geo_monitor/constants.rb | 6 ++++++ lib/geo_monitor/engine.rb | 11 +++++------ lib/geo_monitor/lat_lng_point.rb | 4 ++-- lib/geo_monitor/requests/wms.rb | 4 ++-- 8 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 lib/geo_monitor/constants.rb diff --git a/app/models/geo_monitor/layer.rb b/app/models/geo_monitor/layer.rb index 5ce77ee..4df3cd2 100644 --- a/app/models/geo_monitor/layer.rb +++ b/app/models/geo_monitor/layer.rb @@ -28,7 +28,7 @@ def check bbox: bounding_box, url: url, layers: layername ).tile end - GeoMonitor::Status.from_response(response, self, time.real.to_f) + ::GeoMonitor::Status.from_response(response, self, time.real.to_f) end def availability_score diff --git a/app/models/geo_monitor/status.rb b/app/models/geo_monitor/status.rb index b29602d..75fbcd0 100644 --- a/app/models/geo_monitor/status.rb +++ b/app/models/geo_monitor/status.rb @@ -7,7 +7,7 @@ class Status < ApplicationRecord # Limits the number of statuses per layer to prevent a ballooing database def limit_by_layer statuses_by_layer = self.class.where(layer: layer).count - max = GeoMonitor::Engine.config.max_status_per_layer + max = ::GeoMonitor::Engine.config.max_status_per_layer self.class .where(layer: layer) .last(statuses_by_layer - max + 1) diff --git a/lib/geo_monitor.rb b/lib/geo_monitor.rb index 2154725..3a657ec 100644 --- a/lib/geo_monitor.rb +++ b/lib/geo_monitor.rb @@ -1,6 +1,9 @@ require "geo_monitor/engine" module GeoMonitor + extend ActiveSupport::Autoload + + # A simple structure to conform to Faraday::Response FailedResponse = Struct.new(:env, :status, :headers) end diff --git a/lib/geo_monitor/bounding_box.rb b/lib/geo_monitor/bounding_box.rb index 0681a54..6594413 100644 --- a/lib/geo_monitor/bounding_box.rb +++ b/lib/geo_monitor/bounding_box.rb @@ -38,10 +38,10 @@ def zoom_level lng_diff = east - west max_diff = [lat_diff, lng_diff].max - if max_diff < ::GeoMonitor::DEGREES_IN_CIRCLE / 2**20 + if max_diff < ::GeoMonitor::Constants::DEGREES_IN_CIRCLE / 2**20 zoom = 21 else - zoom = -1 * ((Math.log(max_diff) / Math.log(2)) - (Math.log(::GeoMonitor::DEGREES_IN_CIRCLE) / Math.log(2))) + zoom = -1 * ((Math.log(max_diff) / Math.log(2)) - (Math.log(::GeoMonitor::Constants::DEGREES_IN_CIRCLE) / Math.log(2))) zoom = 1 if zoom < 1 end zoom.ceil @@ -52,7 +52,7 @@ def zoom_level def tile_number lat_rad = south / 180 * Math::PI n = 2.0**zoom_level - x = ((west + 180.0) / ::GeoMonitor::DEGREES_IN_CIRCLE * n).to_i + x = ((west + 180.0) / ::GeoMonitor::Constants::DEGREES_IN_CIRCLE * n).to_i y = ((1.0 - Math.log(Math.tan(lat_rad) + (1 / Math.cos(lat_rad))) / Math::PI) / 2.0 * n) y = if y.infinite?.nil? y.to_i diff --git a/lib/geo_monitor/constants.rb b/lib/geo_monitor/constants.rb new file mode 100644 index 0000000..f6511b8 --- /dev/null +++ b/lib/geo_monitor/constants.rb @@ -0,0 +1,6 @@ +module GeoMonitor + module Constants + R = 6_378_137 # Radius of Earth in meters + DEGREES_IN_CIRCLE = 360.0 + end +end diff --git a/lib/geo_monitor/engine.rb b/lib/geo_monitor/engine.rb index acc9299..819f4d4 100644 --- a/lib/geo_monitor/engine.rb +++ b/lib/geo_monitor/engine.rb @@ -1,16 +1,15 @@ require 'rails/all' require 'faraday' -require 'geo_monitor/bounding_box' -require 'geo_monitor/lat_lng_point' -require 'geo_monitor/requests/wms' module GeoMonitor - R = 6_378_137 # Radius of Earth in meters - DEGREES_IN_CIRCLE = 360.0 - ## # Top level Rails Engine class class Engine < ::Rails::Engine + require 'geo_monitor/constants' + require 'geo_monitor/bounding_box' + require 'geo_monitor/lat_lng_point' + require 'geo_monitor/requests/wms' + isolate_namespace GeoMonitor GeoMonitor::Engine.config.max_status_per_layer = 5 diff --git a/lib/geo_monitor/lat_lng_point.rb b/lib/geo_monitor/lat_lng_point.rb index b57e608..6d91644 100644 --- a/lib/geo_monitor/lat_lng_point.rb +++ b/lib/geo_monitor/lat_lng_point.rb @@ -14,8 +14,8 @@ def to_3857 max = 1 - 1E-15 sin = [[Math.sin(lng * d), max].min, -max].max self.class.new( - lat: GeoMonitor::R * lat * d, - lng: GeoMonitor::R * Math.log((1 + sin) / (1 - sin)) / 2 + lat: ::GeoMonitor::Constants::R * lat * d, + lng: ::GeoMonitor::Constants::R * Math.log((1 + sin) / (1 - sin)) / 2 ) end diff --git a/lib/geo_monitor/requests/wms.rb b/lib/geo_monitor/requests/wms.rb index 1ee7ab7..8b2288b 100644 --- a/lib/geo_monitor/requests/wms.rb +++ b/lib/geo_monitor/requests/wms.rb @@ -34,7 +34,7 @@ def request_params # Request the tile. def tile unless url.present? - return GeoMonitor::FailedResponse.new( + return ::GeoMonitor::FailedResponse.new( { url: url }, 'No URL provided', {} ) end @@ -46,7 +46,7 @@ def tile request.options.open_timeout = 10 end rescue Faraday::ConnectionFailed, Faraday::TimeoutError => e - GeoMonitor::FailedResponse.new( + ::GeoMonitor::FailedResponse.new( { url: conn.url_prefix.to_s }, e.class, nil From 903b97081c450c7d12238f5b925878c1d9db9700 Mon Sep 17 00:00:00 2001 From: Jack Reed Date: Fri, 12 Jan 2018 09:27:17 -0700 Subject: [PATCH 2/4] add support for type of IIIF layer --- app/models/geo_monitor/layer.rb | 9 +++++-- spec/fixtures/image.json | 34 +++++++++++++++++++++++++++ spec/models/geo_monitor/layer_spec.rb | 30 ++++++++++++++++------- 3 files changed, 63 insertions(+), 10 deletions(-) create mode 100644 spec/fixtures/image.json diff --git a/app/models/geo_monitor/layer.rb b/app/models/geo_monitor/layer.rb index 4df3cd2..281fdc6 100644 --- a/app/models/geo_monitor/layer.rb +++ b/app/models/geo_monitor/layer.rb @@ -8,10 +8,15 @@ def self.from_geoblacklight(schema_json) schema = JSON.parse(schema_json) references = JSON.parse(schema['dct_references_s']) find_or_create_by(slug: schema['layer_slug_s']) do |layer| - layer.checktype = 'WMS' + layer.checktype = + if references['http://www.opengis.net/def/serviceType/ogc/wms'] + 'WMS' + elsif references['http://iiif.io/api/image'] + 'IIIF' + end layer.layername = schema['layer_id_s'] layer.bbox = schema['solr_geom'] - layer.url = references['http://www.opengis.net/def/serviceType/ogc/wms'] + layer.url = references['http://www.opengis.net/def/serviceType/ogc/wms'] || references['http://iiif.io/api/image'] layer.active = true end end diff --git a/spec/fixtures/image.json b/spec/fixtures/image.json new file mode 100644 index 0000000..673f20c --- /dev/null +++ b/spec/fixtures/image.json @@ -0,0 +1,34 @@ +{ + "solr_year_i": "1777", + "dc_description_s": "The provinces of New York, and New Jersey; with part of Pensilvania and the province of Quebec. Drawn by Major Holland, Surveyor General of the Northern District in America", + "layer_modified_dt": "2015-01-25T02:06:19Z", + "dc_creator_sm": [ + "Holland, Samuel,", + "Coentgen, Heinrich Hugo.", + "Brönner, H. L.", + "Fox, N. B.," + ], + "layer_id_s": "jd472z992", + "dc_type_s": "Image", + "dc_identifier_s": "jd472z992", + "dct_provenance_s": "Princeton", + "dc_title_s": "The provinces of New York, and New Jersey; with part of Pensilvania and the province of Quebec. Drawn by Major Holland, Surveyor General of the Northern District in America", + "layer_slug_s": "princeton-jd472z992", + "dc_format_s": "Scanned Map", + "dct_references_s": "{\"http://www.loc.gov/standards/marcxml\": \"https://geowebservices.princeton.edu/download/items/jd472z992/marc.xml\", \"http://iiif.io/api/image\": \"https://libimages.princeton.edu/loris/pulmap/jd/47/2z/99/2/00000001.jp2/info.json\"}", + "layer_geom_type_s": "Image", + "dct_temporal_sm": [ + "1777" + ], + "solr_geom": "ENVELOPE(-73.727775, -66.8850751, 47.459686, 40.950943)", + "dc_rights_s": "Public", + "dct_spatial_sm": [ + "New England", + "New York (State)", + "New Jersey", + "Québec (Province)", + "New York (State)", + "New Jersey" + ], + "geoblacklight_version": "1.0" +} diff --git a/spec/models/geo_monitor/layer_spec.rb b/spec/models/geo_monitor/layer_spec.rb index 33ae173..13bfc78 100644 --- a/spec/models/geo_monitor/layer_spec.rb +++ b/spec/models/geo_monitor/layer_spec.rb @@ -4,14 +4,28 @@ let(:schema_json) { File.read('spec/fixtures/cz128vq0535.json') } subject { described_class.from_geoblacklight(schema_json) } describe '.from_geoblacklight' do - it 'creates an instance with expected fields' do - expect(subject.checktype).to eq 'WMS' - expect(subject.slug).to eq 'stanford-cz128vq0535' - expect(subject.layername).to eq 'druid:cz128vq0535' - expect(subject.bbox) - .to eq 'ENVELOPE(29.572742, 35.000308, 4.234077, -1.478794)' - expect(subject.active).to eq true - expect(subject.url).to eq 'https://geowebservices.stanford.edu/geoserver/wms' + context 'WMS layer' do + it 'creates an instance with expected fields' do + expect(subject.checktype).to eq 'WMS' + expect(subject.slug).to eq 'stanford-cz128vq0535' + expect(subject.layername).to eq 'druid:cz128vq0535' + expect(subject.bbox) + .to eq 'ENVELOPE(29.572742, 35.000308, 4.234077, -1.478794)' + expect(subject.active).to eq true + expect(subject.url).to eq 'https://geowebservices.stanford.edu/geoserver/wms' + end + end + context 'IIIF layer' do + let(:schema_json) { File.read('spec/fixtures/image.json') } + it 'creates an instance with expected fields' do + expect(subject.checktype).to eq 'IIIF' + expect(subject.slug).to eq 'princeton-jd472z992' + expect(subject.layername).to eq 'jd472z992' + expect(subject.bbox) + .to eq 'ENVELOPE(-73.727775, -66.8850751, 47.459686, 40.950943)' + expect(subject.active).to eq true + expect(subject.url).to eq 'https://libimages.princeton.edu/loris/pulmap/jd/47/2z/99/2/00000001.jp2/info.json' + end end end describe '#bounding_box' do From 5aababd5e03c08803c21e23cd71c14eaacb7f461 Mon Sep 17 00:00:00 2001 From: Jack Reed Date: Fri, 12 Jan 2018 15:25:28 -0700 Subject: [PATCH 3/4] add content_type to status used to calculate availability score --- app/models/geo_monitor/layer.rb | 2 +- app/models/geo_monitor/status.rb | 5 +++-- db/migrate/20180112220603_add_content_type_to_status.rb | 5 +++++ spec/models/geo_monitor/layer_spec.rb | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 db/migrate/20180112220603_add_content_type_to_status.rb diff --git a/app/models/geo_monitor/layer.rb b/app/models/geo_monitor/layer.rb index 281fdc6..c4b2bf5 100644 --- a/app/models/geo_monitor/layer.rb +++ b/app/models/geo_monitor/layer.rb @@ -37,7 +37,7 @@ def check end def availability_score - statuses.where(res_code: '200').count.to_f / statuses.count + statuses.where(res_code: '200', content_type: 'image/png').count.to_f / statuses.count end end end diff --git a/app/models/geo_monitor/status.rb b/app/models/geo_monitor/status.rb index 75fbcd0..c3f1e97 100644 --- a/app/models/geo_monitor/status.rb +++ b/app/models/geo_monitor/status.rb @@ -15,7 +15,7 @@ def limit_by_layer end ## - # @param [Faraday::Resposne] response + # @param [Faraday::Response] response # @param [GeoMonitor::Layer] layer # @param [Float] time def self.from_response(response, layer, time) @@ -24,7 +24,8 @@ def self.from_response(response, layer, time) res_code: response.status, submitted_query: response.env[:url].to_s, layer: layer, - res_headers: response.headers + res_headers: response.headers, + content_type: response.headers.try(:[], :content_type) ) end end diff --git a/db/migrate/20180112220603_add_content_type_to_status.rb b/db/migrate/20180112220603_add_content_type_to_status.rb new file mode 100644 index 0000000..5cb7cce --- /dev/null +++ b/db/migrate/20180112220603_add_content_type_to_status.rb @@ -0,0 +1,5 @@ +class AddContentTypeToStatus < ActiveRecord::Migration[5.1] + def change + add_column :geo_monitor_statuses, :content_type, :string + end +end diff --git a/spec/models/geo_monitor/layer_spec.rb b/spec/models/geo_monitor/layer_spec.rb index 13bfc78..90acc30 100644 --- a/spec/models/geo_monitor/layer_spec.rb +++ b/spec/models/geo_monitor/layer_spec.rb @@ -60,7 +60,7 @@ end describe '#availability_score' do before do - 4.times { GeoMonitor::Status.create(layer: subject, res_code: '200') } + 4.times { GeoMonitor::Status.create(layer: subject, res_code: '200', content_type: 'image/png') } 1.times { GeoMonitor::Status.create(layer: subject, res_code: '404') } end it 'calculates the availability score' do From 5cd004c2fc558867264e7f0fd797afcb3a9f9fbd Mon Sep 17 00:00:00 2001 From: Jack Reed Date: Tue, 16 Jan 2018 08:10:08 -0700 Subject: [PATCH 4/4] store rights and institution for a layer --- app/models/geo_monitor/layer.rb | 2 ++ db/migrate/20180116150504_add_more_layer_fields.rb | 6 ++++++ spec/models/geo_monitor/layer_spec.rb | 4 ++++ 3 files changed, 12 insertions(+) create mode 100644 db/migrate/20180116150504_add_more_layer_fields.rb diff --git a/app/models/geo_monitor/layer.rb b/app/models/geo_monitor/layer.rb index c4b2bf5..c804512 100644 --- a/app/models/geo_monitor/layer.rb +++ b/app/models/geo_monitor/layer.rb @@ -14,6 +14,8 @@ def self.from_geoblacklight(schema_json) elsif references['http://iiif.io/api/image'] 'IIIF' end + layer.institution = schema['dct_provenance_s'] + layer.rights = schema['dc_rights_s'] layer.layername = schema['layer_id_s'] layer.bbox = schema['solr_geom'] layer.url = references['http://www.opengis.net/def/serviceType/ogc/wms'] || references['http://iiif.io/api/image'] diff --git a/db/migrate/20180116150504_add_more_layer_fields.rb b/db/migrate/20180116150504_add_more_layer_fields.rb new file mode 100644 index 0000000..92192c0 --- /dev/null +++ b/db/migrate/20180116150504_add_more_layer_fields.rb @@ -0,0 +1,6 @@ +class AddMoreLayerFields < ActiveRecord::Migration[5.1] + def change + add_column :geo_monitor_layers, :rights, :string + add_column :geo_monitor_layers, :institution, :string + end +end diff --git a/spec/models/geo_monitor/layer_spec.rb b/spec/models/geo_monitor/layer_spec.rb index 90acc30..a4130e6 100644 --- a/spec/models/geo_monitor/layer_spec.rb +++ b/spec/models/geo_monitor/layer_spec.rb @@ -12,6 +12,8 @@ expect(subject.bbox) .to eq 'ENVELOPE(29.572742, 35.000308, 4.234077, -1.478794)' expect(subject.active).to eq true + expect(subject.institution).to eq 'Stanford' + expect(subject.rights).to eq 'Public' expect(subject.url).to eq 'https://geowebservices.stanford.edu/geoserver/wms' end end @@ -24,6 +26,8 @@ expect(subject.bbox) .to eq 'ENVELOPE(-73.727775, -66.8850751, 47.459686, 40.950943)' expect(subject.active).to eq true + expect(subject.institution).to eq 'Princeton' + expect(subject.rights).to eq 'Public' expect(subject.url).to eq 'https://libimages.princeton.edu/loris/pulmap/jd/47/2z/99/2/00000001.jp2/info.json' end end