From 7f700e6e6c001ce3be9b6aba89a546eaf4a13a8d Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Mon, 5 Feb 2024 19:01:40 +0545 Subject: [PATCH 1/5] Upgraded oauth login from 1 to 2 --- core/settings/base.py | 3 +- core/settings/contrib.py | 85 ++++++++++++++++++++----------------- requirements.txt | 2 +- ui/app/actions/meta.js | 1 + ui/templates/osm/email.html | 2 +- ui/views.py | 7 +-- 6 files changed, 54 insertions(+), 46 deletions(-) diff --git a/core/settings/base.py b/core/settings/base.py index af3bf90ff..655b7e30f 100644 --- a/core/settings/base.py +++ b/core/settings/base.py @@ -123,7 +123,8 @@ ) AUTHENTICATION_BACKENDS = ( - "social_core.backends.openstreetmap.OpenStreetMapOAuth", + # "social_core.backends.openstreetmap.OpenStreetMapOAuth", + "social_core.backends.openstreetmap_oauth2.OpenStreetMapOAuth2", "oauth2_provider.backends.OAuth2Backend", "social_core.backends.email.EmailAuth", "social_core.backends.username.UsernameAuth", diff --git a/core/settings/contrib.py b/core/settings/contrib.py index 53116df06..77466cdc5 100644 --- a/core/settings/contrib.py +++ b/core/settings/contrib.py @@ -8,58 +8,63 @@ # Extra installed apps INSTALLED_APPS += ( # any 3rd party apps - 'rest_framework', - 'rest_framework_gis', - 'rest_framework.authtoken', - 'social_django', + "rest_framework", + "rest_framework_gis", + "rest_framework.authtoken", + "social_django", ) # 3rd party specific app settings OAUTH2_PROVIDER = { - 'ACCESS_TOKEN_EXPIRE_SECONDS': 10 * 365 * 24 * 60 * 60, + "ACCESS_TOKEN_EXPIRE_SECONDS": 10 * 365 * 24 * 60 * 60, } REST_FRAMEWORK = { - 'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.SearchFilter', - 'rest_framework.filters.OrderingFilter'), - 'DEFAULT_AUTHENTICATION_CLASSES': ('rest_framework.authentication.TokenAuthentication', - 'oauth2_provider.contrib.rest_framework.OAuth2Authentication', - 'rest_framework.authentication.SessionAuthentication',), - 'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAuthenticated',), - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - 'api.renderers.HOTExportApiRenderer', + "DEFAULT_FILTER_BACKENDS": ( + "rest_framework.filters.SearchFilter", + "rest_framework.filters.OrderingFilter", ), - 'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.AcceptHeaderVersioning', - 'DEFAULT_VERSION': '1.0', - 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', - 'PAGE_SIZE': 20 + "DEFAULT_AUTHENTICATION_CLASSES": ( + "rest_framework.authentication.TokenAuthentication", + "oauth2_provider.contrib.rest_framework.OAuth2Authentication", + "rest_framework.authentication.SessionAuthentication", + ), + "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), + "DEFAULT_RENDERER_CLASSES": ( + "rest_framework.renderers.JSONRenderer", + "api.renderers.HOTExportApiRenderer", + ), + "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.AcceptHeaderVersioning", + "DEFAULT_VERSION": "1.0", + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", + "PAGE_SIZE": 20, } + # OAuth login settings -SOCIAL_AUTH_OPENSTREETMAP_LOGIN_URL = '/osm/login/' -SOCIAL_AUTH_OPENSTREETMAP_KEY = os.getenv('OSM_API_KEY') -SOCIAL_AUTH_OPENSTREETMAP_SECRET = os.getenv('OSM_API_SECRET') -SOCIAL_AUTH_LOGIN_REDIRECT_URL = '/' -SOCIAL_AUTH_LOGIN_ERROR_URL = '/osm/error' -SOCIAL_AUTH_URL_NAMESPACE = 'osm' -SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = ['username', 'first_name', 'email'] +SOCIAL_AUTH_OPENSTREETMAP_LOGIN_URL = "/osm/login/" +SOCIAL_AUTH_OPENSTREETMAP_OAUTH2_KEY = os.getenv("OSM_API_KEY") +SOCIAL_AUTH_OPENSTREETMAP_OAUTH2_SECRET = os.getenv("OSM_API_SECRET") +SOCIAL_AUTH_LOGIN_REDIRECT_URL = "/" +SOCIAL_AUTH_LOGIN_ERROR_URL = "/osm/error" +SOCIAL_AUTH_URL_NAMESPACE = "osm" +SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = ["username", "first_name", "email"] SOCIAL_AUTH_FORCE_EMAIL_VALIDATION = True -SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = 'ui.pipeline.email_validation' -SOCIAL_AUTH_EMAIL_VALIDATION_URL = '/osm/email_verify_sent/' +SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = "ui.pipeline.email_validation" +SOCIAL_AUTH_EMAIL_VALIDATION_URL = "/osm/email_verify_sent/" SOCIAL_AUTH_PIPELINE = ( - 'social_core.pipeline.social_auth.social_details', - 'social_core.pipeline.social_auth.social_uid', - 'social_core.pipeline.social_auth.auth_allowed', - 'social_core.pipeline.social_auth.social_user', - 'social_core.pipeline.user.get_username', - 'ui.pipeline.require_email', - 'social_core.pipeline.mail.mail_validation', - 'social_core.pipeline.social_auth.associate_by_email', - 'social_core.pipeline.user.create_user', - 'social_core.pipeline.social_auth.associate_user', - 'social_core.pipeline.debug.debug', - 'social_core.pipeline.social_auth.load_extra_data', - 'social_core.pipeline.user.user_details' + "social_core.pipeline.social_auth.social_details", + "social_core.pipeline.social_auth.social_uid", + "social_core.pipeline.social_auth.auth_allowed", + "social_core.pipeline.social_auth.social_user", + "social_core.pipeline.user.get_username", + "ui.pipeline.require_email", + "social_core.pipeline.mail.mail_validation", + "social_core.pipeline.social_auth.associate_by_email", + "social_core.pipeline.user.create_user", + "social_core.pipeline.social_auth.associate_user", + "social_core.pipeline.debug.debug", + "social_core.pipeline.social_auth.load_extra_data", + "social_core.pipeline.user.user_details", ) diff --git a/requirements.txt b/requirements.txt index 87aaa0904..b32183eac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ mercantile~=0.10.0 psycopg2 python3-openid==3.2.0 social-auth-app-django==5.4.0 -social-auth-core==4.4.2 +social-auth-core==4.4.2 ### Upgrade this to include oauth2 pytz pyyaml>=5.3 raven diff --git a/ui/app/actions/meta.js b/ui/app/actions/meta.js index 04b5293cc..3025d07fa 100644 --- a/ui/app/actions/meta.js +++ b/ui/app/actions/meta.js @@ -27,6 +27,7 @@ if (window.OAUTH_CLIENT_ID == null) { } const oauthConfig = { + // url: window.EXPORTS_API_URL + "/o/openstreetmap_oauth2", url: window.EXPORTS_API_URL + "/o/authorize?approval_prompt=auto", client: window.OAUTH_CLIENT_ID, redirect: `${window.location.protocol}//${hostname}/authorized` diff --git a/ui/templates/osm/email.html b/ui/templates/osm/email.html index 3239f274e..f94eb197c 100644 --- a/ui/templates/osm/email.html +++ b/ui/templates/osm/email.html @@ -14,7 +14,7 @@ {% blocktrans %}Please provide a valid email address. This email address will be used to notifiy you when your exports are ready. A verification link will be sent to the email address you provide.{% endblocktrans %}

-
+ {% csrf_token %}
diff --git a/ui/views.py b/ui/views.py index ce64c7004..a5496b931 100644 --- a/ui/views.py +++ b/ui/views.py @@ -30,7 +30,9 @@ def login(request): if not request.user.is_authenticated: # preserve redirects ("next" in request.GET) return redirect( - reverse("osm:begin", args=["openstreetmap"]) + "?" + request.GET.urlencode() + reverse("osm:begin", args=["openstreetmap-oauth2"]) + + "?" + + request.GET.urlencode() ) else: return redirect("/v3/") @@ -48,7 +50,7 @@ def v3(request, *args, **kwargs): except Application.DoesNotExist: ui_app = Application.objects.create( name="OSM Export Tool UI", - redirect_uris="http://localhost/authorized http://localhost:8080/authorized http://localhost:8000/authorized", + redirect_uris="http://localhost/authorized http://127.0.0.1:8000/authorized http://localhost:8080/authorized http://localhost:8000/authorized", client_type=Application.CLIENT_PUBLIC, authorization_grant_type=Application.GRANT_IMPLICIT, skip_authorization=True, @@ -70,7 +72,6 @@ def redirect_to_v3(request): def worker_dashboard(request): if not request.user.is_superuser: return HttpResponseForbidden() - # return HttpResponse('test') return HttpResponseRedirect(f"/{settings.WORKER_SECRET_KEY}/") From f0aba756e10bdf27f1cdf9c94fc8e2c41546e846 Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Mon, 5 Feb 2024 19:11:32 +0545 Subject: [PATCH 2/5] Upgrade documentation for oauth 2 --- ops/README.md | 2 +- ops/systemd/export_workers.env | 6 +++--- ui/app/components/help/API.js | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ops/README.md b/ops/README.md index 5df2f6082..6358dad39 100644 --- a/ops/README.md +++ b/ops/README.md @@ -62,7 +62,7 @@ Examples of how to backup and restore the database: psql exports < export_tool_2023-03-06.pgdump ``` -8. Modify the OAuth1 application with your hostname's `redirect_uris` +8. Modify the OAuth2 application with your hostname's `redirect_uris` ### Storage and Environment Variables diff --git a/ops/systemd/export_workers.env b/ops/systemd/export_workers.env index 9b5951271..4d33eeb54 100644 --- a/ops/systemd/export_workers.env +++ b/ops/systemd/export_workers.env @@ -8,9 +8,9 @@ EMAIL_HOST_USER= EMAIL_HOST_PASSWORD= REPLY_TO_EMAIL= -##OAUTH 1 Settings -OSM_API_KEY= -OSM_API_SECRET= +##OAUTH 2 Settings +OSM_API_KEY= +OSM_API_SECRET= ## Workers WORKER_SECRET_KEY= diff --git a/ui/app/components/help/API.js b/ui/app/components/help/API.js index 212e7b140..a4388ca18 100644 --- a/ui/app/components/help/API.js +++ b/ui/app/components/help/API.js @@ -33,7 +33,7 @@ export default () =>

User authentication and authorization is a two-step process.

The Export Tool requires that users log into OpenStreetMap using{" "} - OAuth 1.0a (you don't need to know + OAuth 2.0a (you don't need to know this). This provides user identity, specifically an OSM username to associate with exports. You generally don't need to care about this, except to know that usernames are the same as on OSM. From 88c48c496db72fca16f6c86147f6ac6b20c5a61d Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Tue, 6 Feb 2024 21:01:34 +0545 Subject: [PATCH 3/5] Added data completeness stats info --- ui/app/components/ExportForm.js | 83 ++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/ui/app/components/ExportForm.js b/ui/app/components/ExportForm.js index d46fe020e..b28ca378f 100644 --- a/ui/app/components/ExportForm.js +++ b/ui/app/components/ExportForm.js @@ -1,4 +1,5 @@ import area from "@turf/area"; +import axios from "axios"; import bbox from "@turf/bbox"; import React, { Component } from "react"; import { Col, Nav, Panel, Row } from "react-bootstrap"; @@ -8,7 +9,6 @@ import { Redirect, Route, Switch } from "react-router"; import { NavLink } from "react-router-dom"; import { Fields, formValueSelector, reduxForm } from "redux-form"; import { pointToTile } from "tilebelt"; - import ChooseFormats from "./ChooseFormats"; import DescribeExport from "./DescribeExport"; import ExportAOIField from "./ExportAOIField"; @@ -128,6 +128,86 @@ export class ExportForm extends Component { }; } + async fetchData(geometry) { + const url = "https://api-prod.raw-data.hotosm.org/v1/stats/polygon/"; + try { + const response = await axios.post(url, { + geometry: geometry + }, { + headers: {"Content-Type": "application/json"} + }); + + if (response.data) { + + this.setState({ fetchedInfo: response.data }); + } + } catch (error) { + console.error("Failed to fetch summary data", error); + + } + } + + componentDidUpdate(prevProps) { + if (this.props.formValues.the_geom !== prevProps.formValues.the_geom) { + this.fetchData(this.props.formValues.the_geom); + } + } + + renderFetchedInfo() { + const { fetchedInfo } = this.state; + + if (!fetchedInfo) return null; + + // Function to trigger the download of the raw data as a JSON file + const downloadRawData = () => { + const filename = "raw_region_summary.json"; + const jsonStr = JSON.stringify(fetchedInfo, null, 4); + const element = document.createElement('a'); + element.setAttribute('href', 'data:text/json;charset=utf-8,' + encodeURIComponent(jsonStr)); + element.setAttribute('download', filename); + element.style.display = 'none'; + document.body.appendChild(element); + element.click(); + document.body.removeChild(element); + }; + + return ( + +

+
+ Buildings: +

+ +

+
+
+ Roads: +

+ +

+
+
+ More info: + Download, + Indicators, + Metrics +
+
+ + ); + } + + + + componentWillMount() { const { getConfigurations, getOverpassTimestamp, getGalaxyTimestamp} = this.props; @@ -281,6 +361,7 @@ export class ExportForm extends Component { />} /> + {this.renderFetchedInfo()} Date: Wed, 7 Feb 2024 11:11:19 +0545 Subject: [PATCH 4/5] Update requirements.txt to point towards fork, Changes raw data api url to read from env variable , Moved mbtiles to top --- docs/setup-development.md | 2 +- requirements.txt | 2 +- ui/app/actions/exports.js | 4 ++-- ui/app/actions/meta.js | 4 ++++ ui/app/components/ExportForm.js | 8 ++++---- ui/app/components/utils.js | 10 +++++----- ui/templates/ui/v3.html | 1 + ui/views.py | 4 +++- ui/webpack.config.js | 4 +++- 9 files changed, 24 insertions(+), 15 deletions(-) diff --git a/docs/setup-development.md b/docs/setup-development.md index 965b29dc8..ae7f824f3 100644 --- a/docs/setup-development.md +++ b/docs/setup-development.md @@ -111,7 +111,7 @@ Most of these environment variables have reasonable default settings. - `GARMIN_CONFIG`, `GARMIN_MKGMAP` absolute paths to garmin JARs - `OVERPASS_API_URL` url of Overpass api endpoint -- `RAW_DATA_API_URL` url of Galaxy api endpoint +- `RAW_DATA_API_URL` url of Raw data api endpoint - `DATABASE_URL` Database URL. Defaults to `postgres:///exports` - `DEBUG` Whether to enable debug mode. Defaults to `False` (production). diff --git a/requirements.txt b/requirements.txt index b32183eac..9a7faec3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ mercantile~=0.10.0 psycopg2 python3-openid==3.2.0 social-auth-app-django==5.4.0 -social-auth-core==4.4.2 ### Upgrade this to include oauth2 +social-auth-core @ git+https://github.com/kshitijrajsharma/social-core.git ### Upgrade this to include osm oauth2 when released pytz pyyaml>=5.3 raven diff --git a/ui/app/actions/exports.js b/ui/app/actions/exports.js index 0512af16c..d583ba5f9 100644 --- a/ui/app/actions/exports.js +++ b/ui/app/actions/exports.js @@ -170,8 +170,8 @@ export const getOverpassTimestamp = () => (dispatch, getState) => { export const getGalaxyTimestamp = () => (dispatch, getState) => { return axios({ - baseURL: "https://api-prod.raw-data.hotosm.org/v1", - url: "/status/" + baseURL: window.RAW_DATA_API_URL, + url: "v1/status/" }) .then(response => dispatch({ diff --git a/ui/app/actions/meta.js b/ui/app/actions/meta.js index 3025d07fa..ab5ad30a8 100644 --- a/ui/app/actions/meta.js +++ b/ui/app/actions/meta.js @@ -10,6 +10,10 @@ if (window.location.port) { hostname += `:${window.location.port}`; } +if (window.RAW_DATA_API_URL == null) { + window.RAW_DATA_API_URL = process.env.RAW_DATA_API_URL; +} + if (window.EXPORTS_API_URL == null) { window.EXPORTS_API_URL = process.env.EXPORTS_API_URL; diff --git a/ui/app/components/ExportForm.js b/ui/app/components/ExportForm.js index b28ca378f..0e4e623b2 100644 --- a/ui/app/components/ExportForm.js +++ b/ui/app/components/ExportForm.js @@ -129,7 +129,7 @@ export class ExportForm extends Component { } async fetchData(geometry) { - const url = "https://api-prod.raw-data.hotosm.org/v1/stats/polygon/"; + const url = window.RAW_DATA_API_URL + "v1/stats/polygon/"; try { const response = await axios.post(url, { geometry: geometry @@ -155,7 +155,7 @@ export class ExportForm extends Component { renderFetchedInfo() { const { fetchedInfo } = this.state; - + if (!this.props.formValues.the_geom) return null; if (!fetchedInfo) return null; // Function to trigger the download of the raw data as a JSON file @@ -172,7 +172,7 @@ export class ExportForm extends Component { }; return ( - +
Buildings: @@ -365,7 +365,7 @@ export class ExportForm extends Component { diff --git a/ui/app/components/utils.js b/ui/app/components/utils.js index 58fbcb9b0..4fcd6e892 100644 --- a/ui/app/components/utils.js +++ b/ui/app/components/utils.js @@ -58,6 +58,11 @@ export const AVAILABLE_EXPORT_FORMATS = { SQL .sql ), + mbtiles: ( + + MBTiles .mbtiles + + ), garmin_img: ( Garmin .img @@ -83,11 +88,6 @@ export const AVAILABLE_EXPORT_FORMATS = { OsmAnd .obf ), - mbtiles: ( - - MBTiles .mbtiles - - ), bundle: ( POSM bundle diff --git a/ui/templates/ui/v3.html b/ui/templates/ui/v3.html index 7f9fcfd17..1649f8f67 100644 --- a/ui/templates/ui/v3.html +++ b/ui/templates/ui/v3.html @@ -30,6 +30,7 @@ diff --git a/ui/views.py b/ui/views.py index a5496b931..2cfd4b98e 100644 --- a/ui/views.py +++ b/ui/views.py @@ -56,7 +56,9 @@ def v3(request, *args, **kwargs): skip_authorization=True, ) - context = dict(client_id=ui_app.client_id) + context = dict( + client_id=ui_app.client_id, RAW_DATA_API_URL=settings.RAW_DATA_API_URL + ) if settings.MATOMO_URL is not None and settings.MATOMO_SITEID is not None: context.update( {"MATOMO_URL": settings.MATOMO_URL, "MATOMO_SITEID": settings.MATOMO_SITEID} diff --git a/ui/webpack.config.js b/ui/webpack.config.js index 858b0313e..ed8ec9e2b 100644 --- a/ui/webpack.config.js +++ b/ui/webpack.config.js @@ -93,7 +93,8 @@ const config = { plugins: [new webpack.DefinePlugin({ "process.env": { CLIENT_ID: JSON.stringify(process.env.CLIENT_ID), - EXPORTS_API_URL: JSON.stringify(process.env.EXPORTS_API_URL) + EXPORTS_API_URL: JSON.stringify(process.env.EXPORTS_API_URL), + RAW_DATA_API_URL: JSON.stringify(process.env.RAW_DATA_API_URL) } }), new webpack.NamedModulesPlugin(), new WriteFilePlugin()], resolve: { @@ -110,6 +111,7 @@ if (process.env.NODE_ENV === "production") { "process.env": { CLIENT_ID: JSON.stringify(process.env.CLIENT_ID), EXPORTS_API_URL: JSON.stringify(process.env.EXPORTS_API_URL), + RAW_DATA_API_URL: JSON.stringify(process.env.RAW_DATA_API_URL), NODE_ENV: JSON.stringify("production") } }), From da80e3b822d6b727c14c13a03a990db908e666b2 Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Wed, 7 Feb 2024 18:06:15 +0545 Subject: [PATCH 5/5] Added message for oauth upgrade --- ui/app/app.js | 2 ++ ui/app/components/Message.js | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 ui/app/components/Message.js diff --git a/ui/app/app.js b/ui/app/app.js index c350ffa4f..4ed34b0c0 100644 --- a/ui/app/app.js +++ b/ui/app/app.js @@ -25,6 +25,7 @@ import Home from "./components/Home"; import NavBar from "./components/NavBar"; import Stats from "./components/Stats"; import Banner from "./components/Banner"; +import Message from "./components/Message.js"; import { requireAuth } from "./components/utils"; import "@blueprintjs/core/dist/blueprint.css"; @@ -49,6 +50,7 @@ export default ({ history }) => {
+ diff --git a/ui/app/components/Message.js b/ui/app/components/Message.js new file mode 100644 index 000000000..9fb51f7e9 --- /dev/null +++ b/ui/app/components/Message.js @@ -0,0 +1,34 @@ +import React, { Component } from "react"; + +class Message extends Component { + constructor(props) { + super(props); + // Use "messageClosed" as the key to check if the message was previously closed + const messageClosed = localStorage.getItem("messageClosed") === "true"; + this.state = { + isVisible: !messageClosed, + }; + } + + handleClose = () => { + this.setState({ isVisible: false }); + // When closing, set "messageClosed" in localStorage to "true" + localStorage.setItem("messageClosed", "true"); + }; + + render() { + if (!this.state.isVisible) { + return null; + } + return ( +
+

We have recently upgraded from OAuth 1.0 to 2.0. Please Logout and Login again before use!

+ +
+ ); + } +} + +export default Message;