Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature : Data completeness stats #526

Merged
merged 5 commits into from
Feb 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion core/settings/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@
)

AUTHENTICATION_BACKENDS = (
"social_core.backends.openstreetmap.OpenStreetMapOAuth",
# "social_core.backends.openstreetmap.OpenStreetMapOAuth",
"social_core.backends.openstreetmap_oauth2.OpenStreetMapOAuth2",
"oauth2_provider.backends.OAuth2Backend",
"social_core.backends.email.EmailAuth",
"social_core.backends.username.UsernameAuth",
Expand Down
85 changes: 45 additions & 40 deletions core/settings/contrib.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,58 +8,63 @@
# Extra installed apps
INSTALLED_APPS += (
# any 3rd party apps
'rest_framework',
'rest_framework_gis',
'rest_framework.authtoken',
'social_django',
"rest_framework",
"rest_framework_gis",
"rest_framework.authtoken",
"social_django",
)

# 3rd party specific app settings
OAUTH2_PROVIDER = {
'ACCESS_TOKEN_EXPIRE_SECONDS': 10 * 365 * 24 * 60 * 60,
"ACCESS_TOKEN_EXPIRE_SECONDS": 10 * 365 * 24 * 60 * 60,
}

REST_FRAMEWORK = {
'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.SearchFilter',
'rest_framework.filters.OrderingFilter'),
'DEFAULT_AUTHENTICATION_CLASSES': ('rest_framework.authentication.TokenAuthentication',
'oauth2_provider.contrib.rest_framework.OAuth2Authentication',
'rest_framework.authentication.SessionAuthentication',),
'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAuthenticated',),
'DEFAULT_RENDERER_CLASSES': (
'rest_framework.renderers.JSONRenderer',
'api.renderers.HOTExportApiRenderer',
"DEFAULT_FILTER_BACKENDS": (
"rest_framework.filters.SearchFilter",
"rest_framework.filters.OrderingFilter",
),
'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.AcceptHeaderVersioning',
'DEFAULT_VERSION': '1.0',
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
'PAGE_SIZE': 20
"DEFAULT_AUTHENTICATION_CLASSES": (
"rest_framework.authentication.TokenAuthentication",
"oauth2_provider.contrib.rest_framework.OAuth2Authentication",
"rest_framework.authentication.SessionAuthentication",
),
"DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",),
"DEFAULT_RENDERER_CLASSES": (
"rest_framework.renderers.JSONRenderer",
"api.renderers.HOTExportApiRenderer",
),
"DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.AcceptHeaderVersioning",
"DEFAULT_VERSION": "1.0",
"DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination",
"PAGE_SIZE": 20,
}


# OAuth login settings
SOCIAL_AUTH_OPENSTREETMAP_LOGIN_URL = '/osm/login/'
SOCIAL_AUTH_OPENSTREETMAP_KEY = os.getenv('OSM_API_KEY')
SOCIAL_AUTH_OPENSTREETMAP_SECRET = os.getenv('OSM_API_SECRET')
SOCIAL_AUTH_LOGIN_REDIRECT_URL = '/'
SOCIAL_AUTH_LOGIN_ERROR_URL = '/osm/error'
SOCIAL_AUTH_URL_NAMESPACE = 'osm'
SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = ['username', 'first_name', 'email']
SOCIAL_AUTH_OPENSTREETMAP_LOGIN_URL = "/osm/login/"
SOCIAL_AUTH_OPENSTREETMAP_OAUTH2_KEY = os.getenv("OSM_API_KEY")
SOCIAL_AUTH_OPENSTREETMAP_OAUTH2_SECRET = os.getenv("OSM_API_SECRET")
SOCIAL_AUTH_LOGIN_REDIRECT_URL = "/"
SOCIAL_AUTH_LOGIN_ERROR_URL = "/osm/error"
SOCIAL_AUTH_URL_NAMESPACE = "osm"
SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = ["username", "first_name", "email"]
SOCIAL_AUTH_FORCE_EMAIL_VALIDATION = True
SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = 'ui.pipeline.email_validation'
SOCIAL_AUTH_EMAIL_VALIDATION_URL = '/osm/email_verify_sent/'
SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = "ui.pipeline.email_validation"
SOCIAL_AUTH_EMAIL_VALIDATION_URL = "/osm/email_verify_sent/"

SOCIAL_AUTH_PIPELINE = (
'social_core.pipeline.social_auth.social_details',
'social_core.pipeline.social_auth.social_uid',
'social_core.pipeline.social_auth.auth_allowed',
'social_core.pipeline.social_auth.social_user',
'social_core.pipeline.user.get_username',
'ui.pipeline.require_email',
'social_core.pipeline.mail.mail_validation',
'social_core.pipeline.social_auth.associate_by_email',
'social_core.pipeline.user.create_user',
'social_core.pipeline.social_auth.associate_user',
'social_core.pipeline.debug.debug',
'social_core.pipeline.social_auth.load_extra_data',
'social_core.pipeline.user.user_details'
"social_core.pipeline.social_auth.social_details",
"social_core.pipeline.social_auth.social_uid",
"social_core.pipeline.social_auth.auth_allowed",
"social_core.pipeline.social_auth.social_user",
"social_core.pipeline.user.get_username",
"ui.pipeline.require_email",
"social_core.pipeline.mail.mail_validation",
"social_core.pipeline.social_auth.associate_by_email",
"social_core.pipeline.user.create_user",
"social_core.pipeline.social_auth.associate_user",
"social_core.pipeline.debug.debug",
"social_core.pipeline.social_auth.load_extra_data",
"social_core.pipeline.user.user_details",
)
2 changes: 1 addition & 1 deletion docs/setup-development.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ Most of these environment variables have reasonable default settings.
- `GARMIN_CONFIG`, `GARMIN_MKGMAP` absolute paths to garmin JARs
- `OVERPASS_API_URL` url of Overpass api endpoint

- `RAW_DATA_API_URL` url of Galaxy api endpoint
- `RAW_DATA_API_URL` url of Raw data api endpoint

- `DATABASE_URL` Database URL. Defaults to `postgres:///exports`
- `DEBUG` Whether to enable debug mode. Defaults to `False` (production).
Expand Down
2 changes: 1 addition & 1 deletion ops/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ Examples of how to backup and restore the database:
psql exports < export_tool_2023-03-06.pgdump
```

8. Modify the OAuth1 application with your hostname's `redirect_uris`
8. Modify the OAuth2 application with your hostname's `redirect_uris`

### Storage and Environment Variables

Expand Down
6 changes: 3 additions & 3 deletions ops/systemd/export_workers.env
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ EMAIL_HOST_USER=<INSECURE_SMTP_USERNAME>
EMAIL_HOST_PASSWORD=<SECURE_SMTP_PASSWORD>
REPLY_TO_EMAIL=<INSECURE_REPLY_EMAIL>

##OAUTH 1 Settings
OSM_API_KEY=<SECURE_OSM_OAUTH1_APP_CONSUMER_KEY>
OSM_API_SECRET=<SECURE_OSM_OAUTH1_APP_CONSUMER_SECRET>
##OAUTH 2 Settings
OSM_API_KEY=<SECURE_OSM_OAUTH2_APP_CONSUMER_KEY>
OSM_API_SECRET=<SECURE_OSM_OAUTH2_APP_CONSUMER_SECRET>

## Workers
WORKER_SECRET_KEY=<SECURE_WORKER_KEY>
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ mercantile~=0.10.0
psycopg2
python3-openid==3.2.0
social-auth-app-django==5.4.0
social-auth-core==4.4.2
social-auth-core @ git+https://github.com/kshitijrajsharma/social-core.git ### Upgrade this to include osm oauth2 when released
pytz
pyyaml>=5.3
raven
Expand Down
4 changes: 2 additions & 2 deletions ui/app/actions/exports.js
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,8 @@ export const getOverpassTimestamp = () => (dispatch, getState) => {

export const getGalaxyTimestamp = () => (dispatch, getState) => {
return axios({
baseURL: "https://api-prod.raw-data.hotosm.org/v1",
url: "/status/"
baseURL: window.RAW_DATA_API_URL,
url: "v1/status/"
})
.then(response =>
dispatch({
Expand Down
5 changes: 5 additions & 0 deletions ui/app/actions/meta.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ if (window.location.port) {
hostname += `:${window.location.port}`;
}

if (window.RAW_DATA_API_URL == null) {
window.RAW_DATA_API_URL = process.env.RAW_DATA_API_URL;
}

if (window.EXPORTS_API_URL == null) {
window.EXPORTS_API_URL = process.env.EXPORTS_API_URL;

Expand All @@ -27,6 +31,7 @@ if (window.OAUTH_CLIENT_ID == null) {
}

const oauthConfig = {
// url: window.EXPORTS_API_URL + "/o/openstreetmap_oauth2",
url: window.EXPORTS_API_URL + "/o/authorize?approval_prompt=auto",
client: window.OAUTH_CLIENT_ID,
redirect: `${window.location.protocol}//${hostname}/authorized`
Expand Down
2 changes: 2 additions & 0 deletions ui/app/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import Home from "./components/Home";
import NavBar from "./components/NavBar";
import Stats from "./components/Stats";
import Banner from "./components/Banner";
import Message from "./components/Message.js";
import { requireAuth } from "./components/utils";

import "@blueprintjs/core/dist/blueprint.css";
Expand All @@ -49,6 +50,7 @@ export default ({ history }) => {
<div style={{ height: "100%" }}>
<Auth />
<NavBar />
<Message/>
<Banner />
<Switch>
<Route path="/authorized" component={Authorized} />
Expand Down
85 changes: 83 additions & 2 deletions ui/app/components/ExportForm.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import area from "@turf/area";
import axios from "axios";
import bbox from "@turf/bbox";
import React, { Component } from "react";
import { Col, Nav, Panel, Row } from "react-bootstrap";
Expand All @@ -8,7 +9,6 @@ import { Redirect, Route, Switch } from "react-router";
import { NavLink } from "react-router-dom";
import { Fields, formValueSelector, reduxForm } from "redux-form";
import { pointToTile } from "tilebelt";

import ChooseFormats from "./ChooseFormats";
import DescribeExport from "./DescribeExport";
import ExportAOIField from "./ExportAOIField";
Expand Down Expand Up @@ -128,6 +128,86 @@ export class ExportForm extends Component {
};
}

async fetchData(geometry) {
const url = window.RAW_DATA_API_URL + "v1/stats/polygon/";
try {
const response = await axios.post(url, {
geometry: geometry
}, {
headers: {"Content-Type": "application/json"}
});

if (response.data) {

this.setState({ fetchedInfo: response.data });
}
} catch (error) {
console.error("Failed to fetch summary data", error);

}
}

componentDidUpdate(prevProps) {
if (this.props.formValues.the_geom !== prevProps.formValues.the_geom) {
this.fetchData(this.props.formValues.the_geom);
}
}

renderFetchedInfo() {
const { fetchedInfo } = this.state;
if (!this.props.formValues.the_geom) return null;
if (!fetchedInfo) return null;

// Function to trigger the download of the raw data as a JSON file
const downloadRawData = () => {
const filename = "raw_region_summary.json";
const jsonStr = JSON.stringify(fetchedInfo, null, 4);
const element = document.createElement('a');
element.setAttribute('href', 'data:text/json;charset=utf-8,' + encodeURIComponent(jsonStr));
element.setAttribute('download', filename);
element.style.display = 'none';
document.body.appendChild(element);
element.click();
document.body.removeChild(element);
};

return (
<Panel style={{ marginTop: "10px" }}>
<div>
<div>
<strong style={{ fontSize: "smaller" }}>Buildings:</strong>
<p style={{ fontSize: "smaller", textAlign: "justify", margin: "10px 0" }}>
<FormattedMessage
id="export.dc.stats.buildings"
defaultMessage="{buildings}"
values={{ buildings: fetchedInfo.summary.buildings }}
/>
</p>
</div>
<div>
<strong style={{ fontSize: "smaller" }}>Roads:</strong>
<p style={{ fontSize: "smaller", textAlign: "justify", margin: "10px 0" }}>
<FormattedMessage
id="export.dc.stats.roads"
defaultMessage="{roads}"
values={{ roads: fetchedInfo.summary.roads }}
/>
</p>
</div>
<div style={{ fontSize: "smaller", marginTop: "10px" }}>
More info:
<a href="#" onClick={downloadRawData} style={{ marginLeft: "5px" }}>Download</a>,
<a href={fetchedInfo.meta.indicators} target="_blank" rel="noopener noreferrer" style={{ marginLeft: "5px" }}>Indicators</a>,
<a href={fetchedInfo.meta.metrics} target="_blank" rel="noopener noreferrer" style={{ marginLeft: "5px" }}>Metrics</a>
</div>
</div>
</Panel>
);
}




componentWillMount() {
const { getConfigurations, getOverpassTimestamp, getGalaxyTimestamp} = this.props;

Expand Down Expand Up @@ -281,10 +361,11 @@ export class ExportForm extends Component {
/>}
/>
</Switch>
{this.renderFetchedInfo()}
<Panel style={{ marginTop: "20px" }}>
<FormattedMessage
id="ui.overpass_last_updated"
defaultMessage="Img/pbf/obf/ updated {overpassLastUpdated}, Rest of other formats updated {galaxyLastUpdated} "
defaultMessage="Img/pbf/obf updated {overpassLastUpdated}, Rest of other formats updated {galaxyLastUpdated} "
values={{ overpassLastUpdated, galaxyLastUpdated }}
/>
</Panel>
Expand Down
34 changes: 34 additions & 0 deletions ui/app/components/Message.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import React, { Component } from "react";

class Message extends Component {
constructor(props) {
super(props);
// Use "messageClosed" as the key to check if the message was previously closed
const messageClosed = localStorage.getItem("messageClosed") === "true";
this.state = {
isVisible: !messageClosed,
};
}

handleClose = () => {
this.setState({ isVisible: false });
// When closing, set "messageClosed" in localStorage to "true"
localStorage.setItem("messageClosed", "true");
};

render() {
if (!this.state.isVisible) {
return null;
}
return (
<div className="banner" style={{ backgroundColor: "#ffcc00", color: "black", textAlign: "center", padding: "10px", position: "relative" }}>
<p>We have recently upgraded from OAuth 1.0 to 2.0. Please Logout and Login again before use!</p>
<button onClick={this.handleClose} style={{ position: "absolute", top: "5px", right: "10px", cursor: "pointer" }}>
×
</button>
</div>
);
}
}

export default Message;
2 changes: 1 addition & 1 deletion ui/app/components/help/API.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export default () =>
<p>User authentication and authorization is a two-step process.</p>
<p>
The Export Tool requires that users log into OpenStreetMap using{" "}
<a href="https://oauth.net/1/">OAuth 1.0a</a> (you don't need to know
<a href="https://oauth.net/2/">OAuth 2.0a</a> (you don't need to know
this). This provides user identity, specifically an OSM username to
associate with exports. You generally don't need to care about this,
except to know that usernames are the same as on OSM.
Expand Down
10 changes: 5 additions & 5 deletions ui/app/components/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ export const AVAILABLE_EXPORT_FORMATS = {
SQL <code>.sql</code>
</span>
),
mbtiles: (
<span key="mbtiles">
MBTiles <code>.mbtiles</code>
</span>
),
garmin_img: (
<span key="garmin_img">
Garmin <code>.img</code>
Expand All @@ -83,11 +88,6 @@ export const AVAILABLE_EXPORT_FORMATS = {
OsmAnd <code>.obf</code>
</span>
),
mbtiles: (
<span key="osmand_obf">
MBTiles <code>.mbtiles</code>
</span>
),
bundle: (
<span key="bundle">
<a href="http://posm.io/">POSM</a> bundle
Expand Down
2 changes: 1 addition & 1 deletion ui/templates/osm/email.html
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
{% blocktrans %}Please provide a valid email address. This email address will be used to notifiy you when your exports are ready.
A verification link will be sent to the email address you provide.{% endblocktrans %}
<br/><br/>
<form method="post" action="{% url 'osm:complete' 'openstreetmap' %}" id="registerEmail">
<form method="post" action="{% url 'osm:complete' 'openstreetmap-oauth2' %}" id="registerEmail">
{% csrf_token %}
<div class="form-group row" id="form-group-email">
<div class="col-md-12">
Expand Down
1 change: 1 addition & 0 deletions ui/templates/ui/v3.html
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
<script>
var EXPORTS_API_URL = "{{ request.scheme }}://{{ request.get_host }}";
var OAUTH_CLIENT_ID = "{{ client_id }}";
var RAW_DATA_API_URL = "{{ RAW_DATA_API_URL }}";
</script>
<script src="{% static 'ui/js/bundle.js' %}"></script>
</body>
Expand Down
Loading
Loading