From e654f23eddff2520c1cc20c8429dbdfe8b37caf7 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 14 Aug 2023 13:54:40 -0400 Subject: [PATCH 01/10] /scoring refactoring and add model-run viewer endpoints --- django/src/rdwatch_scoring/api.py | 10 +- django/src/rdwatch_scoring/models.py | 24 +-- django/src/rdwatch_scoring/views/model_run.py | 184 ++++++++++++++++++ django/src/rdwatch_scoring/views/performer.py | 32 +++ django/src/rdwatch_scoring/views/region.py | 33 ++++ .../{views.py => views/scores.py} | 2 +- vue/src/client/services/ApiService.ts | 6 +- 7 files changed, 273 insertions(+), 18 deletions(-) create mode 100644 django/src/rdwatch_scoring/views/model_run.py create mode 100644 django/src/rdwatch_scoring/views/performer.py create mode 100644 django/src/rdwatch_scoring/views/region.py rename django/src/rdwatch_scoring/{views.py => views/scores.py} (99%) diff --git a/django/src/rdwatch_scoring/api.py b/django/src/rdwatch_scoring/api.py index c48f7e897..891883997 100644 --- a/django/src/rdwatch_scoring/api.py +++ b/django/src/rdwatch_scoring/api.py @@ -1,7 +1,13 @@ from ninja import NinjaAPI -from .views import router as scores +from .views.model_run import router as modelruns +from .views.performer import router as performers +from .views.region import router as regions +from .views.scores import router as scores api = NinjaAPI(urls_namespace='scoring') -api.add_router('/scores/', scores) +api.add_router('/scoring/scores/', scores) +api.add_router('/scoring/performers/', performers) +api.add_router('/scoring/regions/', regions) +api.add_router('/scoring/model-runs/', modelruns) diff --git a/django/src/rdwatch_scoring/models.py b/django/src/rdwatch_scoring/models.py index 744d85466..5fa471072 100644 --- a/django/src/rdwatch_scoring/models.py +++ b/django/src/rdwatch_scoring/models.py @@ -48,7 +48,7 @@ class Addrfeat(models.Model): rtotyp = models.CharField(max_length=1, blank=True, null=True) offsetl = models.CharField(max_length=1, blank=True, null=True) offsetr = models.CharField(max_length=1, blank=True, null=True) - the_geom = models.LineStringField(srid=4269, blank=True, null=True) + the_geom = models.LineStringField(srid=4326, blank=True, null=True) class Meta: managed = False @@ -79,7 +79,7 @@ class Bg(models.Model): awater = models.FloatField(blank=True, null=True) intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) class Meta: managed = False @@ -106,7 +106,7 @@ class County(models.Model): awater = models.FloatField(blank=True, null=True) intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) class Meta: managed = False @@ -162,7 +162,7 @@ class Cousub(models.Model): awater = models.DecimalField(max_digits=14, decimal_places=0, blank=True, null=True) intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) class Meta: managed = False @@ -213,7 +213,7 @@ class Edges(models.Model): offsetr = models.CharField(max_length=1, blank=True, null=True) tnidf = models.DecimalField(max_digits=10, decimal_places=0, blank=True, null=True) tnidt = models.DecimalField(max_digits=10, decimal_places=0, blank=True, null=True) - the_geom = models.MultiLineStringField(srid=4269, blank=True, null=True) + the_geom = models.MultiLineStringField(srid=4326, blank=True, null=True) class Meta: managed = False @@ -513,7 +513,7 @@ class Faces(models.Model): atotal = models.FloatField(blank=True, null=True) intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) tractce20 = models.CharField(max_length=6, blank=True, null=True) blkgrpce20 = models.CharField(max_length=1, blank=True, null=True) blockce20 = models.CharField(max_length=4, blank=True, null=True) @@ -754,7 +754,7 @@ class Place(models.Model): awater = models.BigIntegerField(blank=True, null=True) intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) class Meta: managed = False @@ -851,7 +851,7 @@ class State(models.Model): awater = models.BigIntegerField(blank=True, null=True) intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) class Meta: managed = False @@ -898,7 +898,7 @@ class Tabblock(models.Model): awater = models.FloatField(blank=True, null=True) intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) class Meta: managed = False @@ -922,7 +922,7 @@ class Tabblock20(models.Model): awater = models.FloatField(blank=True, null=True) intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) housing = models.FloatField(blank=True, null=True) pop = models.FloatField(blank=True, null=True) @@ -958,7 +958,7 @@ class Tract(models.Model): awater = models.FloatField(blank=True, null=True) intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) class Meta: managed = False @@ -978,7 +978,7 @@ class Zcta5(models.Model): intptlat = models.CharField(max_length=11, blank=True, null=True) intptlon = models.CharField(max_length=12, blank=True, null=True) partflg = models.CharField(max_length=1, blank=True, null=True) - the_geom = models.MultiPolygonField(srid=4269, blank=True, null=True) + the_geom = models.MultiPolygonField(srid=4326, blank=True, null=True) class Meta: managed = False diff --git a/django/src/rdwatch_scoring/views/model_run.py b/django/src/rdwatch_scoring/views/model_run.py new file mode 100644 index 000000000..2691efce5 --- /dev/null +++ b/django/src/rdwatch_scoring/views/model_run.py @@ -0,0 +1,184 @@ +from datetime import datetime, timedelta + +import iso3166 +from celery.result import AsyncResult +from ninja import Field, FilterSchema, Query, Schema +from ninja.errors import ValidationError +from ninja.pagination import RouterPaginated +from ninja.schema import validator +from pydantic import constr # type: ignore + +from django.contrib.postgres.aggregates import JSONBAgg +from django.db import transaction +from django.db.models import ( + Avg, + Case, + Count, + CharField, + F, + Value, + Func, + JSONField, + Max, + Min, + OuterRef, + Q, + Subquery, + Aggregate, + When, +) +from django.db.models.functions import Coalesce, JSONObject, Concat +from django.http import Http404, HttpRequest +from django.shortcuts import get_object_or_404 +from django.contrib.gis.db.models.fields import PolygonField +from django.contrib.gis.db.models.functions import AsGeoJSON, Envelope, Transform + +from rdwatch.db.functions import ( + AggregateArraySubquery, + ExtractEpoch, +) +from rdwatch.models import ( + HyperParameters, + Region, + SatelliteFetching, + SiteEvaluation, + lookups, +) +from rdwatch.schemas import RegionModel, SiteModel +from rdwatch.schemas.common import TimeRangeSchema +from rdwatch.views.performer import PerformerSchema +from rdwatch.views.region import RegionSchema +from rdwatch.views.site_observation import get_site_observation_images +from django.db.models.functions import Cast, NullIf + +from rdwatch_scoring.models import EvaluationRun + +from rdwatch.db.functions import ExtractEpoch + +router = RouterPaginated() + +class TimeRangeJSON(NullIf): + """Represents the min/max time of a field as JSON""" + + def __init__(self, min, max, performer, site_originator): + json = JSONObject( + min=ExtractEpoch(Min(min, filter=F(performer) == F(site_originator))), + max=ExtractEpoch(Max(max, filter=F(performer) == F(site_originator))), + ) + null = Value({'min': None, 'max': None}, output_field=JSONField()) + return super().__init__(json, null) + +class ModelRunFilterSchema(FilterSchema): + performer: str | None + region: str | None + +class BoundingBoxPolygon(Aggregate): + """Gets the WGS-84 bounding box of a geometry stored in Web Mercator coordinates""" + + template = 'ST_Extent(%(expressions)s)' + arity = 1 + output_field = PolygonField() # type: ignore + + +class BoundingBoxGeoJSON(Cast): + """Gets the GeoJSON bounding box of a geometry in Web Mercator coordinates""" + + def __init__(self, field): + bbox = BoundingBoxPolygon(field) + json_str = AsGeoJSON(bbox) + return super().__init__(json_str, JSONField()) + + +class HyperParametersDetailSchema(Schema): + id: str + title: str + region: str | None = None + performer: str + # parameters: dict + numsites: int + # downloading: int | None = None + # score: float | None = None + timestamp: int | None = None + timerange: TimeRangeSchema | None = None + bbox: dict | None + # created: datetime + expiration_time: str | None = None + evaluation: int | None = None + evaluation_run: int | None = None + + +class EvaluationListSchema(Schema): + id: int + siteNumber: int + region: RegionSchema | None + + +class HyperParametersListSchema(Schema): + count: int + timerange: TimeRangeSchema | None = None + bbox: dict | None = None + results: list[HyperParametersDetailSchema] + + +def get_queryset(): + return ( + EvaluationRun.objects.select_related('site', 'observation') + .order_by('start_datetime',) + .annotate( + json=JSONObject( + id='pk', + title=Concat( + F('performer'), Value('_'), + F('region'), Value('_'), + F('evaluation_number'), Value('_'), + F('evaluation_run_number'), + output_field=CharField() + ), + performer='performer', + region='region', + numsites=Count('site', filter=F('performer') == F('site__originator')), + evaluation='evaluation_number', + evaluation_run='evaluation_run_number', + timerange=TimeRangeJSON('site__start_date', 'site__end_date', 'performer', 'site_originator'), + timestamp=ExtractEpoch('start_datetime'), + # timerange=TimeRangeJSON('evaluations__observations__timestamp'), + bbox=BoundingBoxGeoJSON('site__observation__geometry'), + ) + ) + ) + + +@router.get('/', response={200: HyperParametersListSchema}) +def list_model_runs( + request: HttpRequest, + filters: ModelRunFilterSchema = Query(...), # noqa: B008 + limit: int = 25, + page: int = 1, +): + queryset = get_queryset() + queryset = filters.filter(queryset=queryset) + + if page < 1 or (not limit and page != 1): + raise ValidationError(f"Invalid page '{page}'") + + # Calculate total number of model runs prior to paginating queryset + total_model_run_count = queryset.count() + + subquery = queryset[(page - 1) * limit: page * limit] if limit else queryset + aggregate = queryset.defer('json').aggregate( + timerange=TimeRangeJSON('site__start_date', 'site__end_date', 'performer', 'site_originator'), + results=AggregateArraySubquery(subquery.values('json')), + ) + + aggregate['count'] = total_model_run_count + + if filters.region is not None: + aggregate |= queryset.defer('json').aggregate( + # Use the region polygon for the bbox if it exists. + # Otherwise, fall back on the site polygon. + bbox=BoundingBoxGeoJSON('site__observation__geometry'), + ) + + return 200, aggregate + + diff --git a/django/src/rdwatch_scoring/views/performer.py b/django/src/rdwatch_scoring/views/performer.py new file mode 100644 index 000000000..b5de9228a --- /dev/null +++ b/django/src/rdwatch_scoring/views/performer.py @@ -0,0 +1,32 @@ +from ninja import Schema +from ninja.pagination import RouterPaginated + +from django.db.models import F +from django.http import HttpRequest +from django.shortcuts import get_object_or_404 + +from rdwatch.models.lookups import Performer +from rdwatch_scoring.models import EvaluationRun + +class PerformerSchema(Schema): + id: int + team_name: str + short_code: str + + +router = RouterPaginated() + + +@router.get('/', response=list[PerformerSchema]) +def list_performers(request: HttpRequest): + unique_performers = EvaluationRun.objects.order_by().values_list('performer', flat=True).distinct() + performers_list = [ + { + 'id': idx + 1, + 'team_name': performer, + 'short_code': performer.lower().replace(' ', '_') + } + for idx, performer in enumerate(unique_performers) + ] + return performers_list + diff --git a/django/src/rdwatch_scoring/views/region.py b/django/src/rdwatch_scoring/views/region.py new file mode 100644 index 000000000..c056fe7ff --- /dev/null +++ b/django/src/rdwatch_scoring/views/region.py @@ -0,0 +1,33 @@ +import iso3166 +from ninja import Schema +from ninja.pagination import RouterPaginated +from rdwatch_scoring.models import EvaluationRun + +from django.http import HttpRequest +from django.shortcuts import get_object_or_404 + +from rdwatch.models import Region + + +class RegionSchema(Schema): + id: int + name: str + + +router = RouterPaginated() + + +@router.get('/', response=list[RegionSchema]) +def list_regions(request: HttpRequest): + unique_regions = EvaluationRun.objects.order_by().values_list('region', flat=True).distinct() + + region_list = [ + { + 'id': idx + 1, + 'name': region + } + for idx, region in enumerate(unique_regions) + ] + + return region_list + diff --git a/django/src/rdwatch_scoring/views.py b/django/src/rdwatch_scoring/views/scores.py similarity index 99% rename from django/src/rdwatch_scoring/views.py rename to django/src/rdwatch_scoring/views/scores.py index 91a1a6554..5a92c87e8 100644 --- a/django/src/rdwatch_scoring/views.py +++ b/django/src/rdwatch_scoring/views/scores.py @@ -11,7 +11,7 @@ from rdwatch.models import HyperParameters, Region from rdwatch.views.region import RegionSchema -from .models import ( +from ..models import ( EvaluationActivityClassificationTemporalIou, EvaluationBroadAreaSearchDetection, EvaluationRun, diff --git a/vue/src/client/services/ApiService.ts b/vue/src/client/services/ApiService.ts index 3d35b42a7..8bc70db88 100644 --- a/vue/src/client/services/ApiService.ts +++ b/vue/src/client/services/ApiService.ts @@ -376,7 +376,7 @@ export class ApiService { { return __request(OpenAPI, { method: "GET", - url: "/api/scores/details", + url: "/api/scoring/scores/details", query: { configurationId, regionId, siteNumber, version }, }); } @@ -388,7 +388,7 @@ export class ApiService { { return __request(OpenAPI, { method: "GET", - url: "/api/scores/has-scores", + url: "/api/scoring/scores/has-scores", query: { configurationId, regionId }, }); } @@ -399,7 +399,7 @@ export class ApiService { { return __request(OpenAPI, { method: "GET", - url: "/api/scores/region-colors", + url: "/api/scoring/scores/region-colors", query: { configurationId, regionId }, }); } From d8a95ca8730df66ac3029e6388622d66c884d0c3 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 15 Aug 2023 09:40:03 -0400 Subject: [PATCH 02/10] add front-end for visualizing scoring run list --- django/src/rdwatch_scoring/views/model_run.py | 104 +++++++----------- django/src/rdwatch_scoring/views/performer.py | 11 +- django/src/rdwatch_scoring/views/region.py | 16 +-- scripts/scoreMultiple.py | 79 +++++++++++++ vue/src/client/models/ModelRun.ts | 2 +- vue/src/client/services/ApiService.ts | 39 ++++--- vue/src/components/ModelRunList.vue | 11 +- vue/src/components/filters/RegionFilter.vue | 14 ++- vue/src/router.ts | 17 ++- 9 files changed, 187 insertions(+), 106 deletions(-) create mode 100644 scripts/scoreMultiple.py diff --git a/django/src/rdwatch_scoring/views/model_run.py b/django/src/rdwatch_scoring/views/model_run.py index 2691efce5..032a6b24e 100644 --- a/django/src/rdwatch_scoring/views/model_run.py +++ b/django/src/rdwatch_scoring/views/model_run.py @@ -1,62 +1,22 @@ -from datetime import datetime, timedelta - -import iso3166 -from celery.result import AsyncResult -from ninja import Field, FilterSchema, Query, Schema +from ninja import FilterSchema, Query, Schema from ninja.errors import ValidationError from ninja.pagination import RouterPaginated -from ninja.schema import validator -from pydantic import constr # type: ignore - -from django.contrib.postgres.aggregates import JSONBAgg -from django.db import transaction -from django.db.models import ( - Avg, - Case, - Count, - CharField, - F, - Value, - Func, - JSONField, - Max, - Min, - OuterRef, - Q, - Subquery, - Aggregate, - When, -) -from django.db.models.functions import Coalesce, JSONObject, Concat -from django.http import Http404, HttpRequest -from django.shortcuts import get_object_or_404 + from django.contrib.gis.db.models.fields import PolygonField -from django.contrib.gis.db.models.functions import AsGeoJSON, Envelope, Transform - -from rdwatch.db.functions import ( - AggregateArraySubquery, - ExtractEpoch, -) -from rdwatch.models import ( - HyperParameters, - Region, - SatelliteFetching, - SiteEvaluation, - lookups, -) -from rdwatch.schemas import RegionModel, SiteModel +from django.contrib.gis.db.models.functions import AsGeoJSON +from django.db.models import Aggregate, CharField, Count, F, JSONField, Max, Min, Value +from django.db.models.functions import Cast, Concat, JSONObject, NullIf, Upper +from django.http import HttpRequest + +from rdwatch.db.functions import AggregateArraySubquery, ExtractEpoch from rdwatch.schemas.common import TimeRangeSchema from rdwatch.views.performer import PerformerSchema from rdwatch.views.region import RegionSchema -from rdwatch.views.site_observation import get_site_observation_images -from django.db.models.functions import Cast, NullIf - from rdwatch_scoring.models import EvaluationRun -from rdwatch.db.functions import ExtractEpoch - router = RouterPaginated() + class TimeRangeJSON(NullIf): """Represents the min/max time of a field as JSON""" @@ -68,10 +28,12 @@ def __init__(self, min, max, performer, site_originator): null = Value({'min': None, 'max': None}, output_field=JSONField()) return super().__init__(json, null) + class ModelRunFilterSchema(FilterSchema): - performer: str | None + performer: str | None region: str | None + class BoundingBoxPolygon(Aggregate): """Gets the WGS-84 bounding box of a geometry stored in Web Mercator coordinates""" @@ -92,15 +54,16 @@ def __init__(self, field): class HyperParametersDetailSchema(Schema): id: str title: str - region: str | None = None - performer: str + region: RegionSchema | None = None + performer: PerformerSchema # parameters: dict numsites: int # downloading: int | None = None - # score: float | None = None + score: float | None = None timestamp: int | None = None timerange: TimeRangeSchema | None = None bbox: dict | None + ground_truth: bool # created: datetime expiration_time: str | None = None evaluation: int | None = None @@ -123,24 +86,35 @@ class HyperParametersListSchema(Schema): def get_queryset(): return ( EvaluationRun.objects.select_related('site', 'observation') - .order_by('start_datetime',) + .order_by( + 'start_datetime', + ) .annotate( json=JSONObject( id='pk', title=Concat( - F('performer'), Value('_'), - F('region'), Value('_'), - F('evaluation_number'), Value('_'), + F('performer'), + Value('_'), + F('region'), + Value('_'), + F('evaluation_number'), + Value('_'), F('evaluation_run_number'), - output_field=CharField() + output_field=CharField(), + ), + performer=JSONObject( + id=Value(-1), team_name='performer', short_code=Upper('performer') ), - performer='performer', - region='region', + region=JSONObject(id=Value(-1), name='region'), + score=None, numsites=Count('site', filter=F('performer') == F('site__originator')), evaluation='evaluation_number', evaluation_run='evaluation_run_number', - timerange=TimeRangeJSON('site__start_date', 'site__end_date', 'performer', 'site_originator'), + timerange=TimeRangeJSON( + 'site__start_date', 'site__end_date', 'performer', 'site_originator' + ), timestamp=ExtractEpoch('start_datetime'), + ground_truth=False, # timerange=TimeRangeJSON('evaluations__observations__timestamp'), bbox=BoundingBoxGeoJSON('site__observation__geometry'), ) @@ -164,9 +138,11 @@ def list_model_runs( # Calculate total number of model runs prior to paginating queryset total_model_run_count = queryset.count() - subquery = queryset[(page - 1) * limit: page * limit] if limit else queryset + subquery = queryset[(page - 1) * limit : page * limit] if limit else queryset aggregate = queryset.defer('json').aggregate( - timerange=TimeRangeJSON('site__start_date', 'site__end_date', 'performer', 'site_originator'), + timerange=TimeRangeJSON( + 'site__start_date', 'site__end_date', 'performer', 'site_originator' + ), results=AggregateArraySubquery(subquery.values('json')), ) @@ -180,5 +156,3 @@ def list_model_runs( ) return 200, aggregate - - diff --git a/django/src/rdwatch_scoring/views/performer.py b/django/src/rdwatch_scoring/views/performer.py index b5de9228a..4be23c5cd 100644 --- a/django/src/rdwatch_scoring/views/performer.py +++ b/django/src/rdwatch_scoring/views/performer.py @@ -1,13 +1,11 @@ from ninja import Schema from ninja.pagination import RouterPaginated -from django.db.models import F from django.http import HttpRequest -from django.shortcuts import get_object_or_404 -from rdwatch.models.lookups import Performer from rdwatch_scoring.models import EvaluationRun + class PerformerSchema(Schema): id: int team_name: str @@ -19,14 +17,15 @@ class PerformerSchema(Schema): @router.get('/', response=list[PerformerSchema]) def list_performers(request: HttpRequest): - unique_performers = EvaluationRun.objects.order_by().values_list('performer', flat=True).distinct() + unique_performers = ( + EvaluationRun.objects.order_by().values_list('performer', flat=True).distinct() + ) performers_list = [ { 'id': idx + 1, 'team_name': performer, - 'short_code': performer.lower().replace(' ', '_') + 'short_code': performer.lower().replace(' ', '_'), } for idx, performer in enumerate(unique_performers) ] return performers_list - diff --git a/django/src/rdwatch_scoring/views/region.py b/django/src/rdwatch_scoring/views/region.py index c056fe7ff..4ef679f89 100644 --- a/django/src/rdwatch_scoring/views/region.py +++ b/django/src/rdwatch_scoring/views/region.py @@ -1,12 +1,9 @@ -import iso3166 from ninja import Schema from ninja.pagination import RouterPaginated -from rdwatch_scoring.models import EvaluationRun from django.http import HttpRequest -from django.shortcuts import get_object_or_404 -from rdwatch.models import Region +from rdwatch_scoring.models import EvaluationRun class RegionSchema(Schema): @@ -19,15 +16,12 @@ class RegionSchema(Schema): @router.get('/', response=list[RegionSchema]) def list_regions(request: HttpRequest): - unique_regions = EvaluationRun.objects.order_by().values_list('region', flat=True).distinct() + unique_regions = ( + EvaluationRun.objects.order_by().values_list('region', flat=True).distinct() + ) region_list = [ - { - 'id': idx + 1, - 'name': region - } - for idx, region in enumerate(unique_regions) + {'id': idx + 1, 'name': region} for idx, region in enumerate(unique_regions) ] return region_list - diff --git a/scripts/scoreMultiple.py b/scripts/scoreMultiple.py new file mode 100644 index 000000000..b0dc7259a --- /dev/null +++ b/scripts/scoreMultiple.py @@ -0,0 +1,79 @@ +import os +import argparse +import subprocess +# script allows for scoring/uploading multiple scores when there is a root folder +# and each sub folder has a site_models and a region_models folder. +# it is intended to be run at a root location where the scoring folder exists, the annotations folder +# exists for ground truth and the metrics-and-test-framework folder exists as well. + +database_URI = 'postgresql+psycopg2://scoring:secretkey@localhost:5433/scoring' +ground_truth = '../annotations/site_models' + +def main(args): + folder_path = args.folder + performer = args.performer + eval_run = args.eval_run + eval_run_num = args.eval_run_num + + commands = [] + + for root, _, _ in os.walk(folder_path): + site_models_path = os.path.join(root, 'site_models') + region_models_path = os.path.join(root, 'region_models') + + if os.path.isdir(site_models_path): + geojson_files = [file for file in os.listdir(site_models_path) if file.endswith('.geojson')] + + if geojson_files: + first_geojson = geojson_files[0] + identifier = first_geojson[:7] + module_path = os.path.join('iarpa_smart_metrics', 'run_evaluation') + + subprocess_args = [ + 'python', '-m', 'iarpa_smart_metrics.run_evaluation', + '--roi', identifier, + '--gt_dir', ground_truth, + '--rm_dir', region_models_path.replace('./', '../'), + '--sm_dir', site_models_path.replace('./', '../'), + '--output_dir', f'../{identifier}/output', + '--eval_num', str(eval_run), + '--eval_run_num', str(eval_run_num), + '--performer', performer, + '--no-viz', + '--no-viz-detection-table', + '--no-viz-comparison-table', + '--no-viz-associate-metrics', + '--no-viz-activity-metrics', + '--sequestered_id', identifier, + '--db_conn_str', database_URI, + ] + + if args.output_commands: # Check if the flag is provided + command_string = ' '.join(subprocess_args) + commands.append(command_string) + else: + process = subprocess.Popen(subprocess_args, cwd='./metrics-and-test-framework', stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + stdout, stderr = process.communicate() + + # Print subprocess output to script's stdout + print("Subprocess stdout:") + print(stdout) + print("Subprocess stderr:") + print(stderr) + + if args.output_commands: + # Save the formatted commands to the file + with open('scoring_commands.txt', 'w') as f: + for idx, cmd in enumerate(commands, start=1): + f.write(f"Command {idx}:\n{cmd}\n{'='*40}\n") + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Process folders and run evaluation subprocess.') + parser.add_argument('--folder', type=str, help='Main folder path') + parser.add_argument('--performer', type=str, help='Performer name') + parser.add_argument('--eval_run', type=int, help='Evaluation run number') + parser.add_argument('--eval_run_num', type=int, help='Evaluation run number') + parser.add_argument('--output_commands', action='store_true', help='Output commands instead of executing subprocesses') + args = parser.parse_args() + + main(args) diff --git a/vue/src/client/models/ModelRun.ts b/vue/src/client/models/ModelRun.ts index da8a05053..56239af8d 100755 --- a/vue/src/client/models/ModelRun.ts +++ b/vue/src/client/models/ModelRun.ts @@ -7,7 +7,7 @@ export type ModelRun = { title: string; performer: Performer; // eslint-disable-next-line @typescript-eslint/no-explicit-any - parameters: Record; + parameters?: Record; numsites: number; downloading: number; score: number | null; diff --git a/vue/src/client/services/ApiService.ts b/vue/src/client/services/ApiService.ts index 8bc70db88..a02854177 100644 --- a/vue/src/client/services/ApiService.ts +++ b/vue/src/client/services/ApiService.ts @@ -57,6 +57,11 @@ export interface ModelRunEvaluations { } export class ApiService { + private static apiPrefix = "/api/scoring"; + + public static setApiPrefix(prefix: string) { + ApiService.apiPrefix = prefix; + } /** * @returns ServerStatus * @throws ApiError @@ -77,7 +82,7 @@ export class ApiService { ): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/evaluations", + url: `${this.apiPrefix}/evaluations`, query, }); } @@ -92,7 +97,7 @@ export class ApiService { ): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/evaluations/{id}", + url:`${this.apiPrefix}/evaluations/{id}`, path: { id: id, }, @@ -110,7 +115,7 @@ export class ApiService { ): CancelablePromise { return __request(OpenAPI, { method: "POST", - url: "/api/observations/{id}/generate-images", + url: `${this.apiPrefix}observations/{id}/generate-images`, path: { id: id, }, @@ -131,7 +136,7 @@ export class ApiService { ): CancelablePromise { return __request(OpenAPI, { method: "POST", - url: "/api/model-runs/{id}/generate-images/", + url: `${this.apiPrefix}/model-runs/{id}/generate-images/`, path: { id: id, }, @@ -151,7 +156,7 @@ export class ApiService { ): CancelablePromise { return __request(OpenAPI, { method: "PUT", - url: "/api/observations/{id}/cancel-generate-images/", + url: `${this.apiPrefix}/observations/{id}/cancel-generate-images/`, path: { id: id, }, @@ -168,7 +173,7 @@ export class ApiService { ): CancelablePromise { return __request(OpenAPI, { method: "PUT", - url: "/api/model-runs/{id}/cancel-generate-images/", + url: `${this.apiPrefix}/model-runs/{id}/cancel-generate-images/`, path: { id: id, }, @@ -184,7 +189,7 @@ export class ApiService { ): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/model-runs", + url: `${this.apiPrefix}/model-runs`, query: Object.fromEntries( Object.entries(query).filter(([key, value]) => value !== undefined) ), @@ -198,7 +203,7 @@ export class ApiService { public static getModelRunEvaluations(id: number): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/model-runs/{id}/evaluations", + url: `${this.apiPrefix}/model-runs/{id}/evaluations`, path: { id: id, }, @@ -215,7 +220,7 @@ export class ApiService { public static getModelRun(id: number): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/model-runs/{id}", + url: `${this.apiPrefix}/model-runs/{id}`, path: { id: id, }, @@ -229,7 +234,7 @@ export class ApiService { public static getPerformers(): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/performers", + url: `${this.apiPrefix}/performers`, }); } @@ -241,7 +246,7 @@ export class ApiService { public static getPerformer(id: number): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/performers/{id}", + url: `${this.apiPrefix}/performers/{id}`, path: { id: id, }, @@ -255,7 +260,7 @@ export class ApiService { public static getRegions(): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/regions", + url: `${this.apiPrefix}/regions`, }); } @@ -267,7 +272,7 @@ export class ApiService { public static getRegion(id: number): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/regions/{id}", + url: `${this.apiPrefix}/regions/{id}`, path: { id: id, }, @@ -299,7 +304,7 @@ export class ApiService { const bboxstr = `${minY},${minX},${maxY},${maxX}`;; return __request(OpenAPI, { method: "GET", - url: "/api/satellite-image/timestamps", + url: `api/satellite-image/timestamps`, query: { constellation, level, spectrum, start_timestamp: startTime, end_timestamp: endTime, bbox: bboxstr, } }); @@ -331,7 +336,7 @@ export class ApiService { const bboxstr = `${minY},${minX},${maxY},${maxX}`; return __request(OpenAPI, { method: "GET", - url: "/api/satellite-image/visual-timestamps", + url: `api/satellite-image/visual-timestamps`, query: { constellation, level, spectrum, start_timestamp: startTime, end_timestamp: endTime, bbox: bboxstr, } }); @@ -361,7 +366,7 @@ export class ApiService { const bboxstr = `${minY},${minX},${maxY},${maxX}`; return __request(OpenAPI, { method: "GET", - url: "/api/satellite-image/all-timestamps", + url: `api/satellite-image/all-timestamps`, query: { constellation, level, spectrum, start_timestamp: startTime, end_timestamp: endTime, bbox: bboxstr, } }); @@ -407,7 +412,7 @@ export class ApiService { public static getEvaluationImages(id: number): CancelablePromise { return __request(OpenAPI, { method: "GET", - url: "/api/evaluations/images/{id}", + url: `${this.apiPrefix}/evaluations/images/{id}`, path: { id: id, }, diff --git a/vue/src/components/ModelRunList.vue b/vue/src/components/ModelRunList.vue index dccbd93fa..b7d9a8aac 100755 --- a/vue/src/components/ModelRunList.vue +++ b/vue/src/components/ModelRunList.vue @@ -57,8 +57,9 @@ async function loadMore() { totalModelRuns.value = modelRunList.count; // sort list to show ground truth near the top + console.log(modelRunList); const modelRunResults = modelRunList.results.sort((a, b) => - b.parameters["ground_truth"] === true ? 1 : -1 + b.parameters && b.parameters["ground_truth"] === true ? 1 : -1 ); const keyedModelRunResults = modelRunResults.map((val, i) => { return { @@ -224,8 +225,14 @@ function handleToggle(modelRun: KeyedModelRun) { .filter((modelRun) => state.openedModelRuns.has(modelRun.key)) .map((modelRun) => { configurationIds.add(modelRun.id); - if (modelRun.region) { + if (modelRun.region && modelRun.region.id !== -1) { regionIds.add(modelRun.region?.id); + } else if (modelRun.region) { + const found = Object.entries(state.regionMap).find(([,name]) => name === modelRun.region?.name); + if (found) { + regionIds.add(parseInt(found[0], 10)); + } + } }); state.filters = { diff --git a/vue/src/components/filters/RegionFilter.vue b/vue/src/components/filters/RegionFilter.vue index 9e7500c5c..ebdbcc038 100755 --- a/vue/src/components/filters/RegionFilter.vue +++ b/vue/src/components/filters/RegionFilter.vue @@ -27,8 +27,14 @@ watchEffect(async () => { regions.value = regionResults; regionItems.value = regionResults.map((item) => ({title: item.name, value: item.id})); const generatedMap: Record = {} + let counter = 0; regionResults.forEach((item) => { + if (item.id === -1) { + generatedMap[counter] = item.name; + counter += 1; + } else { generatedMap[item.id] = item.name; + } }) state.regionMap = generatedMap; }); @@ -40,9 +46,13 @@ watch(() => props.modelValue, () => { }); watch(selectedRegion, (val) => { - let prepend = '/' + let prepend = '' + if (router.currentRoute.value.fullPath.includes('scoring')) { + prepend = '/scoring/' + } + if (router.currentRoute.value.fullPath.includes('annotation')) { - prepend='/annotation/' + prepend=`${prepend}/annotation/` } if (val !== undefined) { const found = regions.value.find((item) => item.id === val); diff --git a/vue/src/router.ts b/vue/src/router.ts index 6b236d3cf..667d3de77 100644 --- a/vue/src/router.ts +++ b/vue/src/router.ts @@ -2,10 +2,13 @@ import { createRouter, createWebHashHistory } from "vue-router"; import RGD from './views/RGD.vue'; import Annotation from './views/AnnotationViewer.vue'; +import { ApiService } from './client/services/ApiService'; // Import your ApiService implementation const routes = [ - { path: '/:region?/:selected?', component: RGD, props:true, }, - { path: '/annotation/:region?/:selected?', component: Annotation, props:true, }, + { path: '/:region?/:selected?', component: RGD, props:true, }, + { path: '/annotation/:region?/:selected?', component: Annotation, props:true, }, + { path: '/scoring/:region?/:selected?', component: RGD, props:true, }, + { path: '/scoring/annotation/:region?/:selected?', component: Annotation, props:true, }, ] const router = createRouter({ @@ -14,4 +17,14 @@ const router = createRouter({ routes, // short for `routes: routes` }) +router.beforeEach((to, from, next) => { + // Check if the current route has a prefix '/scoring' + const isScoringRoute = to.path.startsWith('/scoring'); + + // Set the ApiService prefix URL accordingly + ApiService.setApiPrefix(isScoringRoute ? "/api/scoring" : "/api"); + + next(); // Continue with the navigation + }); + export default router; \ No newline at end of file From 9d9992f9552f5186ecefa3b9da443952130e5f60 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Wed, 16 Aug 2023 13:36:42 -0400 Subject: [PATCH 03/10] count unique site_ids and not total observations --- django/src/rdwatch_scoring/views/model_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/src/rdwatch_scoring/views/model_run.py b/django/src/rdwatch_scoring/views/model_run.py index 032a6b24e..d16cd5999 100644 --- a/django/src/rdwatch_scoring/views/model_run.py +++ b/django/src/rdwatch_scoring/views/model_run.py @@ -107,7 +107,7 @@ def get_queryset(): ), region=JSONObject(id=Value(-1), name='region'), score=None, - numsites=Count('site', filter=F('performer') == F('site__originator')), + numsites=Count('site__site_id', filter=F('performer') == F('site__originator'), distinct=True), evaluation='evaluation_number', evaluation_run='evaluation_run_number', timerange=TimeRangeJSON( From 5889c37578e3721c39b0e5c05c1e8935b59e76f4 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 22 Aug 2023 09:58:54 -0400 Subject: [PATCH 04/10] linting --- django/src/rdwatch_scoring/views/model_run.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django/src/rdwatch_scoring/views/model_run.py b/django/src/rdwatch_scoring/views/model_run.py index d16cd5999..71f23708f 100644 --- a/django/src/rdwatch_scoring/views/model_run.py +++ b/django/src/rdwatch_scoring/views/model_run.py @@ -107,7 +107,11 @@ def get_queryset(): ), region=JSONObject(id=Value(-1), name='region'), score=None, - numsites=Count('site__site_id', filter=F('performer') == F('site__originator'), distinct=True), + numsites=Count( + 'site__site_id', + filter=F('performer') == F('site__originator'), + distinct=True, + ), evaluation='evaluation_number', evaluation_run='evaluation_run_number', timerange=TimeRangeJSON( From 169060629112ac9240a87584a3ddf1e1a7bd5356 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Wed, 23 Aug 2023 07:40:03 -0400 Subject: [PATCH 05/10] making sure proposals work --- vue/src/client/services/ApiService.ts | 4 ++-- vue/src/router.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/vue/src/client/services/ApiService.ts b/vue/src/client/services/ApiService.ts index 98d1059ef..45874e52f 100644 --- a/vue/src/client/services/ApiService.ts +++ b/vue/src/client/services/ApiService.ts @@ -134,7 +134,7 @@ export class ApiService { ): CancelablePromise { return __request(OpenAPI, { method: "GET", - url:`${this.apiPrefix}/evaluations/{id}`, + url:`${this.apiPrefix}/observations/{id}/`, path: { id: id, }, @@ -494,7 +494,7 @@ export class ApiService { public static startModelRunDownload(id: number): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: "/api/model-runs/{id}/download", + url: "/api/model-runs/{id}/download/", path: { id: id, }, diff --git a/vue/src/router.ts b/vue/src/router.ts index 667d3de77..636af906e 100644 --- a/vue/src/router.ts +++ b/vue/src/router.ts @@ -6,9 +6,10 @@ import { ApiService } from './client/services/ApiService'; // Import your ApiSer const routes = [ { path: '/:region?/:selected?', component: RGD, props:true, }, + { path: '/proposals/:region?/:selected?', component: Annotation, props:true, }, { path: '/annotation/:region?/:selected?', component: Annotation, props:true, }, { path: '/scoring/:region?/:selected?', component: RGD, props:true, }, - { path: '/scoring/annotation/:region?/:selected?', component: Annotation, props:true, }, + { path: '/scoring/proposals/:region?/:selected?', component: Annotation, props:true, }, ] const router = createRouter({ From e915b22782a90c41cc5fef1082827aeba4098126 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 28 Aug 2023 08:09:04 -0400 Subject: [PATCH 06/10] Remove observation searching for geometry in model-run endpoint --- django/src/rdwatch/views/model_run.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/django/src/rdwatch/views/model_run.py b/django/src/rdwatch/views/model_run.py index 492baa6a9..a1af43697 100644 --- a/django/src/rdwatch/views/model_run.py +++ b/django/src/rdwatch/views/model_run.py @@ -174,7 +174,6 @@ def get_queryset(): .order_by('-groundtruth', '-id') .alias( region_id=F('evaluations__region_id'), - observation_count=Count('evaluations__observations'), evaluation_configuration=F('evaluations__configuration'), proposal_val=F('proposal'), ) @@ -230,17 +229,7 @@ def get_queryset(): min=ExtractEpoch(Min('evaluations__start_date')), max=ExtractEpoch(Max('evaluations__end_date')), ), - bbox=Case( - # If there are no site observations associated with this - # site evaluation, return the bbox of the site polygon. - # Otherwise, return the bounding box of all observation - # polygons. - When( - observation_count=0, - then=BoundingBoxGeoJSON('evaluations__geom'), - ), - default=BoundingBoxGeoJSON('evaluations__observations__geom'), - ), + bbox=BoundingBoxGeoJSON('evaluations__geom'), proposal='proposal_val', adjudicated=Case( When( From 9ef12c6cdd98a1dec009b5819ac7d942de7b52e6 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 28 Aug 2023 08:20:51 -0400 Subject: [PATCH 07/10] updating download endpoint --- vue/src/client/services/ApiService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vue/src/client/services/ApiService.ts b/vue/src/client/services/ApiService.ts index fb184dc2c..68bffa956 100644 --- a/vue/src/client/services/ApiService.ts +++ b/vue/src/client/services/ApiService.ts @@ -489,7 +489,7 @@ export class ApiService { public static startModelRunDownload(id: number): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: "/api/model-runs/{id}/download", + url: "/api/model-runs/{id}/download/", path: { id: id, }, From 1534b723ef80835f780ce77ff3445608bc7d48f9 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 28 Aug 2023 08:35:20 -0400 Subject: [PATCH 08/10] fix proposal loading script --- scripts/loadModelRun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/loadModelRun.py b/scripts/loadModelRun.py index 49388af24..3403a157a 100644 --- a/scripts/loadModelRun.py +++ b/scripts/loadModelRun.py @@ -99,7 +99,7 @@ def upload_to_rgd( post_model_data["evaluation"] = eval_num if eval_run_num is not None: post_model_data["evaluation_run"] = eval_run_num - if proposal is not None: + if proposal is True: post_model_data["proposal"] = True if title == "Ground Truth": post_model_data["parameters"] = {"ground_truth": True} From aceb3a9992b0e88727e5a2855217f123d7851edd Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 28 Aug 2023 08:37:30 -0400 Subject: [PATCH 09/10] swap bbox calculation to site-geometry instead of observation --- django/src/rdwatch_scoring/views/model_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/src/rdwatch_scoring/views/model_run.py b/django/src/rdwatch_scoring/views/model_run.py index 71f23708f..4c98efeac 100644 --- a/django/src/rdwatch_scoring/views/model_run.py +++ b/django/src/rdwatch_scoring/views/model_run.py @@ -120,7 +120,7 @@ def get_queryset(): timestamp=ExtractEpoch('start_datetime'), ground_truth=False, # timerange=TimeRangeJSON('evaluations__observations__timestamp'), - bbox=BoundingBoxGeoJSON('site__observation__geometry'), + bbox=BoundingBoxGeoJSON('site__geometry'), ) ) ) From 8b674368cdc68c73df010383d4398c3fad3a4e86 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 28 Aug 2023 08:47:12 -0400 Subject: [PATCH 10/10] meant to be union_geometry --- django/src/rdwatch_scoring/views/model_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/src/rdwatch_scoring/views/model_run.py b/django/src/rdwatch_scoring/views/model_run.py index 4c98efeac..bb708bff0 100644 --- a/django/src/rdwatch_scoring/views/model_run.py +++ b/django/src/rdwatch_scoring/views/model_run.py @@ -120,7 +120,7 @@ def get_queryset(): timestamp=ExtractEpoch('start_datetime'), ground_truth=False, # timerange=TimeRangeJSON('evaluations__observations__timestamp'), - bbox=BoundingBoxGeoJSON('site__geometry'), + bbox=BoundingBoxGeoJSON('site__union_geometry'), ) ) ) @@ -156,7 +156,7 @@ def list_model_runs( aggregate |= queryset.defer('json').aggregate( # Use the region polygon for the bbox if it exists. # Otherwise, fall back on the site polygon. - bbox=BoundingBoxGeoJSON('site__observation__geometry'), + bbox=BoundingBoxGeoJSON('site__union_geometry'), ) return 200, aggregate