Feature/status board

api/serializers.py

@@ -5,10 +5,14 @@
 from rest_framework_json_api import serializers
 
 from share import models
-from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult
 
-from api import fields
 
+from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, HarvestLog, SourceConfig
+
+
+from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, logs
+
+from api import fields
 
 class BaseShareSerializer(serializers.ModelSerializer):
 
@@ -132,12 +136,21 @@ class Meta:
  'is_active', 'gravatar', 'locale', 'time_zone', 'is_trusted'
  )
 
-
 class SourceSerializer(ShareModelSerializer):
  class Meta:
  model = models.Source
  fields = ('name', 'home_page', 'long_title', 'icon')
 
+class HarvestLogSerializer(ShareModelSerializer):
+ class Meta:
+ model = models.HarvestLog
+ fields = '__all__'
+
+class SourceConfigSerializer(ShareModelSerializer):
+ class Meta:
+ model = models.SourceConfig
+ fields = '__all__'
+
 
 class SiteBannerSerializer(ShareModelSerializer):
  color = serializers.SerializerMethodField()

api/urls.py

@@ -14,6 +14,8 @@
 from api.serializers import BaseShareSerializer
 from api.views.share import ShareObjectViewSet
 
+
+
 app_name = 'api'
 
 router = DefaultRouter()
@@ -86,6 +88,8 @@ def register_url(self, subclass, viewset):
 register_route(r'rawdata', views.RawDatumViewSet)
 register_route(r'user', views.ShareUserViewSet)
 register_route(r'sources', views.SourceViewSet)
+register_route(r'HarvestLog', views.HarvestLogViewSet)
+register_route(r'SourceConfig', views.SourceConfigViewSet)
 
 router.register(r'normalizeddata', views.NormalizedDataViewSet, base_name='normalizeddata')
 

api/views/__init__.py

@@ -5,3 +5,5 @@
 from .registration import * # noqa
 from .schema import * # noqa
 from .banner import * # noqa
+from .harvestlogs import * # noqa
+from .sourceConfig import * # noqa

api/views/harvestlogs.py

@@ -0,0 +1,23 @@
+from rest_framework import viewsets
+from rest_framework import filters
+from django_filters.filters import MultipleChoiceFilter
+from api.views import ShareObjectViewSet
+from share.util import IDObfuscator
+
+from api.serializers import HarvestLogSerializer
+from share.models import HarvestLog
+
+class SourceConfigFilterBackend(MultipleChoiceFilter):
+ def filter_queryset(self, request, queryset, view, conjoined=True):
+ if 'source_config_id' in request.GET:
+ decoded = IDObfuscator.decode_id(request.GET['source_config_id'])
+ queryset = queryset.filter(source_config_id=decoded)
+ if 'status' in request.GET:
+ queryset = queryset.filter(status__in=request.GET.getlist('status'))
+ return queryset
+
+class HarvestLogViewSet(ShareObjectViewSet):
+ serializer_class = HarvestLogSerializer
+ queryset = HarvestLog.objects.all()
+ filter_backends = (SourceConfigFilterBackend, )
+ filter_fields = ('source_config_id', 'status',)

api/views/share.py

@@ -28,13 +28,15 @@ class ShareObjectViewSet(viewsets.ReadOnlyModelViewSet):
 
  # Override get_queryset to handle items marked as deleted.
  def get_queryset(self, list=True):
+ # import ipdb; ipdb.set_trace()
  queryset = super().get_queryset()
  if list and hasattr(queryset.model, 'is_deleted'):
  return queryset.exclude(is_deleted=True)
  return queryset
 
  # Override to convert encoded pk to an actual pk
  def get_object(self):
+ import ipdb; ipdb.set_trace()
  queryset = self.filter_queryset(self.get_queryset(False))
 
  # Perform the lookup filtering.

api/views/sourceConfig.py

@@ -0,0 +1,18 @@
+from rest_framework import viewsets
+from api.views import ShareObjectViewSet
+from rest_framework import filters
+from api.serializers import SourceConfigSerializer
+from share.models import SourceConfig
+from api.pagination import FuzzyPageNumberPagination
+from django.db.models import Count
+from django.db.models import Aggregate
+from share.models import HarvestLog
+from django.db.models import OuterRef, Subquery
+from django.db.models import IntegerField
+
+class SourceConfigViewSet(ShareObjectViewSet):
+ serializer_class = SourceConfigSerializer
+ pagination_class = FuzzyPageNumberPagination
+ queryset = SourceConfig.objects.all()
+
+

api/views/workflow.py

@@ -20,8 +20,8 @@
 from api.pagination import CursorPagination
 from api.authentication import APIV1TokenBackPortAuthentication
 from api.permissions import ReadOnlyOrTokenHasScopeOrIsAuthenticated
-from api.serializers import FullNormalizedDataSerializer, BasicNormalizedDataSerializer, \
- RawDatumSerializer, ShareUserSerializer, SourceSerializer
+from api.serializers import FullNormalizedDataSerializer, BasicNormalizedDataSerializer,RawDatumSerializer, ShareUserSerializer, SourceSerializer
+
 from share.models import RawDatum, NormalizedData, Source, SourceConfig, Transformer, ShareUser
 from share.tasks import disambiguate
 from share.harvest.serialization import DictSerializer
@@ -56,6 +56,7 @@ class SourceViewSet(viewsets.ReadOnlyModelViewSet):
  def get_queryset(self):
  return Source.objects.exclude(icon='').exclude(is_deleted=True)
 
+
  def create(self, request, *args, **kwargs):
  try:
  long_title = request.data['long_title']
@@ -129,6 +130,7 @@ def create(self, request, *args, **kwargs):
  )
 
 
+
 class NormalizedDataViewSet(viewsets.ModelViewSet):
  """View showing all normalized data in the SHARE Dataset.
 

docs/elasticsearch.rst

@@ -26,6 +26,7 @@ Elasticsearch can be used to search the following fields in the normalized data:
  'contributors'
  'funders'
  'publishers'
+ 'id'
 
 
 Accessing the Search API