Skip to content

Commit

Permalink
Merge pull request #40 from Onemind-Services-LLC/feat/api
Browse files Browse the repository at this point in the history
Add MetaType APIs for NetBox v3.6
  • Loading branch information
kprince28 authored Mar 11, 2024
2 parents 9e0fa29 + 80fc093 commit b3350e8
Show file tree
Hide file tree
Showing 9 changed files with 319 additions and 77 deletions.
1 change: 1 addition & 0 deletions netbox_metatype_importer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ class NetBoxMetatypeImporterConfig(PluginConfig):
version = metadata.get('Version')
author = metadata.get('Author')
author_email = metadata.get('Author-email')
base_url = "meta-types"
min_version = '3.6.0'
max_version = '3.6.99'
default_settings = {
Expand Down
Empty file.
9 changes: 9 additions & 0 deletions netbox_metatype_importer/api/serializers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from rest_framework import serializers

from ..models import MetaType


class MetaTypeSerializer(serializers.ModelSerializer):
class Meta:
model = MetaType
fields = "__all__"
17 changes: 17 additions & 0 deletions netbox_metatype_importer/api/urls.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
from netbox.api.routers import NetBoxRouter

from . import views

router = NetBoxRouter()
router.APIRootView = views.MetaTypeRootView

router.register('device-types', views.DeviceTypeListViewSet, basename='device-types')
router.register('module-types', views.ModuleTypeListViewSet, basename="module-types")

router.register('device-type-load', views.MetaDeviceTypeLoadViewSet, basename='device-type-load')
router.register('module-type-load', views.MetaModuleTypeLoadViewSet, basename="module-type-load")

router.register('device-type-import', views.MetaDeviceTypeImportViewSet, basename='device-type-import')
router.register('module-type-import', views.MetaModuleTypeImportViewSet, basename="module-type-import")

urlpatterns = router.urls
217 changes: 217 additions & 0 deletions netbox_metatype_importer/api/views.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
from collections import OrderedDict
from urllib.parse import urlencode

from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
from django.db.models import Q
from django.shortcuts import reverse
from django.utils.text import slugify
from rest_framework import mixins as drf_mixins, status
from rest_framework.response import Response
from rest_framework.routers import APIRootView

from dcim import forms
from dcim.models import DeviceType, Manufacturer, ModuleType
from netbox.api.viewsets import BaseViewSet
from netbox_metatype_importer.filters import MetaTypeFilterSet
from netbox_metatype_importer.forms import MetaTypeFilterForm
from utilities.exceptions import AbortTransaction, PermissionsViolation
from utilities.forms.bulk_import import BulkImportForm
from . import serializers
from ..choices import TypeChoices
from ..gql import GQLError, GitHubGqlAPI
from ..models import MetaType
from ..utils import *


class MetaTypeRootView(APIRootView):
"""
MetaType API root view
"""

def get_view_name(self):
return 'MetaType'


class DeviceTypeListViewSet(drf_mixins.ListModelMixin, BaseViewSet):
serializer_class = serializers.MetaTypeSerializer
queryset = MetaType.objects.filter(type=TypeChoices.TYPE_DEVICE)
filterset_class = MetaTypeFilterSet


class ModuleTypeListViewSet(drf_mixins.ListModelMixin, BaseViewSet):
serializer_class = serializers.MetaTypeSerializer
queryset = MetaType.objects.filter(type=TypeChoices.TYPE_MODULE)
filterset_class = MetaTypeFilterSet


class MetaTypeLoadViewSetBase(BaseViewSet):
serializer_class = serializers.MetaTypeSerializer
queryset = MetaType.objects.all()
type_choice = None

def create(self, request, *args, **kwargs):
if not request.user.has_perm('netbox_metatype_importer.add_metatype'):
return Response(status=status.HTTP_403_FORBIDDEN)

try:
loaded, created, updated = load_data(self.type_choice)

response_data = {
'loaded': loaded,
'created': created,
'updated': updated
}
return Response(response_data, status=status.HTTP_200_OK)
except Exception as e:
return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)


class MetaDeviceTypeLoadViewSet(MetaTypeLoadViewSetBase):
type_choice = TypeChoices.TYPE_DEVICE


class MetaModuleTypeLoadViewSet(MetaTypeLoadViewSetBase):
type_choice = TypeChoices.TYPE_MODULE


class MetaTypeImportViewSetBase(BaseViewSet):
serializer_class = serializers.MetaTypeSerializer
queryset = MetaType.objects.all()
filterset = MetaTypeFilterSet
filterset_form = MetaTypeFilterForm
type = None
type_model = None
model_form = None
related_object = None

def create(self, request, *args, **kwargs):
if not request.user.has_perm('netbox_metatype_importer.add_metatype'):
return Response(status=status.HTTP_403_FORBIDDEN)

vendor_count = 0
errored = 0
imported_dt = []
model = self.queryset.model

if name := request.data.get("name"):
instance = MetaType.objects.filter(
Q(name__in=[f"{name}.yaml", f"{name}.yml", name]), type=self.type
).values_list("pk", flat=True)
pk_list = list(instance)
else:
return Response({"error": "Name field is required"}, status=status.HTTP_400_BAD_REQUEST)

plugin_settings = settings.PLUGINS_CONFIG.get('netbox_metatype_importer', {})
token = plugin_settings.get('github_token')
repo = plugin_settings.get('repo')
branch = plugin_settings.get('branch')
owner = plugin_settings.get('repo_owner')

gh_api = GitHubGqlAPI(token=token, owner=owner, repo=repo, branch=branch, path=self.type)

query_data = {}
# check already imported mdt
already_imported_mdt = model.objects.filter(pk__in=pk_list, is_imported=True, type=self.type)
if already_imported_mdt.exists():
for _mdt in already_imported_mdt:
if self.type_model.objects.filter(pk=_mdt.imported_dt).exists() is False:
_mdt.imported_dt = None
_mdt.is_imported = False
_mdt.save()
vendors_for_cre = set(model.objects.filter(pk__in=pk_list).values_list('vendor', flat=True).distinct())
for vendor, name, sha in model.objects.filter(pk__in=pk_list, is_imported=False).values_list(
'vendor', 'name', 'sha'
):
query_data[sha] = f'{vendor}/{name}'
if not query_data:
return Response({'message': 'Nothing to import'}, status=status.HTTP_400_BAD_REQUEST)
try:
dt_files = gh_api.get_files(query_data)
except GQLError as e:
return Response({'error': f'GraphQL API Error: {e.message}'}, status=status.HTTP_400_BAD_REQUEST)

# create manufacturer
for vendor in vendors_for_cre:
manufacturer, created = Manufacturer.objects.get_or_create(name=vendor, slug=slugify(vendor))
if created:
vendor_count += 1

for sha, yaml_text in dt_files.items():
form = BulkImportForm(data={'data': yaml_text, 'format': 'yaml'})
if form.is_valid():
data = form.cleaned_data['data']

if isinstance(data, list):
data = data[0]

model_form = self.model_form(data)

for field_name, field in model_form.fields.items():
if field_name not in data and hasattr(field, 'initial'):
model_form.data[field_name] = field.initial

if model_form.is_valid():
try:
with transaction.atomic():
obj = model_form.save()

for field_name, related_object_form in related_object_forms().items():
related_obj_pks = []
for i, rel_obj_data in enumerate(data.get(field_name, list())):
rel_obj_data.update({self.related_object: obj})
f = related_object_form(rel_obj_data)
for subfield_name, field in f.fields.items():
if subfield_name not in rel_obj_data and hasattr(field, 'initial'):
f.data[subfield_name] = field.initial
if f.is_valid():
related_obj = f.save()
related_obj_pks.append(related_obj.pk)
else:
for subfield_name, errors in f.errors.items():
for err in errors:
err_msg = "{}[{}] {}: {}".format(field_name, i, subfield_name, err)
model_form.add_error(None, err_msg)
raise AbortTransaction()
except AbortTransaction:
# log ths
pass
except PermissionsViolation:
errored += 1
continue
if model_form.errors:
errored += 1
else:
imported_dt.append(obj.pk)
metadt = MetaType.objects.get(sha=sha)
metadt.imported_dt = obj.pk
metadt.save()
else:
errored += 1

if imported_dt:
if errored:
return Response({'message': f'Imported: {len(imported_dt)}, Failed: {errored}'},
status=status.HTTP_206_PARTIAL_CONTENT)
else:
qparams = urlencode({'id': imported_dt}, doseq=True)
url = reverse(f'dcim:{str(self.type).replace("-", "").rstrip("s")}_list') + '?' + qparams
return Response({'message': f'Imported: {len(imported_dt)}', 'url': url},
status=status.HTTP_201_CREATED)
else:
return Response({'error': f'Can not import {self.type}'}, status=status.HTTP_400_BAD_REQUEST)


class MetaDeviceTypeImportViewSet(MetaTypeImportViewSetBase):
type = TypeChoices.TYPE_DEVICE
type_model = DeviceType
model_form = forms.DeviceTypeImportForm
related_object = 'device_type'


class MetaModuleTypeImportViewSet(MetaTypeImportViewSetBase):
type = TypeChoices.TYPE_MODULE
type_model = ModuleType
model_form = forms.ModuleTypeImportForm
related_object = 'module_type'
16 changes: 5 additions & 11 deletions netbox_metatype_importer/filters.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import django_filters
from django.db.models import Q

from utilities.filters import MultiValueCharFilter
from .models import MetaType


Expand All @@ -10,9 +11,8 @@ class MetaTypeFilterSet(django_filters.FilterSet):
label='Search',
)

name = django_filters.CharFilter(
method='by_model',
label='Model',
name = MultiValueCharFilter(
lookup_expr='iexact'
)

vendor = django_filters.CharFilter(
Expand All @@ -22,13 +22,7 @@ class MetaTypeFilterSet(django_filters.FilterSet):

class Meta:
model = MetaType
fields = ['name', 'vendor']

def by_model(self, queryset, name, value):
if not value.strip():
return queryset

return queryset.filter(Q(name__icontains=value))
fields = ['id', 'name', 'vendor']

def by_vendor(self, queryset, name, value):
if not value.strip():
Expand All @@ -46,6 +40,6 @@ def search(self, queryset, name, value):
return queryset

qs_filter = (
Q(name__icontains=value) | Q(vendor__icontains=value)
Q(name__icontains=value) | Q(vendor__icontains=value)
)
return queryset.filter(qs_filter)
65 changes: 65 additions & 0 deletions netbox_metatype_importer/utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
from collections import OrderedDict

from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist

from dcim import forms
from .gql import GitHubGqlAPI, GQLError
from .models import MetaType

__all__ = ['load_data', 'related_object_forms']


def load_data(type_choice):
loaded = created = updated = 0
plugin_settings = settings.PLUGINS_CONFIG.get('netbox_metatype_importer', {})
token = plugin_settings.get('github_token')
repo = plugin_settings.get('repo')
branch = plugin_settings.get('branch')
owner = plugin_settings.get('repo_owner')
gh_api_instance = GitHubGqlAPI(token=token, owner=owner, repo=repo, branch=branch, path=type_choice)

try:
models = gh_api_instance.get_tree()
except GQLError as e:
return Exception(f'GraphQL API Error: {e.message}')

for vendor, models_data in models.items():
for model, model_data in models_data.items():
loaded += 1
try:
meta_type = MetaType.objects.get(vendor=vendor, name=model, type=type_choice)
if meta_type.sha != model_data['sha']:
meta_type.is_new = True
meta_type.save()
updated += 1
else:
meta_type.is_new = False
meta_type.save()
continue
except ObjectDoesNotExist:
MetaType.objects.create(
vendor=vendor,
name=model,
sha=model_data['sha'],
type=type_choice
)
created += 1

return loaded, created, updated


def related_object_forms():
return OrderedDict((
('console-ports', forms.ConsolePortTemplateImportForm),
('console-server-ports', forms.ConsoleServerPortTemplateImportForm),
('power-ports', forms.PowerPortTemplateImportForm),
('power-outlets', forms.PowerOutletTemplateImportForm),
('interfaces', forms.InterfaceTemplateImportForm),
('rear-ports', forms.RearPortTemplateImportForm),
('front-ports', forms.FrontPortTemplateImportForm),
('device-bays', forms.DeviceBayTemplateImportForm),
('inventory-items', forms.InventoryItemTemplateImportForm),
('module-bays', forms.ModuleBayTemplateImportForm),
('device-bays', forms.DeviceBayTemplateImportForm)
))
Loading

0 comments on commit b3350e8

Please sign in to comment.