Merge pull request #42 from Onemind-Services-LLC/dev

Release v0.3.1

CHANGELOG.md

@@ -1,5 +1,34 @@
 # Changelog
 
+## [v0.2.1](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/tree/v0.2.1) (2024-03-11)
+
+[Full Changelog](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/compare/v0.1.2...v0.2.1)
+
+**Closed issues:**
+
+- \[Feature\]: Ability to programmatically import device types [\#6](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/issues/6)
+
+## [v0.1.2](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/tree/v0.1.2) (2024-02-13)
+
+[Full Changelog](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/compare/v0.3.0...v0.1.2)
+
+**Closed issues:**
+
+- Support git based data sources [\#22](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/issues/22)
+- \[Feature\]: Return to importer after adding device [\#15](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/issues/15)
+
+## [v0.3.0](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/tree/v0.3.0) (2024-01-18)
+
+[Full Changelog](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/compare/v0.2.0...v0.3.0)
+
+**Closed issues:**
+
+- Add support for Netbox 3.7.x [\#35](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/issues/35)
+
+**Merged pull requests:**
+
+- Release v0.3.0 [\#37](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/pull/37) ([kprince28](https://github.com/kprince28))
+
 ## [v0.2.0](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/tree/v0.2.0) (2024-01-16)
 
 [Full Changelog](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/compare/v0.1.1...v0.2.0)

netbox_metatype_importer/__init__.py

@@ -12,6 +12,7 @@ class NetBoxMetatypeImporterConfig(PluginConfig):
     version = metadata.get('Version')
     author = metadata.get('Author')
     author_email = metadata.get('Author-email')
+    base_url = "meta-types"
     min_version = '3.7.0'
     max_version = '3.7.99'
     default_settings = {

netbox_metatype_importer/api/serializers.py

@@ -0,0 +1,9 @@
+from rest_framework import serializers
+
+from ..models import MetaType
+
+
+class MetaTypeSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = MetaType
+        fields = "__all__"

netbox_metatype_importer/api/urls.py

@@ -0,0 +1,17 @@
+from netbox.api.routers import NetBoxRouter
+
+from . import views
+
+router = NetBoxRouter()
+router.APIRootView = views.MetaTypeRootView
+
+router.register('device-types', views.DeviceTypeListViewSet, basename='device-types')
+router.register('module-types', views.ModuleTypeListViewSet, basename="module-types")
+
+router.register('device-type-load', views.MetaDeviceTypeLoadViewSet, basename='device-type-load')
+router.register('module-type-load', views.MetaModuleTypeLoadViewSet, basename="module-type-load")
+
+router.register('device-type-import', views.MetaDeviceTypeImportViewSet, basename='device-type-import')
+router.register('module-type-import', views.MetaModuleTypeImportViewSet, basename="module-type-import")
+
+urlpatterns = router.urls

netbox_metatype_importer/api/views.py

@@ -0,0 +1,216 @@
+from collections import OrderedDict
+from urllib.parse import urlencode
+
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import transaction
+from django.db.models import Q
+from django.shortcuts import reverse
+from django.utils.text import slugify
+from rest_framework import mixins as drf_mixins, status
+from rest_framework.response import Response
+from rest_framework.routers import APIRootView
+
+from dcim import forms
+from dcim.models import DeviceType, Manufacturer, ModuleType
+from netbox.api.viewsets import BaseViewSet
+from netbox_metatype_importer.filters import MetaTypeFilterSet
+from netbox_metatype_importer.forms import MetaTypeFilterForm
+from utilities.exceptions import AbortTransaction, PermissionsViolation
+from utilities.forms.bulk_import import BulkImportForm
+from . import serializers
+from ..choices import TypeChoices
+from ..gql import GQLError, GitHubGqlAPI
+from ..models import MetaType
+from ..utils import *
+
+
+class MetaTypeRootView(APIRootView):
+    """
+    MetaType API root view
+    """
+
+    def get_view_name(self):
+        return 'MetaType'
+
+
+class DeviceTypeListViewSet(drf_mixins.ListModelMixin, BaseViewSet):
+    serializer_class = serializers.MetaTypeSerializer
+    queryset = MetaType.objects.filter(type=TypeChoices.TYPE_DEVICE)
+    filterset_class = MetaTypeFilterSet
+
+
+class ModuleTypeListViewSet(drf_mixins.ListModelMixin, BaseViewSet):
+    serializer_class = serializers.MetaTypeSerializer
+    queryset = MetaType.objects.filter(type=TypeChoices.TYPE_MODULE)
+    filterset_class = MetaTypeFilterSet
+
+
+class MetaTypeLoadViewSetBase(BaseViewSet):
+    serializer_class = serializers.MetaTypeSerializer
+    queryset = MetaType.objects.all()
+    type_choice = None
+
+    def create(self, request, *args, **kwargs):
+        if not request.user.has_perm('netbox_metatype_importer.add_metatype'):
+            return Response(status=status.HTTP_403_FORBIDDEN)
+
+        try:
+            loaded, created, updated = load_data(self.type_choice)
+
+            response_data = {'loaded': loaded, 'created': created, 'updated': updated}
+            return Response(response_data, status=status.HTTP_200_OK)
+        except Exception as e:
+            return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
+
+
+class MetaDeviceTypeLoadViewSet(MetaTypeLoadViewSetBase):
+    type_choice = TypeChoices.TYPE_DEVICE
+
+
+class MetaModuleTypeLoadViewSet(MetaTypeLoadViewSetBase):
+    type_choice = TypeChoices.TYPE_MODULE
+
+
+class MetaTypeImportViewSetBase(BaseViewSet):
+    serializer_class = serializers.MetaTypeSerializer
+    queryset = MetaType.objects.all()
+    filterset = MetaTypeFilterSet
+    filterset_form = MetaTypeFilterForm
+    type = None
+    type_model = None
+    model_form = None
+    related_object = None
+
+    def create(self, request, *args, **kwargs):
+        if not request.user.has_perm('netbox_metatype_importer.add_metatype'):
+            return Response(status=status.HTTP_403_FORBIDDEN)
+
+        vendor_count = 0
+        errored = 0
+        imported_dt = []
+        model = self.queryset.model
+
+        if name := request.data.get("name"):
+            instance = MetaType.objects.filter(
+                Q(name__in=[f"{name}.yaml", f"{name}.yml", name]), type=self.type
+            ).values_list("pk", flat=True)
+            pk_list = list(instance)
+        else:
+            return Response({"error": "Name field is required"}, status=status.HTTP_400_BAD_REQUEST)
+
+        plugin_settings = settings.PLUGINS_CONFIG.get('netbox_metatype_importer', {})
+        token = plugin_settings.get('github_token')
+        repo = plugin_settings.get('repo')
+        branch = plugin_settings.get('branch')
+        owner = plugin_settings.get('repo_owner')
+
+        gh_api = GitHubGqlAPI(token=token, owner=owner, repo=repo, branch=branch, path=self.type)
+
+        query_data = {}
+        # check already imported mdt
+        already_imported_mdt = model.objects.filter(pk__in=pk_list, is_imported=True, type=self.type)
+        if already_imported_mdt.exists():
+            for _mdt in already_imported_mdt:
+                if self.type_model.objects.filter(pk=_mdt.imported_dt).exists() is False:
+                    _mdt.imported_dt = None
+                    _mdt.is_imported = False
+                    _mdt.save()
+        vendors_for_cre = set(model.objects.filter(pk__in=pk_list).values_list('vendor', flat=True).distinct())
+        for vendor, name, sha in model.objects.filter(pk__in=pk_list, is_imported=False).values_list(
+            'vendor', 'name', 'sha'
+        ):
+            query_data[sha] = f'{vendor}/{name}'
+        if not query_data:
+            return Response({'message': 'Nothing to import'}, status=status.HTTP_400_BAD_REQUEST)
+        try:
+            dt_files = gh_api.get_files(query_data)
+        except GQLError as e:
+            return Response({'error': f'GraphQL API Error: {e.message}'}, status=status.HTTP_400_BAD_REQUEST)
+
+        # create manufacturer
+        for vendor in vendors_for_cre:
+            manufacturer, created = Manufacturer.objects.get_or_create(name=vendor, slug=slugify(vendor))
+            if created:
+                vendor_count += 1
+
+        for sha, yaml_text in dt_files.items():
+            form = BulkImportForm(data={'data': yaml_text, 'format': 'yaml'})
+            if form.is_valid():
+                data = form.cleaned_data['data']
+
+                if isinstance(data, list):
+                    data = data[0]
+
+                model_form = self.model_form(data)
+
+                for field_name, field in model_form.fields.items():
+                    if field_name not in data and hasattr(field, 'initial'):
+                        model_form.data[field_name] = field.initial
+
+                if model_form.is_valid():
+                    try:
+                        with transaction.atomic():
+                            obj = model_form.save()
+
+                            for field_name, related_object_form in related_object_forms().items():
+                                related_obj_pks = []
+                                for i, rel_obj_data in enumerate(data.get(field_name, list())):
+                                    rel_obj_data.update({self.related_object: obj})
+                                    f = related_object_form(rel_obj_data)
+                                    for subfield_name, field in f.fields.items():
+                                        if subfield_name not in rel_obj_data and hasattr(field, 'initial'):
+                                            f.data[subfield_name] = field.initial
+                                    if f.is_valid():
+                                        related_obj = f.save()
+                                        related_obj_pks.append(related_obj.pk)
+                                    else:
+                                        for subfield_name, errors in f.errors.items():
+                                            for err in errors:
+                                                err_msg = "{}[{}] {}: {}".format(field_name, i, subfield_name, err)
+                                                model_form.add_error(None, err_msg)
+                                        raise AbortTransaction()
+                    except AbortTransaction:
+                        # log ths
+                        pass
+                    except PermissionsViolation:
+                        errored += 1
+                        continue
+                if model_form.errors:
+                    errored += 1
+                else:
+                    imported_dt.append(obj.pk)
+                    metadt = MetaType.objects.get(sha=sha)
+                    metadt.imported_dt = obj.pk
+                    metadt.save()
+            else:
+                errored += 1
+
+        if imported_dt:
+            if errored:
+                return Response(
+                    {'message': f'Imported: {len(imported_dt)}, Failed: {errored}'},
+                    status=status.HTTP_206_PARTIAL_CONTENT,
+                )
+            else:
+                qparams = urlencode({'id': imported_dt}, doseq=True)
+                url = reverse(f'dcim:{str(self.type).replace("-", "").rstrip("s")}_list') + '?' + qparams
+                return Response(
+                    {'message': f'Imported: {len(imported_dt)}', 'url': url}, status=status.HTTP_201_CREATED
+                )
+        else:
+            return Response({'error': f'Can not import {self.type}'}, status=status.HTTP_400_BAD_REQUEST)
+
+
+class MetaDeviceTypeImportViewSet(MetaTypeImportViewSetBase):
+    type = TypeChoices.TYPE_DEVICE
+    type_model = DeviceType
+    model_form = forms.DeviceTypeImportForm
+    related_object = 'device_type'
+
+
+class MetaModuleTypeImportViewSet(MetaTypeImportViewSetBase):
+    type = TypeChoices.TYPE_MODULE
+    type_model = ModuleType
+    model_form = forms.ModuleTypeImportForm
+    related_object = 'module_type'

netbox_metatype_importer/utils.py

@@ -0,0 +1,62 @@
+from collections import OrderedDict
+
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
+
+from dcim import forms
+from .gql import GitHubGqlAPI, GQLError
+from .models import MetaType
+
+__all__ = ['load_data', 'related_object_forms']
+
+
+def load_data(type_choice):
+    loaded = created = updated = 0
+    plugin_settings = settings.PLUGINS_CONFIG.get('netbox_metatype_importer', {})
+    token = plugin_settings.get('github_token')
+    repo = plugin_settings.get('repo')
+    branch = plugin_settings.get('branch')
+    owner = plugin_settings.get('repo_owner')
+    gh_api_instance = GitHubGqlAPI(token=token, owner=owner, repo=repo, branch=branch, path=type_choice)
+
+    try:
+        models = gh_api_instance.get_tree()
+    except GQLError as e:
+        return Exception(f'GraphQL API Error: {e.message}')
+
+    for vendor, models_data in models.items():
+        for model, model_data in models_data.items():
+            loaded += 1
+            try:
+                meta_type = MetaType.objects.get(vendor=vendor, name=model, type=type_choice)
+                if meta_type.sha != model_data['sha']:
+                    meta_type.is_new = True
+                    meta_type.save()
+                    updated += 1
+                else:
+                    meta_type.is_new = False
+                    meta_type.save()
+                continue
+            except ObjectDoesNotExist:
+                MetaType.objects.create(vendor=vendor, name=model, sha=model_data['sha'], type=type_choice)
+                created += 1
+
+    return loaded, created, updated
+
+
+def related_object_forms():
+    return OrderedDict(
+        (
+            ('console-ports', forms.ConsolePortTemplateImportForm),
+            ('console-server-ports', forms.ConsoleServerPortTemplateImportForm),
+            ('power-ports', forms.PowerPortTemplateImportForm),
+            ('power-outlets', forms.PowerOutletTemplateImportForm),
+            ('interfaces', forms.InterfaceTemplateImportForm),
+            ('rear-ports', forms.RearPortTemplateImportForm),
+            ('front-ports', forms.FrontPortTemplateImportForm),
+            ('device-bays', forms.DeviceBayTemplateImportForm),
+            ('inventory-items', forms.InventoryItemTemplateImportForm),
+            ('module-bays', forms.ModuleBayTemplateImportForm),
+            ('device-bays', forms.DeviceBayTemplateImportForm),
+        )
+    )