diff --git a/CHANGELOG.md b/CHANGELOG.md index 9911bde..9347213 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## [v0.2.1](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/tree/v0.2.1) (2024-03-11) + +[Full Changelog](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/compare/v0.1.2...v0.2.1) + +**Closed issues:** + +- \[Feature\]: Ability to programmatically import device types [\#6](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/issues/6) + +## [v0.1.2](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/tree/v0.1.2) (2024-02-13) + +[Full Changelog](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/compare/v0.3.0...v0.1.2) + +**Closed issues:** + +- Support git based data sources [\#22](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/issues/22) +- \[Feature\]: Return to importer after adding device [\#15](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/issues/15) + +## [v0.3.0](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/tree/v0.3.0) (2024-01-18) + +[Full Changelog](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/compare/v0.2.0...v0.3.0) + +**Closed issues:** + +- Add support for Netbox 3.7.x [\#35](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/issues/35) + +**Merged pull requests:** + +- Release v0.3.0 [\#37](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/pull/37) ([kprince28](https://github.com/kprince28)) + ## [v0.2.0](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/tree/v0.2.0) (2024-01-16) [Full Changelog](https://github.com/Onemind-Services-LLC/netbox-metatype-importer/compare/v0.1.1...v0.2.0) diff --git a/netbox_metatype_importer/__init__.py b/netbox_metatype_importer/__init__.py index 6f8686e..e4d8bc3 100644 --- a/netbox_metatype_importer/__init__.py +++ b/netbox_metatype_importer/__init__.py @@ -12,6 +12,7 @@ class NetBoxMetatypeImporterConfig(PluginConfig): version = metadata.get('Version') author = metadata.get('Author') author_email = metadata.get('Author-email') + base_url = "meta-types" min_version = '3.7.0' max_version = '3.7.99' default_settings = { diff --git a/netbox_metatype_importer/api/__init__.py b/netbox_metatype_importer/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/netbox_metatype_importer/api/serializers.py b/netbox_metatype_importer/api/serializers.py new file mode 100644 index 0000000..97e1405 --- /dev/null +++ b/netbox_metatype_importer/api/serializers.py @@ -0,0 +1,9 @@ +from rest_framework import serializers + +from ..models import MetaType + + +class MetaTypeSerializer(serializers.ModelSerializer): + class Meta: + model = MetaType + fields = "__all__" diff --git a/netbox_metatype_importer/api/urls.py b/netbox_metatype_importer/api/urls.py new file mode 100644 index 0000000..8bd075c --- /dev/null +++ b/netbox_metatype_importer/api/urls.py @@ -0,0 +1,17 @@ +from netbox.api.routers import NetBoxRouter + +from . import views + +router = NetBoxRouter() +router.APIRootView = views.MetaTypeRootView + +router.register('device-types', views.DeviceTypeListViewSet, basename='device-types') +router.register('module-types', views.ModuleTypeListViewSet, basename="module-types") + +router.register('device-type-load', views.MetaDeviceTypeLoadViewSet, basename='device-type-load') +router.register('module-type-load', views.MetaModuleTypeLoadViewSet, basename="module-type-load") + +router.register('device-type-import', views.MetaDeviceTypeImportViewSet, basename='device-type-import') +router.register('module-type-import', views.MetaModuleTypeImportViewSet, basename="module-type-import") + +urlpatterns = router.urls diff --git a/netbox_metatype_importer/api/views.py b/netbox_metatype_importer/api/views.py new file mode 100644 index 0000000..b999ec2 --- /dev/null +++ b/netbox_metatype_importer/api/views.py @@ -0,0 +1,216 @@ +from collections import OrderedDict +from urllib.parse import urlencode + +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist +from django.db import transaction +from django.db.models import Q +from django.shortcuts import reverse +from django.utils.text import slugify +from rest_framework import mixins as drf_mixins, status +from rest_framework.response import Response +from rest_framework.routers import APIRootView + +from dcim import forms +from dcim.models import DeviceType, Manufacturer, ModuleType +from netbox.api.viewsets import BaseViewSet +from netbox_metatype_importer.filters import MetaTypeFilterSet +from netbox_metatype_importer.forms import MetaTypeFilterForm +from utilities.exceptions import AbortTransaction, PermissionsViolation +from utilities.forms.bulk_import import BulkImportForm +from . import serializers +from ..choices import TypeChoices +from ..gql import GQLError, GitHubGqlAPI +from ..models import MetaType +from ..utils import * + + +class MetaTypeRootView(APIRootView): + """ + MetaType API root view + """ + + def get_view_name(self): + return 'MetaType' + + +class DeviceTypeListViewSet(drf_mixins.ListModelMixin, BaseViewSet): + serializer_class = serializers.MetaTypeSerializer + queryset = MetaType.objects.filter(type=TypeChoices.TYPE_DEVICE) + filterset_class = MetaTypeFilterSet + + +class ModuleTypeListViewSet(drf_mixins.ListModelMixin, BaseViewSet): + serializer_class = serializers.MetaTypeSerializer + queryset = MetaType.objects.filter(type=TypeChoices.TYPE_MODULE) + filterset_class = MetaTypeFilterSet + + +class MetaTypeLoadViewSetBase(BaseViewSet): + serializer_class = serializers.MetaTypeSerializer + queryset = MetaType.objects.all() + type_choice = None + + def create(self, request, *args, **kwargs): + if not request.user.has_perm('netbox_metatype_importer.add_metatype'): + return Response(status=status.HTTP_403_FORBIDDEN) + + try: + loaded, created, updated = load_data(self.type_choice) + + response_data = {'loaded': loaded, 'created': created, 'updated': updated} + return Response(response_data, status=status.HTTP_200_OK) + except Exception as e: + return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST) + + +class MetaDeviceTypeLoadViewSet(MetaTypeLoadViewSetBase): + type_choice = TypeChoices.TYPE_DEVICE + + +class MetaModuleTypeLoadViewSet(MetaTypeLoadViewSetBase): + type_choice = TypeChoices.TYPE_MODULE + + +class MetaTypeImportViewSetBase(BaseViewSet): + serializer_class = serializers.MetaTypeSerializer + queryset = MetaType.objects.all() + filterset = MetaTypeFilterSet + filterset_form = MetaTypeFilterForm + type = None + type_model = None + model_form = None + related_object = None + + def create(self, request, *args, **kwargs): + if not request.user.has_perm('netbox_metatype_importer.add_metatype'): + return Response(status=status.HTTP_403_FORBIDDEN) + + vendor_count = 0 + errored = 0 + imported_dt = [] + model = self.queryset.model + + if name := request.data.get("name"): + instance = MetaType.objects.filter( + Q(name__in=[f"{name}.yaml", f"{name}.yml", name]), type=self.type + ).values_list("pk", flat=True) + pk_list = list(instance) + else: + return Response({"error": "Name field is required"}, status=status.HTTP_400_BAD_REQUEST) + + plugin_settings = settings.PLUGINS_CONFIG.get('netbox_metatype_importer', {}) + token = plugin_settings.get('github_token') + repo = plugin_settings.get('repo') + branch = plugin_settings.get('branch') + owner = plugin_settings.get('repo_owner') + + gh_api = GitHubGqlAPI(token=token, owner=owner, repo=repo, branch=branch, path=self.type) + + query_data = {} + # check already imported mdt + already_imported_mdt = model.objects.filter(pk__in=pk_list, is_imported=True, type=self.type) + if already_imported_mdt.exists(): + for _mdt in already_imported_mdt: + if self.type_model.objects.filter(pk=_mdt.imported_dt).exists() is False: + _mdt.imported_dt = None + _mdt.is_imported = False + _mdt.save() + vendors_for_cre = set(model.objects.filter(pk__in=pk_list).values_list('vendor', flat=True).distinct()) + for vendor, name, sha in model.objects.filter(pk__in=pk_list, is_imported=False).values_list( + 'vendor', 'name', 'sha' + ): + query_data[sha] = f'{vendor}/{name}' + if not query_data: + return Response({'message': 'Nothing to import'}, status=status.HTTP_400_BAD_REQUEST) + try: + dt_files = gh_api.get_files(query_data) + except GQLError as e: + return Response({'error': f'GraphQL API Error: {e.message}'}, status=status.HTTP_400_BAD_REQUEST) + + # create manufacturer + for vendor in vendors_for_cre: + manufacturer, created = Manufacturer.objects.get_or_create(name=vendor, slug=slugify(vendor)) + if created: + vendor_count += 1 + + for sha, yaml_text in dt_files.items(): + form = BulkImportForm(data={'data': yaml_text, 'format': 'yaml'}) + if form.is_valid(): + data = form.cleaned_data['data'] + + if isinstance(data, list): + data = data[0] + + model_form = self.model_form(data) + + for field_name, field in model_form.fields.items(): + if field_name not in data and hasattr(field, 'initial'): + model_form.data[field_name] = field.initial + + if model_form.is_valid(): + try: + with transaction.atomic(): + obj = model_form.save() + + for field_name, related_object_form in related_object_forms().items(): + related_obj_pks = [] + for i, rel_obj_data in enumerate(data.get(field_name, list())): + rel_obj_data.update({self.related_object: obj}) + f = related_object_form(rel_obj_data) + for subfield_name, field in f.fields.items(): + if subfield_name not in rel_obj_data and hasattr(field, 'initial'): + f.data[subfield_name] = field.initial + if f.is_valid(): + related_obj = f.save() + related_obj_pks.append(related_obj.pk) + else: + for subfield_name, errors in f.errors.items(): + for err in errors: + err_msg = "{}[{}] {}: {}".format(field_name, i, subfield_name, err) + model_form.add_error(None, err_msg) + raise AbortTransaction() + except AbortTransaction: + # log ths + pass + except PermissionsViolation: + errored += 1 + continue + if model_form.errors: + errored += 1 + else: + imported_dt.append(obj.pk) + metadt = MetaType.objects.get(sha=sha) + metadt.imported_dt = obj.pk + metadt.save() + else: + errored += 1 + + if imported_dt: + if errored: + return Response( + {'message': f'Imported: {len(imported_dt)}, Failed: {errored}'}, + status=status.HTTP_206_PARTIAL_CONTENT, + ) + else: + qparams = urlencode({'id': imported_dt}, doseq=True) + url = reverse(f'dcim:{str(self.type).replace("-", "").rstrip("s")}_list') + '?' + qparams + return Response( + {'message': f'Imported: {len(imported_dt)}', 'url': url}, status=status.HTTP_201_CREATED + ) + else: + return Response({'error': f'Can not import {self.type}'}, status=status.HTTP_400_BAD_REQUEST) + + +class MetaDeviceTypeImportViewSet(MetaTypeImportViewSetBase): + type = TypeChoices.TYPE_DEVICE + type_model = DeviceType + model_form = forms.DeviceTypeImportForm + related_object = 'device_type' + + +class MetaModuleTypeImportViewSet(MetaTypeImportViewSetBase): + type = TypeChoices.TYPE_MODULE + type_model = ModuleType + model_form = forms.ModuleTypeImportForm + related_object = 'module_type' diff --git a/netbox_metatype_importer/utils.py b/netbox_metatype_importer/utils.py new file mode 100644 index 0000000..b88eb54 --- /dev/null +++ b/netbox_metatype_importer/utils.py @@ -0,0 +1,62 @@ +from collections import OrderedDict + +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist + +from dcim import forms +from .gql import GitHubGqlAPI, GQLError +from .models import MetaType + +__all__ = ['load_data', 'related_object_forms'] + + +def load_data(type_choice): + loaded = created = updated = 0 + plugin_settings = settings.PLUGINS_CONFIG.get('netbox_metatype_importer', {}) + token = plugin_settings.get('github_token') + repo = plugin_settings.get('repo') + branch = plugin_settings.get('branch') + owner = plugin_settings.get('repo_owner') + gh_api_instance = GitHubGqlAPI(token=token, owner=owner, repo=repo, branch=branch, path=type_choice) + + try: + models = gh_api_instance.get_tree() + except GQLError as e: + return Exception(f'GraphQL API Error: {e.message}') + + for vendor, models_data in models.items(): + for model, model_data in models_data.items(): + loaded += 1 + try: + meta_type = MetaType.objects.get(vendor=vendor, name=model, type=type_choice) + if meta_type.sha != model_data['sha']: + meta_type.is_new = True + meta_type.save() + updated += 1 + else: + meta_type.is_new = False + meta_type.save() + continue + except ObjectDoesNotExist: + MetaType.objects.create(vendor=vendor, name=model, sha=model_data['sha'], type=type_choice) + created += 1 + + return loaded, created, updated + + +def related_object_forms(): + return OrderedDict( + ( + ('console-ports', forms.ConsolePortTemplateImportForm), + ('console-server-ports', forms.ConsoleServerPortTemplateImportForm), + ('power-ports', forms.PowerPortTemplateImportForm), + ('power-outlets', forms.PowerOutletTemplateImportForm), + ('interfaces', forms.InterfaceTemplateImportForm), + ('rear-ports', forms.RearPortTemplateImportForm), + ('front-ports', forms.FrontPortTemplateImportForm), + ('device-bays', forms.DeviceBayTemplateImportForm), + ('inventory-items', forms.InventoryItemTemplateImportForm), + ('module-bays', forms.ModuleBayTemplateImportForm), + ('device-bays', forms.DeviceBayTemplateImportForm), + ) + ) diff --git a/netbox_metatype_importer/views.py b/netbox_metatype_importer/views.py index 5f083a7..2ecc3aa 100644 --- a/netbox_metatype_importer/views.py +++ b/netbox_metatype_importer/views.py @@ -22,6 +22,7 @@ from .gql import GQLError, GitHubGqlAPI from .models import MetaType from .tables import MetaTypeTable +from .utils import * class MetaDeviceTypeListView(generic.ObjectListView): @@ -49,49 +50,12 @@ def get_required_permission(self): return 'netbox_metatype_importer.add_metatype' def post(self, request): - loaded = 0 - created = 0 - updated = 0 return_url = self.get_return_url(request) if not request.user.has_perm('netbox_metatype_importer.add_metatype'): return HttpResponseForbidden() - plugin_settings = settings.PLUGINS_CONFIG.get('netbox_metatype_importer', {}) - token = plugin_settings.get('github_token') - repo = plugin_settings.get('repo') - branch = plugin_settings.get('branch') - owner = plugin_settings.get('repo_owner') - gh_api = GitHubGqlAPI(token=token, owner=owner, repo=repo, branch=branch, path=self.path) - try: - models = gh_api.get_tree() - except GQLError as e: - messages.error(request, message=f'GraphQL API Error: {e.message}') - return redirect('plugins:netbox_metatype_importer:metadevicetype_list') - - if models is None: - messages.error(request, 'Check your plugin settings and try again') - models = {} - - for vendor, models in models.items(): - for model, model_data in models.items(): - loaded += 1 - try: - metadevietype = MetaType.objects.get(vendor=vendor, name=model, type=self.path) - if metadevietype.sha != model_data['sha']: - metadevietype.is_new = True - # catch save exception - metadevietype.save() - updated += 1 - else: - metadevietype.is_new = False - metadevietype.save() - continue - except ObjectDoesNotExist: - # its new - MetaType.objects.create(vendor=vendor, name=model, sha=model_data['sha'], type=self.path) - created += 1 - if models: - messages.success(request, f'Loaded: {loaded}, Created: {created}, Updated: {updated}') + created, updated, loaded = load_data(self.path) + messages.success(request, f'Loaded: {loaded}, Created: {created}, Updated: {updated}') return redirect(return_url) @@ -111,19 +75,6 @@ class GenericTypeImportView(ContentTypePermissionRequiredMixin, GetReturnURLMixi model_form = None related_object = None - related_object_forms = OrderedDict( - ( - ('console-ports', forms.ConsolePortTemplateImportForm), - ('console-server-ports', forms.ConsoleServerPortTemplateImportForm), - ('power-ports', forms.PowerPortTemplateImportForm), - ('power-outlets', forms.PowerOutletTemplateImportForm), - ('interfaces', forms.InterfaceTemplateImportForm), - ('rear-ports', forms.RearPortTemplateImportForm), - ('front-ports', forms.FrontPortTemplateImportForm), - ('device-bays', forms.DeviceBayTemplateImportForm), - ) - ) - def get_required_permission(self): return 'netbox_metatype_importer.add_metatype' @@ -149,15 +100,6 @@ def post(self, request): branch = plugin_settings.get('branch') owner = plugin_settings.get('repo_owner') - self.related_object_forms.popitem() - self.related_object_forms.update( - { - 'module-bays': forms.ModuleBayTemplateImportForm, - 'device-bays': forms.DeviceBayTemplateImportForm, - 'inventory-items': forms.InventoryItemTemplateImportForm, - } - ) - gh_api = GitHubGqlAPI(token=token, owner=owner, repo=repo, branch=branch, path=self.type) query_data = {} @@ -204,7 +146,7 @@ def post(self, request): with transaction.atomic(): obj = model_form.save() - for field_name, related_object_form in self.related_object_forms.items(): + for field_name, related_object_form in related_object_forms().items(): related_obj_pks = [] for i, rel_obj_data in enumerate(data.get(field_name, list())): rel_obj_data.update({self.related_object: obj}) diff --git a/setup.py b/setup.py index ec8de33..59b1596 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='netbox-metatype-importer', - version='0.3.0', + version='0.3.1', description='Easily import Device and Module types from GitHub repo', long_description='Import MetaTypes into NetBox', long_description_content_type="text/markdown",