Add functionality for installing extensions

+matnwb/+extension/+internal/buildRepoDownloadUrl.m

@@ -0,0 +1,25 @@
+function downloadUrl = buildRepoDownloadUrl(repositoryUrl, branchName)
+% buildRepoDownloadUrl - Build a download URL for a given repository and branch
+    arguments
+        repositoryUrl (1,1) string
+        branchName (1,1) string
+    end
+
+    if endsWith(repositoryUrl, '/')
+        repositoryUrl = extractBefore(repositoryUrl, strlength(repositoryUrl));
+    end
+
+    if contains(repositoryUrl, 'github.com')
+        downloadUrl = sprintf( '%s/archive/refs/heads/%s.zip', repositoryUrl, branchName );
+
+    elseif contains(repositoryUrl, 'gitlab.com')
+        repoPathSegments = strsplit(repositoryUrl, '/');
+        repoName = repoPathSegments{end};
+        downloadUrl = sprintf( '%s/-/archive/%s/%s-%s.zip', ...
+            repositoryUrl, branchName, repoName, branchName);
+
+    else
+        error('NWB:BuildRepoDownloadUrl:UnsupportedRepository', ...
+            'Expected repository URL to point to a GitHub or a GitLab repository')
+    end
+end

+matnwb/+extension/+internal/downloadExtensionRepository.m

@@ -0,0 +1,46 @@
+function [wasDownloaded, repoTargetFolder] = downloadExtensionRepository(...
+    repositoryUrl, repoTargetFolder, extensionName)
+% downloadExtensionRepository - Download the repository (source) for an extension
+%
+%   The metadata for a neurodata extension only provides the url to the
+%   repository containing the extension, not the full download url. This
+%   function tries to download a zipped version of the repository from
+%   either the "main" or the "master" branch.
+%
+%   Works for repositories located on GitHub or GitLab
+%
+%   As of Dec. 2024, this approach works for all registered extensions
+
+    arguments
+        repositoryUrl (1,1) string
+        repoTargetFolder (1,1) string
+        extensionName (1,1) string
+    end
+
+    import matnwb.extension.internal.downloadZippedRepo
+    import matnwb.extension.internal.buildRepoDownloadUrl
+
+    defaultBranchNames = ["main", "master"];
+
+    wasDownloaded = false;
+    for i = 1:2
+        try
+            branchName = defaultBranchNames(i);
+            downloadUrl = buildRepoDownloadUrl(repositoryUrl, branchName);
+            repoTargetFolder = downloadZippedRepo(downloadUrl, repoTargetFolder);
+            wasDownloaded = true;
+            break
+        catch ME
+            if strcmp(ME.identifier, 'MATLAB:webservices:HTTP404StatusCodeError')
+                continue
+            elseif strcmp(ME.identifier, 'NWB:BuildRepoDownloadUrl:UnsupportedRepository')
+                error('NWB:InstallExtension:UnsupportedRepository', ...
+                    ['Extension "%s" is located in an unsupported repository ', ...
+                     '/ source location. \nPlease create an issue on MatNWB''s ', ...
+                     'github page'], extensionName)
+            else
+                rethrow(ME)
+            end
+        end
+    end
+end

+matnwb/+extension/+internal/downloadZippedRepo.m

@@ -0,0 +1,37 @@
+function repoFolder = downloadZippedRepo(githubUrl, targetFolder)
+%downloadZippedRepo - Download a zipped repository
+
+    % Create a temporary path for storing the downloaded file.
+    [~, ~, fileType] = fileparts(githubUrl);
+    tempFilepath = [tempname, fileType];
+
+    % Download the file containing the zipped repository
+    tempFilepath = websave(tempFilepath, githubUrl);
+    fileCleanupObj = onCleanup( @(fname) delete(tempFilepath) );
+
+    unzippedFiles = unzip(tempFilepath, tempdir);
+    unzippedFolder = unzippedFiles{1};
+    if endsWith(unzippedFolder, filesep)
+        unzippedFolder = unzippedFolder(1:end-1);
+    end
+
+    [~, repoFolderName] = fileparts(unzippedFolder);
+    targetFolder = fullfile(targetFolder, repoFolderName);
+
+    if isfolder(targetFolder)
+        try
+            rmdir(targetFolder, 's')
+        catch
+            error('Could not delete previously downloaded extension which is located at:\n"%s"', targetFolder)
+        end
+    else
+        % pass
+    end
+
+    movefile(unzippedFolder, targetFolder);
+
+    % Delete the temp zip file
+    clear fileCleanupObj
+
+    repoFolder = targetFolder;
+end

+matnwb/+extension/getExtensionInfo.m

@@ -0,0 +1,16 @@
+function metadata = getExtensionInfo(extensionName)
+% getExtensionInfo - Get metadata for specified extension
+
+    arguments
+        extensionName (1,1) string
+    end
+
+    T = matnwb.extension.listExtensions();
+    isMatch = T.name == extensionName;
+    extensionList = join( compose("  %s", [T.name]), newline );
+    assert( ...
+        any(isMatch), ...
+        'NWB:DisplayExtensionMetadata:ExtensionNotFound', ...
+        'Extension "%s" was not found in the extension catalog:\n%s', extensionName, extensionList)
+    metadata = table2struct(T(isMatch, :));
+end

+matnwb/+extension/installAll.m

@@ -0,0 +1,6 @@
+function installAll()
+    T = matnwb.extension.listExtensions();
+    for i = 1:height(T)
+        matnwb.extension.installExtension( T.name(i) )
+    end
+end

+matnwb/+extension/installExtension.m

@@ -0,0 +1,49 @@
+function installExtension(extensionName, options)
+% installExtension - Install NWB extension from Neurodata Extensions Catalog
+%
+%   matnwb.extension.nwbInstallExtension(extensionName) installs a Neurodata 
+%   Without Borders (NWB) extension from the Neurodata Extensions Catalog to 
+%   extend the functionality of the core NWB schemas. 
+
+    arguments
+        extensionName (1,1) string
+        options.savedir (1,1) string = misc.getMatnwbDir()
+    end
+
+    import matnwb.extension.internal.downloadExtensionRepository
+
+    repoTargetFolder = fullfile(userpath, "NWB-Extension-Source");
+    if ~isfolder(repoTargetFolder); mkdir(repoTargetFolder); end
+
+    T = matnwb.extension.listExtensions();
+    isMatch = T.name == extensionName;
+
+    extensionList = join( compose("  %s", [T.name]), newline );
+    assert( ...
+        any(isMatch), ...
+        'NWB:InstallExtension:ExtensionNotFound', ...
+        'Extension "%s" was not found in the extension catalog:\n', extensionList)
+
+    repositoryUrl = T{isMatch, 'src'};
+
+    [wasDownloaded, repoTargetFolder] = ...
+        downloadExtensionRepository(repositoryUrl, repoTargetFolder, extensionName);
+
+    if ~wasDownloaded
+        error('NWB:InstallExtension:DownloadFailed', ...
+            'Failed to download spec for extension "%s"', extensionName)
+    end
+    L = dir(fullfile(repoTargetFolder, 'spec', '*namespace.yaml'));
+    assert(...
+        ~isempty(L), ...
+        'NWB:InstallExtension:NamespaceNotFound', ...
+        'No namespace file was found for extension "%s"', extensionName ...
+        )
+    assert(...
+        numel(L)==1, ...
+        'NWB:InstallExtension:MultipleNamespacesFound', ...
+        'More than one namespace file was found for extension "%s"', extensionName ...
+        )
+    generateExtension( fullfile(L.folder, L.name), 'savedir', options.savedir );
+    fprintf("Installed extension ""%s"".\n", extensionName)
+end

+matnwb/+extension/listExtensions.m

@@ -0,0 +1,53 @@
+function extensionTable = listExtensions(options)
+    arguments
+        options.Refresh (1,1) logical = false
+    end
+
+    persistent extensionRecords
+
+    if isempty(extensionRecords) || options.Refresh
+        catalogUrl = "https://raw.githubusercontent.com/nwb-extensions/nwb-extensions.github.io/refs/heads/main/data/records.json";
+        extensionRecords = jsondecode(webread(catalogUrl));
+        extensionRecords = consolidateStruct(extensionRecords);
+
+        extensionRecords = struct2table(extensionRecords);
+
+        fieldsKeep = ["name", "version", "last_updated", "src", "license", "maintainers", "readme"];
+        extensionRecords = extensionRecords(:, fieldsKeep);
+
+        for name = fieldsKeep
+            if ischar(extensionRecords.(name){1})
+                extensionRecords.(name) = string(extensionRecords.(name));
+            end
+        end
+    end
+    extensionTable = extensionRecords;
+end
+
+function structArray = consolidateStruct(S)
+    % Get all field names of S
+    mainFields = fieldnames(S);
+
+    % Initialize an empty struct array
+    structArray = struct();
+
+    % Iterate over each field of S
+    for i = 1:numel(mainFields)
+        subStruct = S.(mainFields{i}); % Extract sub-struct
+
+        % Add all fields of the sub-struct to the struct array
+        fields = fieldnames(subStruct);
+        for j = 1:numel(fields)
+            structArray(i).(fields{j}) = subStruct.(fields{j});
+        end
+    end
+
+    % Ensure consistency by filling missing fields with []
+    allFields = unique([fieldnames(structArray)]);
+    for i = 1:numel(structArray)
+        missingFields = setdiff(allFields, fieldnames(structArray(i)));
+        for j = 1:numel(missingFields)
+            structArray(i).(missingFields{j}) = [];
+        end
+    end
+end

+tests/+unit/InstallExtensionTest.m

@@ -0,0 +1,87 @@
+classdef InstallExtensionTest < matlab.unittest.TestCase
+
+    methods (TestClassSetup)
+        function setupClass(testCase)
+            % Get the root path of the matnwb repository
+            rootPath = misc.getMatnwbDir();
+
+            % Use a fixture to add the folder to the search path
+            testCase.applyFixture(matlab.unittest.fixtures.PathFixture(rootPath));
+
+            % Use a fixture to create a temporary working directory
+            testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture);
+            generateCore('savedir', '.');
+        end
+    end
+
+    methods (Test)
+        function testInstallExtensionFailsWithNoInputArgument(testCase)
+            testCase.verifyError(...
+                @(varargin) nwbInstallExtension(), ...
+                'NWB:InstallExtension:MissingArgument')
+        end
+
+        function testInstallExtension(testCase)
+            nwbInstallExtension("ndx-miniscope", 'savedir', '.')
+
+            testCase.verifyTrue(isfolder('./+types/+ndx_miniscope'), ...
+                'Folder with extension types does not exist')
+        end
+
+        function testUseInstalledExtension(testCase)
+            nwbObject = testCase.initNwbFile();
+
+            miniscopeDevice = types.ndx_miniscope.Miniscope(...
+                'deviceType', 'test_device', ...
+                'compression', 'GREY', ...
+                'frameRate', '30fps', ...
+                'framesPerFile', int8(100) );
+
+            nwbObject.general_devices.set('TestMiniscope', miniscopeDevice);
+
+            testCase.verifyClass(nwbObject.general_devices.get('TestMiniscope'), ...
+                'types.ndx_miniscope.Miniscope')
+        end
+
+        function testGetExtensionMetadata(testCase)
+            extensionName = "ndx-miniscope";
+            metadata = matnwb.extension.getExtensionInfo(extensionName);
+            testCase.verifyClass(metadata, 'struct')
+            testCase.verifyEqual(metadata.name, extensionName)
+        end
+
+        function testDownloadUnknownRepository(testCase)
+            repositoryUrl = "https://www.unknown-repo.com/anon/my_nwb_extension";
+            testCase.verifyError(...
+                @() matnwb.extension.internal.downloadExtensionRepository(repositoryUrl, "", "my_nwb_extension"), ...
+                 'NWB:InstallExtension:UnsupportedRepository');
+        end
+
+        function testBuildRepoDownloadUrl(testCase)
+
+            import matnwb.extension.internal.buildRepoDownloadUrl
+
+            repoUrl = buildRepoDownloadUrl('https://github.com/user/test', 'main');
+            testCase.verifyEqual(repoUrl, 'https://github.com/user/test/archive/refs/heads/main.zip')
+
+            repoUrl = buildRepoDownloadUrl('https://github.com/user/test/', 'main');
+            testCase.verifyEqual(repoUrl, 'https://github.com/user/test/archive/refs/heads/main.zip')
+
+            repoUrl = buildRepoDownloadUrl('https://gitlab.com/user/test', 'main');
+            testCase.verifyEqual(repoUrl, 'https://gitlab.com/user/test/-/archive/main/test-main.zip')
+
+            testCase.verifyError(...
+                @() buildRepoDownloadUrl('https://unsupported.com/user/test', 'main'), ...
+                'NWB:BuildRepoDownloadUrl:UnsupportedRepository')
+        end
+    end
+
+    methods (Static)
+        function nwb = initNwbFile()
+            nwb = NwbFile( ...
+                'session_description', 'test file for nwb extension', ...
+                'identifier', 'export_test', ...
+                'session_start_time', datetime("now", 'TimeZone', 'local') );
+        end
+    end
+end

.github/workflows/update_extension_list.yml

@@ -0,0 +1,49 @@
+name: Update extension list
+
+on:
+  schedule:
+    # Run at 8:15 on working days [Minute Hour Day Month Weekdays]
+    # Run this 15 minutes after source repo is updated
+    # https://github.com/nwb-extensions/nwb-extensions.github.io/blob/main/.github/workflows/data.yml
+    - cron: 15 8 * * 0-5
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  update_extension_list:
+    runs-on: ubuntu-latest
+    steps:
+      # Use deploy key to push back to protected branch
+      - name: Checkout repository using deploy key
+        uses: actions/checkout@v4
+        with:
+          ref: refs/heads/main
+          ssh-key: ${{ secrets.DEPLOY_KEY }}
+
+      - name: Install MATLAB
+        uses: matlab-actions/setup-matlab@v2
+
+      - name: Update extension list in nwbInstallExtensions
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath(genpath("tools"));
+            matnwb_createNwbInstallExtension();
+
+      - name: Commit the updated nwbInstallExtension function
+        run: |
+          set -e  # Exit script on error
+          git config user.name "${{ github.workflow }} by ${{ github.actor }}"
+          git config user.email "<>"
+          git pull --rebase  # Ensure the branch is up-to-date
+
+          if [[ -n $(git status --porcelain nwbInstallExtension.m) ]]; then
+            git add nwbInstallExtension.m
+            git commit -m "Update list of extensions in nwbInstallExtension"
+            git push
+          else
+            echo "Nothing to commit"
+          fi
+

.gitignore

@@ -8,4 +8,14 @@ workspace/
 .DS_Store
 +tests/env.mat
 
+# Ignore everything in the +types/ folder
++types/*
+
+# Explicitly include these subdirectories
+!+types/+core/
+!+types/+hdmf_common/
+!+types/+hdmf_experimental/
+!+types/+untyped/
+!+types/+util/
+
 docs/build

docs/source/pages/functions/index.rst

@@ -13,3 +13,4 @@ These are the main functions of the MatNWB API
    generateCore
    generateExtension
    nwbClearGenerated
+   nwbInstallExtension