fix: add Azure Blob Provisioner extension (#1536)

* add dedicated cloud e2e test module

* added test without container

* add s3 provision and e2e test (wip)

* backport/copy upstream AZ Blob Provisioner

* add s3 destination credentials

* fix lic header format

* avoid NPE in the generator

* DEPENDENCIES

* re-enable FCC crawling in tests

.github/workflows/verify.yaml

@@ -149,3 +149,8 @@ jobs:
         run: |
           ./gradlew compileJava compileTestJava
           ./gradlew -p edc-tests/edc-dataplane test -DincludeTags="CloudTransferTest"
+
+      - name: Run Azure/S3 End-To-End tests
+        run: |
+          ./gradlew compileJava compileTestJava
+          ./gradlew -p edc-tests/edc-end2end test -DincludeTags="EndToEndTest"

edc-controlplane/edc-controlplane-base/build.gradle.kts

@@ -79,6 +79,9 @@ dependencies {
     runtimeOnly(libs.edc.controlplane.callback.dispatcher.event)
     runtimeOnly(libs.edc.controlplane.callback.dispatcher.http)
 
+    // cloud provisioner extensions
+    runtimeOnly(project(":edc-extensions:backport:azblob-provisioner"))
+
     // Federated Catalog Crawler + Query API
     runtimeOnly(project(":edc-extensions:federated-catalog"))
     runtimeOnly(libs.edc.fc.core)

edc-extensions/backport/README.md

@@ -0,0 +1,2 @@
+All modules in this directory are only temporary and they should be replaced with their upstream counterparts at the
+earliest opportunity. changes made here should be upstreamed ASAP, lest we run the risk of feature divergence!

edc-extensions/backport/azblob-provisioner/build.gradle.kts

@@ -0,0 +1,29 @@
+/*
+ *  Copyright (c) 2020, 2021 Microsoft Corporation
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Microsoft Corporation - initial API and implementation
+ *
+ */
+
+plugins {
+    `java-library`
+}
+
+dependencies {
+    api(libs.edc.spi.core)
+    api(libs.edc.azure.blob.core)
+
+    implementation(libs.azure.storage.blob)
+//    implementation(libs.failsafe.core)
+
+    testImplementation(testFixtures(libs.edc.azure.test))
+}
+
+

edc-extensions/backport/azblob-provisioner/src/main/java/org/eclipse/edc/connector/provision/azure/AzureProvisionExtension.java

@@ -0,0 +1,77 @@
+/********************************************************************************
+ * Copyright (c) 2020,2021 Microsoft Corporation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+package org.eclipse.edc.connector.provision.azure;
+
+import dev.failsafe.RetryPolicy;
+import org.eclipse.edc.azure.blob.AzureSasToken;
+import org.eclipse.edc.azure.blob.api.BlobStoreApi;
+import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ProvisionManager;
+import org.eclipse.edc.connector.controlplane.transfer.spi.provision.Provisioner;
+import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ResourceManifestGenerator;
+import org.eclipse.edc.connector.provision.azure.blob.ObjectContainerProvisionedResource;
+import org.eclipse.edc.connector.provision.azure.blob.ObjectStorageConsumerResourceDefinitionGenerator;
+import org.eclipse.edc.connector.provision.azure.blob.ObjectStorageProvisioner;
+import org.eclipse.edc.connector.provision.azure.blob.ObjectStorageResourceDefinition;
+import org.eclipse.edc.runtime.metamodel.annotation.Inject;
+import org.eclipse.edc.spi.system.ServiceExtension;
+import org.eclipse.edc.spi.system.ServiceExtensionContext;
+import org.eclipse.edc.spi.types.TypeManager;
+
+/**
+ * Provides data transfer {@link Provisioner}s backed by Azure services.
+ */
+public class AzureProvisionExtension implements ServiceExtension {
+
+    @Inject
+    private BlobStoreApi blobStoreApi;
+
+    @Inject
+    private RetryPolicy<Object> retryPolicy;
+
+    @Inject
+    private ResourceManifestGenerator manifestGenerator;
+
+    @Inject
+    private TypeManager typeManager;
+
+    @Override
+    public String name() {
+        return "Azure Provision";
+    }
+
+    @Override
+    public void initialize(ServiceExtensionContext context) {
+
+        var monitor = context.getMonitor();
+        var provisionManager = context.getService(ProvisionManager.class);
+
+        provisionManager.register(new ObjectStorageProvisioner(retryPolicy, monitor, blobStoreApi));
+
+        // register the generator
+        manifestGenerator.registerGenerator(new ObjectStorageConsumerResourceDefinitionGenerator());
+
+        registerTypes(typeManager);
+    }
+
+    private void registerTypes(TypeManager typeManager) {
+        typeManager.registerTypes(ObjectContainerProvisionedResource.class, ObjectStorageResourceDefinition.class, AzureSasToken.class);
+    }
+
+}

edc-extensions/backport/azblob-provisioner/src/main/java/org/eclipse/edc/connector/provision/azure/blob/ObjectContainerProvisionedResource.java

@@ -0,0 +1,86 @@
+/********************************************************************************
+ * Copyright (c) 2020,2021 Microsoft Corporation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+package org.eclipse.edc.connector.provision.azure.blob;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
+import org.eclipse.edc.azure.blob.AzureBlobStoreSchema;
+import org.eclipse.edc.connector.controlplane.transfer.spi.types.ProvisionedDataDestinationResource;
+
+import static org.eclipse.edc.azure.blob.AzureBlobStoreSchema.ACCOUNT_NAME;
+import static org.eclipse.edc.azure.blob.AzureBlobStoreSchema.CONTAINER_NAME;
+import static org.eclipse.edc.azure.blob.AzureBlobStoreSchema.FOLDER_NAME;
+import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE;
+
+@JsonDeserialize(builder = ObjectContainerProvisionedResource.Builder.class)
+@JsonTypeName("dataspaceconnector:objectcontainerprovisionedresource")
+public class ObjectContainerProvisionedResource extends ProvisionedDataDestinationResource {
+
+    private ObjectContainerProvisionedResource() {
+    }
+
+    public String getAccountName() {
+        return getDataAddress().getStringProperty(ACCOUNT_NAME);
+    }
+
+    public String getContainerName() {
+        return getDataAddress().getStringProperty(CONTAINER_NAME);
+    }
+
+    @JsonPOJOBuilder(withPrefix = "")
+    public static class Builder extends ProvisionedDataDestinationResource.Builder<ObjectContainerProvisionedResource, Builder> {
+
+        private Builder() {
+            super(new ObjectContainerProvisionedResource());
+            dataAddressBuilder.type(AzureBlobStoreSchema.TYPE);
+        }
+
+        @JsonCreator
+        public static Builder newInstance() {
+            return new Builder();
+        }
+
+        public Builder accountName(String accountName) {
+            dataAddressBuilder.property(EDC_NAMESPACE + ACCOUNT_NAME, accountName);
+            return this;
+        }
+
+        public Builder containerName(String containerName) {
+            dataAddressBuilder.property(EDC_NAMESPACE + CONTAINER_NAME, containerName);
+            return this;
+        }
+
+        @Override
+        public Builder resourceName(String name) {
+            dataAddressBuilder.keyName(name);
+            super.resourceName(name);
+            return this;
+        }
+
+        public Builder folderName(String folderName) {
+            if (folderName != null) {
+                dataAddressBuilder.property(EDC_NAMESPACE + FOLDER_NAME, folderName);
+            }
+            return this;
+        }
+    }
+}

edc-extensions/backport/azblob-provisioner/src/main/java/org/eclipse/edc/connector/provision/azure/blob/ObjectStorageConsumerResourceDefinitionGenerator.java

@@ -0,0 +1,59 @@
+/********************************************************************************
+ * Copyright (c) 2020,2021 Microsoft Corporation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+package org.eclipse.edc.connector.provision.azure.blob;
+
+import org.eclipse.edc.azure.blob.AzureBlobStoreSchema;
+import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ConsumerResourceDefinitionGenerator;
+import org.eclipse.edc.connector.controlplane.transfer.spi.types.ResourceDefinition;
+import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcess;
+import org.eclipse.edc.policy.model.Policy;
+import org.eclipse.edc.spi.types.domain.DataAddress;
+import org.jetbrains.annotations.Nullable;
+
+import static java.util.Optional.ofNullable;
+import static java.util.UUID.randomUUID;
+
+public class ObjectStorageConsumerResourceDefinitionGenerator implements ConsumerResourceDefinitionGenerator {
+
+    @Override
+    public @Nullable ResourceDefinition generate(TransferProcess transferProcess, Policy policy) {
+        var destination = transferProcess.getDataDestination();
+        var id = randomUUID().toString();
+        var account = destination.getStringProperty(AzureBlobStoreSchema.ACCOUNT_NAME);
+        var container = destination.getStringProperty(AzureBlobStoreSchema.CONTAINER_NAME);
+        var folderName = destination.getStringProperty(AzureBlobStoreSchema.FOLDER_NAME);
+
+        if (container == null) {
+            container = randomUUID().toString();
+        }
+        return ObjectStorageResourceDefinition.Builder.newInstance()
+                .id(id)
+                .accountName(account)
+                .containerName(container)
+                .folderName(folderName)
+                .build();
+    }
+
+    @Override
+    public boolean canGenerate(TransferProcess dataRequest, Policy policy) {
+        var type = ofNullable(dataRequest.getDataDestination()).map(DataAddress::getType).orElse(null);
+        return AzureBlobStoreSchema.TYPE.equals(type);
+    }
+}