Support for static encryption of the content. (#105)

Add options to enabled encryption of the content.
Fixes #98.
Remove shaka packager messages from the console (Still logged in the
file)

AssetOptionsBinder.cs

@@ -40,14 +40,14 @@ This is specific to the cloud you are migrating to.
         {
             Arity = ArgumentArity.ZeroOrOne
         };
-                
+
         private readonly Option<string?> _outputManifest = new Option<string?>(
             aliases: new[] { "--output-manifest-name", "-m" },
             description: @"The output manifest name without extension,
 if it is not set, use input asset's manifest name.")
-                    {
-                        Arity = ArgumentArity.ZeroOrOne
-                    };
+        {
+            Arity = ArgumentArity.ZeroOrOne
+        };
 
         private readonly Option<DateTimeOffset?> _creationTimeStart = new Option<DateTimeOffset?>(
             aliases: new[] { "--creation-time-start", "-cs" },
@@ -75,9 +75,9 @@ This is specific to the cloud you are migrating to.
         };
 
         private readonly Option<Packager> _packagerType = new(
-    aliases: new[] { "--packager" },
-    () => Packager.Shaka,
-    description: "The packager to use.")
+            aliases: new[] { "--packager" },
+            () => Packager.Shaka,
+            description: "The packager to use.")
         {
             IsHidden = true,
             IsRequired = false
@@ -112,6 +112,23 @@ This is specific to the cloud you are migrating to.
             () => DefaultBatchSize,
             description: @"Batch size for parallel processing.");
 
+        private readonly Option<bool> _encryptContent = new (
+            aliases: new[] { "-e", "--encrypt-content" },
+            () => false,
+            description: "Encrypt the content  using CENC"
+            );
+
+        private readonly Option<Uri?> _keyVaultUri = new (
+            aliases: new[] { "--key-vault-uri" },
+            description: "The key vault to store encryption keys."
+            );
+
+        private readonly Option<string?> _keyUri = new(
+            aliases: new[] { "--key-uri" },
+            () => "/.clearkeys?kid=${KeyId}",
+            description: "The key URI to use for requesting the key. This is saved to the manifest."
+            );
+
         const int SegmentDurationInSeconds = 2;
 
         public AssetOptionsBinder()
@@ -125,7 +142,8 @@ public AssetOptionsBinder()
                 }
             });
 
-            _pathTemplate.AddValidator(result => {
+            _pathTemplate.AddValidator(result =>
+            {
                 var value = result.GetValueOrDefault<string>();
                 if (!string.IsNullOrEmpty(value))
                 {
@@ -154,15 +172,34 @@ public Command GetCommand(string name, string description)
             command.AddOption(_workingDirectory);
             command.AddOption(_copyNonStreamable);
             command.AddOption(_batchSize);
+            command.AddOption(_encryptContent);
+            command.AddOption(_keyVaultUri);
+            command.AddOption(_keyUri);
+            command.AddValidator(result =>
+            {
+                if (result.GetValueForOption(_encryptContent))
+                {
+                    if (result.FindResultFor(_keyVaultUri) == null || result.FindResultFor(_keyUri) == null)
+                    {
+                        result.ErrorMessage = "Key vault and key URI must be specified when encryption is enabled.";
+                    }
+                    else if (result.GetValueForOption(_packagerType) != Packager.Shaka)
+                    {
+                        result.ErrorMessage = "Static encryption is only supported with shaka packager.";
+                    }
+                }
+            });
             return command;
         }
 
-        protected override AssetOptions GetBoundValue(BindingContext bindingContext)
+        protected override AssetOptions GetBoundValue(BindingContext bindingContext) => GetValue(bindingContext);
+
+        public AssetOptions GetValue(BindingContext bindingContext)
         {
             var workingDirectory = bindingContext.ParseResult.GetValueForOption(_workingDirectory)!;
             Directory.CreateDirectory(workingDirectory);
             return new AssetOptions(
-                bindingContext.ParseResult.GetValueForOption(_sourceAccount)!,               
+                bindingContext.ParseResult.GetValueForOption(_sourceAccount)!,
                 bindingContext.ParseResult.GetValueForOption(_storageAccount)!,
                 bindingContext.ParseResult.GetValueForOption(_packagerType),
                 bindingContext.ParseResult.GetValueForOption(_pathTemplate)!,
@@ -176,9 +213,12 @@ protected override AssetOptions GetBoundValue(BindingContext bindingContext)
                 bindingContext.ParseResult.GetValueForOption(_skipMigrated),
                 SegmentDurationInSeconds,
                 bindingContext.ParseResult.GetValueForOption(_batchSize)
-            );
+            )
+            {
+                EncryptContent = bindingContext.ParseResult.GetValueForOption(_encryptContent),
+                KeyUri = bindingContext.ParseResult.GetValueForOption(_keyUri),
+                KeyVaultUri = bindingContext.ParseResult.GetValueForOption(_keyVaultUri)
+            };
         }
-
-        public AssetOptions GetValue(BindingContext bindingContext) => GetBoundValue(bindingContext);
     }
 }

Program.cs

@@ -17,6 +17,9 @@
 using System.Diagnostics;
 using System.Text;
 using Vertical.SpectreLogger;
+using Vertical.SpectreLogger.Core;
+using Vertical.SpectreLogger.Options;
+using Events = AMSMigrate.Contracts.Events;
 
 namespace AMSMigrate
 {
@@ -158,11 +161,14 @@ static void SetupServices(IServiceCollection collection, GlobalOptions options,
                     {
                         Level = SourceLevels.All
                     };
+                    LogEventFilterDelegate filter = (in LogEventContext context) => context.EventId != Events.ShakaPackager;
                     builder
                         .SetMinimumLevel(LogLevel.Trace)
                         .AddSpectreConsole(builder =>
                             builder
                                 .SetMinimumLevel(options.LogLevel)
+                                .SetLogEventFilter(filter)
+                                .UseSerilogConsoleStyle()
                                 .UseConsole(console)
                                 .WriteInBackground())
                         .AddTraceSource(logSwitch, listener);

README.md

@@ -40,6 +40,7 @@ The content is converted to CMAF format with both a DASH and HLS manifest to sup
 * Support for packaging VOD assets.
 * Support for copying non-streamable assets.
 * Marks migrated assets and provides HTML summary on analyze
+* Support for statically encrypting the cotnent while packaging.
 
 ## Open Issues
 * Live assets are not supported but will be in a future version of this tool.
@@ -63,6 +64,7 @@ You'll need to have the following permissions:
 
 - The identity used to migrate must have 'Contributor' role on the Azure Media Services account being migrated.
 - The identity that runs this migration tool should be added to the ['Storage Blob Data Contributor'](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor) role for the source and destination storage accounts.
+- The identity used to migrate must have the role ['Key Vault Secrets Officer'](https://learn.microsoft.com/en-us/azure/key-vault/general/rbac-guide?tabs=azure-cli#azure-built-in-roles-for-key-vault-data-plane-operations) on the key vault used to store the keys. If the key vault is not using Azure RBAC then you will have to create an Access policy giving secrets management permission to the identity used.
 
 # Quick Start
 

ams/AmsExtensions.cs

@@ -2,6 +2,8 @@
 using Azure.ResourceManager.Media;
 using Azure.Storage.Blobs;
 using Azure;
+using AMSMigrate.Transform;
+using AMSMigrate.Contracts;
 
 namespace AMSMigrate.Ams
 {
@@ -73,5 +75,12 @@ public static async Task CreateStreamingLocator(
                 StreamingPolicyName = "migration"
             });
         }
+
+        public static void GetEncryptionDetails(this AssetDetails details, MigratorOptions options, TemplateMapper templateMapper)
+        {
+            details.EncryptionKey = Guid.NewGuid().ToString("n");
+            details.KeyId = Guid.NewGuid().ToString("n");
+            details.LicenseUri = templateMapper.ExpandKeyUriTemplate(options.KeyUri!, details.KeyId);
+        }
     }
 }

ams/TemplateMapper.cs

@@ -164,6 +164,14 @@ public string ExpandKeyTemplate(StreamingLocatorContentKey contentKey, string? t
             });
         }
 
+        public string ExpandKeyUriTemplate(string uriTemplate, string  keyId)
+        {
+            return ExpandTemplate(uriTemplate, key => key switch {
+                "KeyId" => keyId,
+                _ => null
+            });
+        }
+
         private async Task<string> GetLocatorIdAsync(MediaAssetResource asset)
         {
             var locators = asset.GetStreamingLocatorsAsync();

azure/AzureProvider.cs

@@ -20,7 +20,7 @@ public AzureProvider(
         public IFileUploader GetStorageProvider(MigratorOptions migratorOptions) 
             => new AzureStorageUploader(migratorOptions, _credentials, _loggerFactory.CreateLogger<AzureStorageUploader>());
 
-        public ISecretUploader GetSecretProvider(KeyOptions keyOptions) 
+        public ISecretUploader GetSecretProvider(KeyVaultOptions keyOptions) 
             => new KeyVaultUploader(keyOptions, _credentials, _loggerFactory.CreateLogger<KeyVaultUploader>());
     }
 }

azure/AzureStorageUploader.cs

@@ -7,7 +7,6 @@
 using Azure.Storage.Blobs.Models;
 using Azure.Storage.Blobs.Specialized;
 using Microsoft.Extensions.Logging;
-using System.Reflection.PortableExecutable;
 using System.Text;
 
 namespace AMSMigrate.Azure

azure/KeyVaultUploader.cs

@@ -9,21 +9,21 @@ internal class KeyVaultUploader : ISecretUploader
     {
         private readonly ILogger _logger;
         private readonly SecretClient _secretClient;
-        private readonly KeyOptions _keyOptions;
+        private readonly KeyVaultOptions _options;
 
         public KeyVaultUploader(
-            KeyOptions options,
+            KeyVaultOptions options,
             TokenCredential credential,
             ILogger<KeyVaultUploader> logger)
         {
-            _keyOptions = options;
+            _options = options;
             _logger = logger;
             _secretClient = new SecretClient(options.KeyVaultUri, credential);
         }
 
         public async Task UploadAsync(string secretName, string secretValue, CancellationToken cancellationToken)
         {
-            _logger.LogInformation("Saving secret {name} to key vault {vault}", secretName, _keyOptions.KeyVaultUri);
+            _logger.LogInformation("Saving secret {name} to key vault {vault}", secretName, _options.KeyVaultUri);
             await _secretClient.SetSecretAsync(secretName, secretValue, cancellationToken);
         }
     }

contracts/Events.cs

@@ -0,0 +1,8 @@
+namespace AMSMigrate.Contracts
+{
+    internal class Events
+    {
+        public const int ShakaPackager = 101;
+        public const int Ffmpeg = 102;
+    }
+}

contracts/ICloudProvider.cs

@@ -5,6 +5,6 @@ interface ICloudProvider
     {
         IFileUploader GetStorageProvider(MigratorOptions migratorOptions);
 
-        ISecretUploader GetSecretProvider(KeyOptions keyOptions);
+        ISecretUploader GetSecretProvider(KeyVaultOptions keyOptions);
     }
 }

contracts/KeyOptions.cs

@@ -1,9 +1,11 @@
 namespace AMSMigrate
 {
+    public record KeyVaultOptions(Uri KeyVaultUri);
+
     public record KeyOptions(
         string AccountName,
         string? ResourceFilter,
         Uri KeyVaultUri,
         string? KeyTemplate,
-        int BatchSize);
+        int BatchSize) : KeyVaultOptions(KeyVaultUri);
 }

contracts/MigratorOptions.cs

@@ -14,5 +14,12 @@ public record MigratorOptions(
         bool OverWrite,
         bool SkipMigrated,
         int SegmentDuration,
-        int BatchSize);
+        int BatchSize)
+    {
+        public bool EncryptContent { get; set; }
+
+        public string? KeyUri { get; set; }
+
+        public Uri? KeyVaultUri { get; set; }
+    }
 }

decryption/AssetDecryptor.cs

@@ -5,7 +5,6 @@
 //-----------------------------------------------------------------------
 
 using System.Security.Cryptography;
-using System.Threading;
 using Azure.ResourceManager.Media.Models;
 using Azure.Storage.Blobs.Models;
 using Azure.Storage.Blobs.Specialized;

local/LocalFileProvider.cs

@@ -12,7 +12,7 @@ public LocalFileProvider(ILoggerFactory loggerFactory)
             _loggerFactory = loggerFactory;
         }
 
-        public ISecretUploader GetSecretProvider(KeyOptions keyOptions)
+        public ISecretUploader GetSecretProvider(KeyVaultOptions keyOptions)
         {
             throw new NotImplementedException();
         }

transform/AssetTransform.cs

@@ -1,6 +1,5 @@
 using AMSMigrate.Ams;
 using AMSMigrate.Contracts;
-using AMSMigrate.Decryption;
 using Azure.ResourceManager.Media;
 using Azure.ResourceManager.Media.Models;
 using Azure.Storage.Blobs;

transform/PackageTransform.cs

@@ -9,16 +9,22 @@ namespace AMSMigrate.Transform
     internal class PackageTransform : StorageTransform
     {
         private readonly PackagerFactory _packagerFactory;
+        private readonly ISecretUploader? _secretUploader = default;
 
         public PackageTransform(
             GlobalOptions globalOptions,
             MigratorOptions options,
             ILogger<PackageTransform> logger,
             TemplateMapper templateMapper,
-            IFileUploader uploader,
+            ICloudProvider cloudProvider,
             PackagerFactory factory)
-            : base(globalOptions, options, templateMapper, uploader, logger)
+            : base(globalOptions, options, templateMapper, cloudProvider.GetStorageProvider(options), logger)
         {
+            if (options.EncryptContent)
+            {
+                var vaultOptions = new KeyVaultOptions(options.KeyVaultUri!);
+                _secretUploader = cloudProvider.GetSecretProvider(vaultOptions);
+            }
             _packagerFactory = factory;
         }
 
@@ -44,6 +50,12 @@ protected override async Task<string> TransformAsync(
             using var source = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
             cancellationToken = source.Token;
 
+            if (_options.EncryptContent)
+            {
+                details.GetEncryptionDetails(_options, _templateMapper);
+            }
+
+
             // temporary space for either pipes or files.
             var workingDirectory = Path.Combine(_options.WorkingDirectory, assetName);
             Directory.CreateDirectory(workingDirectory);
@@ -63,10 +75,8 @@ protected override async Task<string> TransformAsync(
                 var blobs = await container.GetListOfBlobsRemainingAsync(manifest, cancellationToken);
                 allTasks.Add(Task.WhenAll(blobs.Select(async blob =>
                 {
-                    using (AesCtrTransform? aesTransform = AssetDecryptor.GetAesCtrTransform(details.DecryptInfo, blob.Name, false)) 
-                    {
-                        await UploadBlobAsync(blob, aesTransform, outputPath, cancellationToken);
-                    }
+                    using var aesTransform = AssetDecryptor.GetAesCtrTransform(details.DecryptInfo, blob.Name, false);
+                    await UploadBlobAsync(blob, aesTransform, outputPath, cancellationToken);
                 })));
 
                 if (packager.UsePipeForInput)
@@ -144,6 +154,12 @@ protected override async Task<string> TransformAsync(
                 // Upload any files pending to be uploaded.
                 await Task.WhenAll(
                     uploadPaths.Select(file => UploadFile(file, uploadHelper, cancellationToken)));
+
+                if (_options.EncryptContent)
+                {
+                    _logger.LogDebug("Saving key with id id: {keyId} for asset: {name} to key vault {vault}", details.KeyId, details.AssetName, _options.KeyVaultUri); ;
+                    await _secretUploader!.UploadAsync(details.KeyId, details.EncryptionKey, cancellationToken);
+                }
             }
             catch (Exception ex)
             {
@@ -171,8 +187,23 @@ private UploadPipe CreateUpload(string filePath, UploadHelper helper)
         private async Task UploadFile(string filePath, UploadHelper uploadHelper, CancellationToken cancellationToken)
         {
             var file = Path.GetFileName(filePath);
+            // Report update for every 1MB.
+            long update = 0;
             var progress = new Progress<long>(p =>
-                _logger.LogTrace("Uploaded {bytes} bytes to {file}", p, file));
+            {
+                if (p >= update)
+                {
+                    lock (this)
+                    {
+                        if (p >= update)
+                        {
+                            _logger.LogTrace("Uploaded {byte} bytes to {file}", p, file);
+                            update += 1024 * 1024;
+                        }
+                    }
+                }
+            });
+
             using var content = File.OpenRead(filePath);
             await uploadHelper.UploadAsync(Path.GetFileName(file), content, GetHeaders(file), progress, cancellationToken);
         }