diff --git a/AssetOptionsBinder.cs b/AssetOptionsBinder.cs index 1a9701f..5692124 100644 --- a/AssetOptionsBinder.cs +++ b/AssetOptionsBinder.cs @@ -40,14 +40,14 @@ This is specific to the cloud you are migrating to. { Arity = ArgumentArity.ZeroOrOne }; - + private readonly Option _outputManifest = new Option( aliases: new[] { "--output-manifest-name", "-m" }, description: @"The output manifest name without extension, if it is not set, use input asset's manifest name.") - { - Arity = ArgumentArity.ZeroOrOne - }; + { + Arity = ArgumentArity.ZeroOrOne + }; private readonly Option _creationTimeStart = new Option( aliases: new[] { "--creation-time-start", "-cs" }, @@ -75,9 +75,9 @@ This is specific to the cloud you are migrating to. }; private readonly Option _packagerType = new( - aliases: new[] { "--packager" }, - () => Packager.Shaka, - description: "The packager to use.") + aliases: new[] { "--packager" }, + () => Packager.Shaka, + description: "The packager to use.") { IsHidden = true, IsRequired = false @@ -112,6 +112,23 @@ This is specific to the cloud you are migrating to. () => DefaultBatchSize, description: @"Batch size for parallel processing."); + private readonly Option _encryptContent = new ( + aliases: new[] { "-e", "--encrypt-content" }, + () => false, + description: "Encrypt the content using CENC" + ); + + private readonly Option _keyVaultUri = new ( + aliases: new[] { "--key-vault-uri" }, + description: "The key vault to store encryption keys." + ); + + private readonly Option _keyUri = new( + aliases: new[] { "--key-uri" }, + () => "/.clearkeys?kid=${KeyId}", + description: "The key URI to use for requesting the key. This is saved to the manifest." + ); + const int SegmentDurationInSeconds = 2; public AssetOptionsBinder() @@ -125,7 +142,8 @@ public AssetOptionsBinder() } }); - _pathTemplate.AddValidator(result => { + _pathTemplate.AddValidator(result => + { var value = result.GetValueOrDefault(); if (!string.IsNullOrEmpty(value)) { @@ -154,15 +172,34 @@ public Command GetCommand(string name, string description) command.AddOption(_workingDirectory); command.AddOption(_copyNonStreamable); command.AddOption(_batchSize); + command.AddOption(_encryptContent); + command.AddOption(_keyVaultUri); + command.AddOption(_keyUri); + command.AddValidator(result => + { + if (result.GetValueForOption(_encryptContent)) + { + if (result.FindResultFor(_keyVaultUri) == null || result.FindResultFor(_keyUri) == null) + { + result.ErrorMessage = "Key vault and key URI must be specified when encryption is enabled."; + } + else if (result.GetValueForOption(_packagerType) != Packager.Shaka) + { + result.ErrorMessage = "Static encryption is only supported with shaka packager."; + } + } + }); return command; } - protected override AssetOptions GetBoundValue(BindingContext bindingContext) + protected override AssetOptions GetBoundValue(BindingContext bindingContext) => GetValue(bindingContext); + + public AssetOptions GetValue(BindingContext bindingContext) { var workingDirectory = bindingContext.ParseResult.GetValueForOption(_workingDirectory)!; Directory.CreateDirectory(workingDirectory); return new AssetOptions( - bindingContext.ParseResult.GetValueForOption(_sourceAccount)!, + bindingContext.ParseResult.GetValueForOption(_sourceAccount)!, bindingContext.ParseResult.GetValueForOption(_storageAccount)!, bindingContext.ParseResult.GetValueForOption(_packagerType), bindingContext.ParseResult.GetValueForOption(_pathTemplate)!, @@ -176,9 +213,12 @@ protected override AssetOptions GetBoundValue(BindingContext bindingContext) bindingContext.ParseResult.GetValueForOption(_skipMigrated), SegmentDurationInSeconds, bindingContext.ParseResult.GetValueForOption(_batchSize) - ); + ) + { + EncryptContent = bindingContext.ParseResult.GetValueForOption(_encryptContent), + KeyUri = bindingContext.ParseResult.GetValueForOption(_keyUri), + KeyVaultUri = bindingContext.ParseResult.GetValueForOption(_keyVaultUri) + }; } - - public AssetOptions GetValue(BindingContext bindingContext) => GetBoundValue(bindingContext); } } diff --git a/Program.cs b/Program.cs index 44e22b3..ab16702 100644 --- a/Program.cs +++ b/Program.cs @@ -17,6 +17,9 @@ using System.Diagnostics; using System.Text; using Vertical.SpectreLogger; +using Vertical.SpectreLogger.Core; +using Vertical.SpectreLogger.Options; +using Events = AMSMigrate.Contracts.Events; namespace AMSMigrate { @@ -158,11 +161,14 @@ static void SetupServices(IServiceCollection collection, GlobalOptions options, { Level = SourceLevels.All }; + LogEventFilterDelegate filter = (in LogEventContext context) => context.EventId != Events.ShakaPackager; builder .SetMinimumLevel(LogLevel.Trace) .AddSpectreConsole(builder => builder .SetMinimumLevel(options.LogLevel) + .SetLogEventFilter(filter) + .UseSerilogConsoleStyle() .UseConsole(console) .WriteInBackground()) .AddTraceSource(logSwitch, listener); diff --git a/README.md b/README.md index 744931b..5ef1471 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ The content is converted to CMAF format with both a DASH and HLS manifest to sup * Support for packaging VOD assets. * Support for copying non-streamable assets. * Marks migrated assets and provides HTML summary on analyze +* Support for statically encrypting the cotnent while packaging. ## Open Issues * Live assets are not supported but will be in a future version of this tool. @@ -63,6 +64,7 @@ You'll need to have the following permissions: - The identity used to migrate must have 'Contributor' role on the Azure Media Services account being migrated. - The identity that runs this migration tool should be added to the ['Storage Blob Data Contributor'](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor) role for the source and destination storage accounts. +- The identity used to migrate must have the role ['Key Vault Secrets Officer'](https://learn.microsoft.com/en-us/azure/key-vault/general/rbac-guide?tabs=azure-cli#azure-built-in-roles-for-key-vault-data-plane-operations) on the key vault used to store the keys. If the key vault is not using Azure RBAC then you will have to create an Access policy giving secrets management permission to the identity used. # Quick Start diff --git a/ams/AmsExtensions.cs b/ams/AmsExtensions.cs index ba9e000..2d256d4 100644 --- a/ams/AmsExtensions.cs +++ b/ams/AmsExtensions.cs @@ -2,6 +2,8 @@ using Azure.ResourceManager.Media; using Azure.Storage.Blobs; using Azure; +using AMSMigrate.Transform; +using AMSMigrate.Contracts; namespace AMSMigrate.Ams { @@ -73,5 +75,12 @@ public static async Task CreateStreamingLocator( StreamingPolicyName = "migration" }); } + + public static void GetEncryptionDetails(this AssetDetails details, MigratorOptions options, TemplateMapper templateMapper) + { + details.EncryptionKey = Guid.NewGuid().ToString("n"); + details.KeyId = Guid.NewGuid().ToString("n"); + details.LicenseUri = templateMapper.ExpandKeyUriTemplate(options.KeyUri!, details.KeyId); + } } } diff --git a/ams/TemplateMapper.cs b/ams/TemplateMapper.cs index 4f63021..2655163 100644 --- a/ams/TemplateMapper.cs +++ b/ams/TemplateMapper.cs @@ -164,6 +164,14 @@ public string ExpandKeyTemplate(StreamingLocatorContentKey contentKey, string? t }); } + public string ExpandKeyUriTemplate(string uriTemplate, string keyId) + { + return ExpandTemplate(uriTemplate, key => key switch { + "KeyId" => keyId, + _ => null + }); + } + private async Task GetLocatorIdAsync(MediaAssetResource asset) { var locators = asset.GetStreamingLocatorsAsync(); diff --git a/azure/AzureProvider.cs b/azure/AzureProvider.cs index 10b0e23..d7e6810 100644 --- a/azure/AzureProvider.cs +++ b/azure/AzureProvider.cs @@ -20,7 +20,7 @@ public AzureProvider( public IFileUploader GetStorageProvider(MigratorOptions migratorOptions) => new AzureStorageUploader(migratorOptions, _credentials, _loggerFactory.CreateLogger()); - public ISecretUploader GetSecretProvider(KeyOptions keyOptions) + public ISecretUploader GetSecretProvider(KeyVaultOptions keyOptions) => new KeyVaultUploader(keyOptions, _credentials, _loggerFactory.CreateLogger()); } } diff --git a/azure/AzureStorageUploader.cs b/azure/AzureStorageUploader.cs index 9d20249..069f6b4 100644 --- a/azure/AzureStorageUploader.cs +++ b/azure/AzureStorageUploader.cs @@ -7,7 +7,6 @@ using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using Microsoft.Extensions.Logging; -using System.Reflection.PortableExecutable; using System.Text; namespace AMSMigrate.Azure diff --git a/azure/KeyVaultUploader.cs b/azure/KeyVaultUploader.cs index 789f036..85c3e7b 100644 --- a/azure/KeyVaultUploader.cs +++ b/azure/KeyVaultUploader.cs @@ -9,21 +9,21 @@ internal class KeyVaultUploader : ISecretUploader { private readonly ILogger _logger; private readonly SecretClient _secretClient; - private readonly KeyOptions _keyOptions; + private readonly KeyVaultOptions _options; public KeyVaultUploader( - KeyOptions options, + KeyVaultOptions options, TokenCredential credential, ILogger logger) { - _keyOptions = options; + _options = options; _logger = logger; _secretClient = new SecretClient(options.KeyVaultUri, credential); } public async Task UploadAsync(string secretName, string secretValue, CancellationToken cancellationToken) { - _logger.LogInformation("Saving secret {name} to key vault {vault}", secretName, _keyOptions.KeyVaultUri); + _logger.LogInformation("Saving secret {name} to key vault {vault}", secretName, _options.KeyVaultUri); await _secretClient.SetSecretAsync(secretName, secretValue, cancellationToken); } } diff --git a/contracts/Events.cs b/contracts/Events.cs new file mode 100644 index 0000000..0a109e4 --- /dev/null +++ b/contracts/Events.cs @@ -0,0 +1,8 @@ +namespace AMSMigrate.Contracts +{ + internal class Events + { + public const int ShakaPackager = 101; + public const int Ffmpeg = 102; + } +} diff --git a/contracts/ICloudProvider.cs b/contracts/ICloudProvider.cs index 9f15f23..1153077 100644 --- a/contracts/ICloudProvider.cs +++ b/contracts/ICloudProvider.cs @@ -5,6 +5,6 @@ interface ICloudProvider { IFileUploader GetStorageProvider(MigratorOptions migratorOptions); - ISecretUploader GetSecretProvider(KeyOptions keyOptions); + ISecretUploader GetSecretProvider(KeyVaultOptions keyOptions); } } diff --git a/contracts/KeyOptions.cs b/contracts/KeyOptions.cs index dcd1619..e1e2f53 100644 --- a/contracts/KeyOptions.cs +++ b/contracts/KeyOptions.cs @@ -1,9 +1,11 @@ namespace AMSMigrate { + public record KeyVaultOptions(Uri KeyVaultUri); + public record KeyOptions( string AccountName, string? ResourceFilter, Uri KeyVaultUri, string? KeyTemplate, - int BatchSize); + int BatchSize) : KeyVaultOptions(KeyVaultUri); } diff --git a/contracts/MigratorOptions.cs b/contracts/MigratorOptions.cs index c3b23dc..89301ed 100644 --- a/contracts/MigratorOptions.cs +++ b/contracts/MigratorOptions.cs @@ -14,5 +14,12 @@ public record MigratorOptions( bool OverWrite, bool SkipMigrated, int SegmentDuration, - int BatchSize); + int BatchSize) + { + public bool EncryptContent { get; set; } + + public string? KeyUri { get; set; } + + public Uri? KeyVaultUri { get; set; } + } } diff --git a/decryption/AssetDecryptor.cs b/decryption/AssetDecryptor.cs index 5c4c8aa..9798bd1 100644 --- a/decryption/AssetDecryptor.cs +++ b/decryption/AssetDecryptor.cs @@ -5,7 +5,6 @@ //----------------------------------------------------------------------- using System.Security.Cryptography; -using System.Threading; using Azure.ResourceManager.Media.Models; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; diff --git a/local/LocalFileProvider.cs b/local/LocalFileProvider.cs index 2e45392..7fb45aa 100644 --- a/local/LocalFileProvider.cs +++ b/local/LocalFileProvider.cs @@ -12,7 +12,7 @@ public LocalFileProvider(ILoggerFactory loggerFactory) _loggerFactory = loggerFactory; } - public ISecretUploader GetSecretProvider(KeyOptions keyOptions) + public ISecretUploader GetSecretProvider(KeyVaultOptions keyOptions) { throw new NotImplementedException(); } diff --git a/transform/AssetTransform.cs b/transform/AssetTransform.cs index 48c4235..cc8f0fb 100644 --- a/transform/AssetTransform.cs +++ b/transform/AssetTransform.cs @@ -1,6 +1,5 @@ using AMSMigrate.Ams; using AMSMigrate.Contracts; -using AMSMigrate.Decryption; using Azure.ResourceManager.Media; using Azure.ResourceManager.Media.Models; using Azure.Storage.Blobs; diff --git a/transform/PackageTransform.cs b/transform/PackageTransform.cs index 875a80c..31ade1a 100644 --- a/transform/PackageTransform.cs +++ b/transform/PackageTransform.cs @@ -9,16 +9,22 @@ namespace AMSMigrate.Transform internal class PackageTransform : StorageTransform { private readonly PackagerFactory _packagerFactory; + private readonly ISecretUploader? _secretUploader = default; public PackageTransform( GlobalOptions globalOptions, MigratorOptions options, ILogger logger, TemplateMapper templateMapper, - IFileUploader uploader, + ICloudProvider cloudProvider, PackagerFactory factory) - : base(globalOptions, options, templateMapper, uploader, logger) + : base(globalOptions, options, templateMapper, cloudProvider.GetStorageProvider(options), logger) { + if (options.EncryptContent) + { + var vaultOptions = new KeyVaultOptions(options.KeyVaultUri!); + _secretUploader = cloudProvider.GetSecretProvider(vaultOptions); + } _packagerFactory = factory; } @@ -44,6 +50,12 @@ protected override async Task TransformAsync( using var source = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cancellationToken = source.Token; + if (_options.EncryptContent) + { + details.GetEncryptionDetails(_options, _templateMapper); + } + + // temporary space for either pipes or files. var workingDirectory = Path.Combine(_options.WorkingDirectory, assetName); Directory.CreateDirectory(workingDirectory); @@ -63,10 +75,8 @@ protected override async Task TransformAsync( var blobs = await container.GetListOfBlobsRemainingAsync(manifest, cancellationToken); allTasks.Add(Task.WhenAll(blobs.Select(async blob => { - using (AesCtrTransform? aesTransform = AssetDecryptor.GetAesCtrTransform(details.DecryptInfo, blob.Name, false)) - { - await UploadBlobAsync(blob, aesTransform, outputPath, cancellationToken); - } + using var aesTransform = AssetDecryptor.GetAesCtrTransform(details.DecryptInfo, blob.Name, false); + await UploadBlobAsync(blob, aesTransform, outputPath, cancellationToken); }))); if (packager.UsePipeForInput) @@ -144,6 +154,12 @@ protected override async Task TransformAsync( // Upload any files pending to be uploaded. await Task.WhenAll( uploadPaths.Select(file => UploadFile(file, uploadHelper, cancellationToken))); + + if (_options.EncryptContent) + { + _logger.LogDebug("Saving key with id id: {keyId} for asset: {name} to key vault {vault}", details.KeyId, details.AssetName, _options.KeyVaultUri); ; + await _secretUploader!.UploadAsync(details.KeyId, details.EncryptionKey, cancellationToken); + } } catch (Exception ex) { @@ -171,8 +187,23 @@ private UploadPipe CreateUpload(string filePath, UploadHelper helper) private async Task UploadFile(string filePath, UploadHelper uploadHelper, CancellationToken cancellationToken) { var file = Path.GetFileName(filePath); + // Report update for every 1MB. + long update = 0; var progress = new Progress(p => - _logger.LogTrace("Uploaded {bytes} bytes to {file}", p, file)); + { + if (p >= update) + { + lock (this) + { + if (p >= update) + { + _logger.LogTrace("Uploaded {byte} bytes to {file}", p, file); + update += 1024 * 1024; + } + } + } + }); + using var content = File.OpenRead(filePath); await uploadHelper.UploadAsync(Path.GetFileName(file), content, GetHeaders(file), progress, cancellationToken); } diff --git a/transform/PackagerFactory.cs b/transform/PackagerFactory.cs index 3afc9d3..0260f96 100644 --- a/transform/PackagerFactory.cs +++ b/transform/PackagerFactory.cs @@ -25,7 +25,7 @@ public BasePackager GetPackager(AssetDetails details, CancellationToken cancella } else { - packager = new ShakaPackager(details, _transMuxer, _loggerFactory.CreateLogger()); + packager = new ShakaPackager(_options, details, _transMuxer, _loggerFactory.CreateLogger()); } return packager; } diff --git a/transform/ShakaPackager.cs b/transform/ShakaPackager.cs index a8ccf9e..c3d979e 100644 --- a/transform/ShakaPackager.cs +++ b/transform/ShakaPackager.cs @@ -1,6 +1,6 @@ -using Microsoft.Extensions.Logging; +using AMSMigrate.Contracts; +using Microsoft.Extensions.Logging; using System.ComponentModel; -using System.Reflection; using System.Text.RegularExpressions; namespace AMSMigrate.Transform @@ -9,19 +9,16 @@ internal class ShakaPackager : BasePackager { private readonly TaskCompletionSource _taskCompletionSource; - public static readonly string Packager; + static readonly string PackagerDirectory = AppContext.BaseDirectory; + static readonly string Executable = $"packager-{(OperatingSystem.IsLinux() ? "linux-x64" : OperatingSystem.IsMacOS() ? "osx-x64" : "win-x64.exe")}"; + public static readonly string Packager = Path.Combine(PackagerDirectory, Executable); - static ShakaPackager() - { - var executable = $"packager-{(OperatingSystem.IsLinux() ? "linux-x64" : OperatingSystem.IsMacOS() ? "osx-x64" : "win-x64.exe")}"; - Packager = Path.Combine( - Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, - executable); - } + private readonly MigratorOptions _options; - public ShakaPackager(AssetDetails assetDetails, TransMuxer transMuxer, ILogger logger) + public ShakaPackager(MigratorOptions options, AssetDetails assetDetails, TransMuxer transMuxer, ILogger logger) : base(assetDetails, transMuxer, logger) { + _options = options; _taskCompletionSource = new TaskCompletionSource(); var manifest = assetDetails.Manifest!; var baseName = Path.GetFileNameWithoutExtension(manifest.FileName); @@ -57,13 +54,16 @@ public ShakaPackager(AssetDetails assetDetails, TransMuxer transMuxer, ILogger GetArguments(IList inputs, IList outputs, IList manifests) { + const string DRM_LABEL = "cenc"; + var drm_label = _options.EncryptContent ? $",drm_label={DRM_LABEL}" : string.Empty; + List arguments = new(SelectedTracks.Select((t, i) => { var ext = t.IsMultiFile ? MEDIA_FILE : string.Empty; var index = Inputs.Count == 1 ? 0 : Inputs.IndexOf($"{t.Source}{ext}"); var stream = Inputs.Count == 1? i.ToString(): t.Type.ToString().ToLowerInvariant(); var language = string.IsNullOrEmpty(t.SystemLanguage) || t.SystemLanguage == "und" ? string.Empty : $"language={t.SystemLanguage},"; - return $"stream={stream},in={inputs[index]},out={outputs[i]},{language}playlist_name={manifests[i]}"; + return $"stream={stream},in={inputs[index]},out={outputs[i]},{language}playlist_name={manifests[i]}{drm_label}"; })); var dash = manifests[manifests.Count - 1]; var hls = manifests[manifests.Count - 2]; @@ -77,6 +77,22 @@ private IEnumerable GetArguments(IList inputs, IList out arguments.Add("--io_block_size"); arguments.Add("65536"); } + + if (_options.EncryptContent) + { + arguments.Add("--enable_raw_key_encryption"); + arguments.Add("--protection_scheme"); + arguments.Add("cbcs"); + arguments.Add("--keys"); + arguments.Add($"label={DRM_LABEL}:key_id={_assetDetails.KeyId}:key={_assetDetails.EncryptionKey}"); + arguments.Add("--hls_key_uri"); + arguments.Add($"{_assetDetails.LicenseUri}"); + arguments.Add("--clear_lead"); + arguments.Add("0"); + } + + arguments.Add("--temp_dir"); + arguments.Add(_options.WorkingDirectory); arguments.Add("--segment_duration"); arguments.Add("2"); arguments.Add("--mpd_output"); @@ -131,7 +147,7 @@ private void LogStandardError(string? line ) var match = ShakaLogRegEx.Match(line); var group = match.Groups["level"]; _ = match.Success && group.Success && LogLevels.TryGetValue(group.Value, out logLevel); - _logger.Log(logLevel, line); + _logger.Log(logLevel, Events.ShakaPackager, line); } } } diff --git a/transform/StorageTransform.cs b/transform/StorageTransform.cs index f0b4692..57131c4 100644 --- a/transform/StorageTransform.cs +++ b/transform/StorageTransform.cs @@ -9,13 +9,29 @@ namespace AMSMigrate.Transform { - public record AssetDetails(string AssetName, BlobContainerClient Container, Manifest? Manifest, ClientManifest? ClientManifest, string? OutputManifest, StorageEncryptedAssetDecryptionInfo? DecryptInfo); + public record AssetDetails(string AssetName, BlobContainerClient Container, Manifest? Manifest, ClientManifest? ClientManifest, string? OutputManifest, StorageEncryptedAssetDecryptionInfo? DecryptInfo) + { + /// + /// The key ID of the encryption key. + /// + public string KeyId { get; set; } = string.Empty; + + /// + /// The encryption key to use. + /// + public string EncryptionKey { get; set; } = string.Empty; + + /// + /// The license URL for key delivery. + /// + public string LicenseUri { get; set; } = string.Empty; + } internal abstract class StorageTransform : ITransform { protected readonly GlobalOptions _globalOptions; protected readonly MigratorOptions _options; - private readonly TemplateMapper _templateMapper; + protected readonly TemplateMapper _templateMapper; protected readonly ILogger _logger; protected readonly IFileUploader _fileUploader; diff --git a/transform/TransformFactory.cs b/transform/TransformFactory.cs index 55a9f18..402ce37 100644 --- a/transform/TransformFactory.cs +++ b/transform/TransformFactory.cs @@ -34,7 +34,7 @@ public IEnumerable GetTransforms(GlobalOptions globalOptions, options, _loggerFactory.CreateLogger(), _templateMapper, - uploader, + _cloudProvider, packagerFactory); } diff --git a/transform/UploadTransform.cs b/transform/UploadTransform.cs index 2e49971..f12334e 100644 --- a/transform/UploadTransform.cs +++ b/transform/UploadTransform.cs @@ -2,7 +2,6 @@ using AMSMigrate.Contracts; using AMSMigrate.Decryption; using Microsoft.Extensions.Logging; -using System.Reflection.Metadata; namespace AMSMigrate.Transform {