From dc42d52cb560b9d9dcc4fd668dabd91765d2e119 Mon Sep 17 00:00:00 2001 From: Yubo Xie Date: Tue, 28 Nov 2023 16:31:12 -0800 Subject: [PATCH] [EncodingAndPackagingTool] Fix command line tool crash issue. [EncodingAndPackagingTool] Fix input file can't be in sub folder issue. [EncodingAndPackagingTool] Add test for command line tool. --- .gitattributes | 1 + .github/workflows/ci-build.yml | 20 ++- .../EncodingAndPackagingExample.sln | 6 + .../EncodingAndPackagingTool.cs | 7 +- .../EncodingAndPackagingTool.Cli.Test.cs | 116 ++++++++++++++++++ .../EncodingAndPackagingTool.Test.csproj | 20 +++ test-data/bunny.640x480.15fps.mp4 | 3 + 7 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 EncodingAndPackagingExample/EncodingAndPackagingTool.Test/EncodingAndPackagingTool.Cli.Test.cs create mode 100644 EncodingAndPackagingExample/EncodingAndPackagingTool.Test/EncodingAndPackagingTool.Test.csproj create mode 100644 test-data/bunny.640x480.15fps.mp4 diff --git a/.gitattributes b/.gitattributes index dfe0770..98c01a3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ # Auto detect text files and perform LF normalization * text=auto +*.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index bda15a6..d51570c 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -128,19 +128,37 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + lfs: true - name: Setup DotNet Environment uses: actions/setup-dotnet@v3 with: dotnet-version: 6.0.x + - name: Install ffmpeg + run: | + apt install ffmpeg -y + + - name: Install certificate + run: | + apt install ca-certificates -y + openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout /tmp/127.0.0.1.key -out /tmp/127.0.0.1.crt -subj "/CN=127.0.0.1" -addext "subjectAltName=IP:127.0.0.1" + cp /tmp/127.0.0.1.crt /usr/local/share/ca-certificates/ + update-ca-certificates + + - name: Run Azurite + run: | + npm install -g azurite + azurite --silent --location /tmp --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --cert /tmp/127.0.0.1.crt --key /tmp/127.0.0.1.key --oauth basic & + - name: Build and publish run: | dotnet publish -c Release EncodingAndPackagingExample - name: Test run: | - dotnet test -c Release --no-build EncodingAndPackagingExample + AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }} AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }} AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }} TEST_DATA=`pwd`/test-data dotnet test -c Release --no-build EncodingAndPackagingExample - name: Build Container Image uses: docker/build-push-action@v4 diff --git a/EncodingAndPackagingExample/EncodingAndPackagingExample.sln b/EncodingAndPackagingExample/EncodingAndPackagingExample.sln index a1d88a6..dd4bd35 100644 --- a/EncodingAndPackagingExample/EncodingAndPackagingExample.sln +++ b/EncodingAndPackagingExample/EncodingAndPackagingExample.sln @@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EncodingAndPackagingTool.Cl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EncodingAndPackagingTool.AzureFunction", "EncodingAndPackagingTool.AzureFunction\EncodingAndPackagingTool.AzureFunction.csproj", "{8D4079CC-067D-401F-B4A7-DAAD0A34A3FE}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EncodingAndPackagingTool.Test", "EncodingAndPackagingTool.Test\EncodingAndPackagingTool.Test.csproj", "{A80B57CB-880C-437F-8351-40582DB08CAA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -27,6 +29,10 @@ Global {8D4079CC-067D-401F-B4A7-DAAD0A34A3FE}.Debug|Any CPU.Build.0 = Debug|Any CPU {8D4079CC-067D-401F-B4A7-DAAD0A34A3FE}.Release|Any CPU.ActiveCfg = Release|Any CPU {8D4079CC-067D-401F-B4A7-DAAD0A34A3FE}.Release|Any CPU.Build.0 = Release|Any CPU + {A80B57CB-880C-437F-8351-40582DB08CAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A80B57CB-880C-437F-8351-40582DB08CAA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A80B57CB-880C-437F-8351-40582DB08CAA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A80B57CB-880C-437F-8351-40582DB08CAA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/EncodingAndPackagingExample/EncodingAndPackagingTool.Core/EncodingAndPackagingTool.cs b/EncodingAndPackagingExample/EncodingAndPackagingTool.Core/EncodingAndPackagingTool.cs index c9bcf3f..bf092bc 100644 --- a/EncodingAndPackagingExample/EncodingAndPackagingTool.Core/EncodingAndPackagingTool.cs +++ b/EncodingAndPackagingExample/EncodingAndPackagingTool.Core/EncodingAndPackagingTool.cs @@ -20,7 +20,7 @@ public class EncodingAndPackagingTool public EncodingAndPackagingTool(ILogger logger, TokenCredential azureCredential) { - if (_azureCredential == null) + if (azureCredential == null) { throw new ArgumentNullException(nameof(azureCredential)); } @@ -47,7 +47,7 @@ public async Task EncodeAndPackageAsync(Uri mp4BlobUri, Uri outputStorageUri, Ca var connectionString = _connectionString; // Prepare a tmp path. - var tmpPath = Path.Combine(Path.GetTempPath(), $"{DateTime.Now.ToString("yyyyMMMdddhhssmm")}-{Guid.NewGuid()}"); + var tmpPath = Path.Combine(Path.GetTempPath(), $"{DateTime.UtcNow.ToString("yyyyMMddhhmmss")}-{Guid.NewGuid()}"); logger.LogInformation($"Create tmp path: {tmpPath}"); try @@ -75,7 +75,7 @@ public async Task EncodeAndPackageAsync(Uri mp4BlobUri, Uri outputStorageUri, Ca } // Download the blob to a local tmp folder - var inputFile = Path.Combine(tmpPath, $"{blob.Name}"); + var inputFile = Path.Combine(tmpPath, $"{Path.GetFileName(blob.Name)}"); logger.LogInformation($"Download blob {blob.Name} to {inputFile}"); using (var inputFileStream = System.IO.File.OpenWrite(inputFile)) using (var inputStream = await blob.OpenReadAsync(new BlobOpenReadOptions(allowModifications: false), cancellationToken).ConfigureAwait(false)) @@ -134,6 +134,7 @@ public async Task EncodeAndPackageAsync(Uri mp4BlobUri, Uri outputStorageUri, Ca var blobTmpContainer = new BlobContainerClient(outputStorageUri); blobContainer = new BlobContainerClient(connectionString, blobTmpContainer.Name); } + await blobContainer.CreateIfNotExistsAsync().ConfigureAwait(false); // Upload. await Task.WhenAll(Directory.GetFiles(outputDir).Select(async file => diff --git a/EncodingAndPackagingExample/EncodingAndPackagingTool.Test/EncodingAndPackagingTool.Cli.Test.cs b/EncodingAndPackagingExample/EncodingAndPackagingTool.Test/EncodingAndPackagingTool.Cli.Test.cs new file mode 100644 index 0000000..c2d7e0b --- /dev/null +++ b/EncodingAndPackagingExample/EncodingAndPackagingTool.Test/EncodingAndPackagingTool.Cli.Test.cs @@ -0,0 +1,116 @@ +using Azure.Identity; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace EncodingAndPackagingTool.Test; + +public class EncodingAndPackagingTool +{ + private static string _storageServiceUri; + private static string _inputContainerUri; + private static string _testDataPath; + private static DefaultAzureCredential _azureCrendentail; + + static EncodingAndPackagingTool() + { + _storageServiceUri = "https://127.0.0.1:10000/devstoreaccount1"; + _inputContainerUri = $"{_storageServiceUri}/encodingandpackagingtooltest"; + _testDataPath = Environment.GetEnvironmentVariable("TEST_DATA") ?? throw new Exception("TEST_DATA environment variable is missing."); + _azureCrendentail = new DefaultAzureCredential(); + + // Upload test video clip. + var container = new BlobContainerClient(new Uri(_inputContainerUri), _azureCrendentail); + container.CreateIfNotExists(); + + Task.WhenAll(Directory.GetFiles(_testDataPath).Select(async file => + { + var blob = container.GetBlockBlobClient($"input/{Path.GetFileName(file)}"); + using var stream = System.IO.File.OpenRead(file); + await container.GetBlobClient($"input/{Path.GetFileName(file)}").UploadAsync(stream, overwrite: true); + })).Wait(); + } + + [Fact] + public async Task EncodingAndPackagingToolTest() + { + var inputUri = $"{_inputContainerUri}/input/bunny.640x480.15fps.mp4"; + var outputContainerUri = $"{_storageServiceUri}/output{Guid.NewGuid().ToString("n")}"; + + // Invoke EncodingAndPackagingTool + var psi = new ProcessStartInfo() + { + FileName = "dotnet", + Arguments = $"EncodingAndPackagingTool.dll --input {inputUri} --output {outputContainerUri}" + }; + var process = Process.Start(psi) ?? throw new Exception("Start process failed."); + await process.WaitForExitAsync(); + Assert.Equal(0, process.ExitCode); + + // We should have the output mpd file. + var blob = new BlobClient(new Uri($"{outputContainerUri}/bunny.640x480.15fps.mpd"), _azureCrendentail); + using (var stream = await blob.OpenReadAsync()) + { + Assert.True(stream.Length > 2000); + } + + // We should have 13 chunk files for stream 0 + for (var i = 1; i <= 13; ++i) + { + blob = new BlobClient(new Uri($"{outputContainerUri}/chunk-stream0-{i.ToString("00000")}.m4s"), _azureCrendentail); + using (var stream = await blob.OpenReadAsync()) + { + Assert.True(stream.Length > 2000); + } + } + + // We should have 24 chunk files for stream 1 + for (var i = 1; i <= 24; ++i) + { + blob = new BlobClient(new Uri($"{outputContainerUri}/chunk-stream1-{i.ToString("00000")}.m4s"), _azureCrendentail); + using (var stream = await blob.OpenReadAsync()) + { + Assert.True(stream.Length > 2000); + } + } + + // We should have 13 chunk files for stream 2 + for (var i = 1; i <= 13; ++i) + { + blob = new BlobClient(new Uri($"{outputContainerUri}/chunk-stream2-{i.ToString("00000")}.m4s"), _azureCrendentail); + using (var stream = await blob.OpenReadAsync()) + { + Assert.True(stream.Length > 2000); + } + } + + // We should have 13 chunk files for stream 3 + for (var i = 1; i <= 13; ++i) + { + blob = new BlobClient(new Uri($"{outputContainerUri}/chunk-stream3-{i.ToString("00000")}.m4s"), _azureCrendentail); + using (var stream = await blob.OpenReadAsync()) + { + Assert.True(stream.Length > 2000); + } + } + + // We should have 4 init chunk files. + for (var i = 0; i < 4; ++i) + { + blob = new BlobClient(new Uri($"{outputContainerUri}/init-stream{i}.m4s"), _azureCrendentail); + using (var stream = await blob.OpenReadAsync()) + { + Assert.True(stream.Length > 500); + } + } + + // Delete the container if success. + var container = new BlobContainerClient(new Uri(outputContainerUri), _azureCrendentail); + await container.DeleteAsync(); + } +} diff --git a/EncodingAndPackagingExample/EncodingAndPackagingTool.Test/EncodingAndPackagingTool.Test.csproj b/EncodingAndPackagingExample/EncodingAndPackagingTool.Test/EncodingAndPackagingTool.Test.csproj new file mode 100644 index 0000000..95c9379 --- /dev/null +++ b/EncodingAndPackagingExample/EncodingAndPackagingTool.Test/EncodingAndPackagingTool.Test.csproj @@ -0,0 +1,20 @@ + + + net6.0 + EncodingAndPackagingTool.Test + EncodingAndPackagingTool.Test + annotations + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + \ No newline at end of file diff --git a/test-data/bunny.640x480.15fps.mp4 b/test-data/bunny.640x480.15fps.mp4 new file mode 100644 index 0000000..6b11586 --- /dev/null +++ b/test-data/bunny.640x480.15fps.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f0596ec3a9f9cc41630ee22f6faab69f7e3938b59e8e1f3ad3ce21e5dc2df7d +size 13371998