Merge pull request #35 from Manuss20/main

Update VideoAnalyzer
Azure · Apr 21, 2024 · 6dbe2cc · 6dbe2cc
2 parents 6a8da3c + fd2a653
commit 6dbe2cc
Show file tree

Hide file tree

Showing 10 changed files with 67 additions and 22 deletions.
diff --git a/infra/main.tf b/infra/main.tf
@@ -24,6 +24,7 @@ locals {
   ca_prep_docs_name       = "${var.ca_prep_docs_name}${local.name_sufix}"
   ca_aihub_name           = "${var.ca_aihub_name}${local.name_sufix}"
   func_name               = "plugin${local.sufix}"
+  cv_name                 = "${var.cv_name}${local.name_sufix}"
 }
 
 resource "azurerm_resource_group" "rg" {
@@ -245,3 +246,10 @@ module "plugin" {
   openai_model             = module.openai.gpt_deployment_name
   openai_endpoint          = module.openai.openai_endpoint
 }
+
+module "cv" {
+  source              = "./modules/cv"
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
+  cv_name             = local.cv_name
+}
diff --git a/infra/modules/cv/main.tf b/infra/modules/cv/main.tf
@@ -0,0 +1,7 @@
+resource "azurerm_cognitive_account" "cv" {
+  name                = var.cv_name
+  location            = var.location
+  resource_group_name = var.resource_group_name
+  kind                = "Face"
+  sku_name            = "S0"
+}
diff --git a/infra/modules/cv/outputs.tf b/infra/modules/cv/outputs.tf
diff --git a/infra/modules/cv/variables.tf b/infra/modules/cv/variables.tf
@@ -0,0 +1,3 @@
+variable "resource_group_name" {}
+variable "location" {}
+variable "cv_name" {}
diff --git a/infra/modules/st/main.tf b/infra/modules/st/main.tf
@@ -41,6 +41,12 @@ resource "azurerm_storage_container" "image-analyzer" {
   storage_account_name  = azurerm_storage_account.sa.name
 }
 
+resource "azurerm_storage_container" "image-analyzer" {
+  name                  = "video-analyzer"
+  container_access_type = "private"
+  storage_account_name  = azurerm_storage_account.sa.name
+}
+
 resource "azurerm_storage_container" "image-moderator" {
   name                  = "image-moderator"
   container_access_type = "private"

diff --git a/infra/variables.tf b/infra/variables.tf
@@ -86,6 +86,10 @@ variable "ca_aihub_name" {
   default = "ca-aihub"
 }
 
+variable "cv_name" {
+  default = "cv-aihub"
+}
+
 variable "ca_chat_image" {
   default = "ghcr.io/azure/activate-genai/aihub-chat:1.0.0-preview.0"
 }

diff --git a/src/AIHub/Controllers/VideoAnalyzerController.cs b/src/AIHub/Controllers/VideoAnalyzerController.cs
@@ -26,6 +26,8 @@ public class VideoAnalyzerController : Controller
     private VideoAnalyzerModel model;
     private HttpClient httpClient;
 
+
+
     public VideoAnalyzerController(IConfiguration config, IHttpClientFactory clientFactory)
     {
         AOAIendpoint = config.GetValue<string>("VideoAnalyzer:OpenAIEndpoint") ?? throw new ArgumentNullException("OpenAIEndpoint");
@@ -117,9 +119,6 @@ public async Task<IActionResult> DenseCaptionVideo(string video_url, string prom
         string VISION_API_ENDPOINT = $"{Visionendpoint}computervision";
         string VISION_API_KEY = VisionKey;
         string VIDEO_INDEX_NAME = Regex.Replace(video_url.Split("/").Last().Split(".").First().GetHashCode().ToString(), "[^a-zA-Z0-9]", "");
-
-
-
         string VIDEO_FILE_SAS_URL = video_url + sasUri.Query;
 
         // Step 1: Create an Index
@@ -149,6 +148,7 @@ public async Task<IActionResult> DenseCaptionVideo(string video_url, string prom
         }
         var payload = new
         {
+            model = "gpt-4-vision-preview",
             dataSources = new[]
             {
                 new
@@ -157,8 +157,8 @@ public async Task<IActionResult> DenseCaptionVideo(string video_url, string prom
                     parameters = new
                     {
                         computerVisionBaseUrl = VISION_API_ENDPOINT,
-                        computerVisionApiKey = VisionKey,
-                        indexName = Regex.Replace(video_url.Split("/").Last().Split(".").First().GetHashCode().ToString(), "[^a-zA-Z0-9]", ""),
+                        computerVisionApiKey = VISION_API_KEY,
+                        indexName = VIDEO_INDEX_NAME,
                         videoUrls = new[] { VIDEO_FILE_SAS_URL }
                     }
                 }
@@ -180,26 +180,31 @@ public async Task<IActionResult> DenseCaptionVideo(string video_url, string prom
                     role = "user", 
                     content = new object[]
                     {
-                        new acvDocumentIdWrapper() {AcvDocumentId = VIDEO_DOCUMENT_ID},
-                        prompt
+                        new {
+                            type = "acv_document_id",
+                            acv_document_id = VIDEO_DOCUMENT_ID
+                        },
+                        new {
+                            type = "text",
+                            text = prompt
+                        }
                     },
                 }
             },
             temperature = 0.7,
             top_p = 0.95,
-            max_tokens = 800
+            max_tokens = 4096
         };
 
         try
         {
             var chatResponse = await httpClient.PostAsync(GPT4V_ENDPOINT, new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"));
             chatResponse.EnsureSuccessStatusCode();
-            var responseContent = JsonSerializer.Deserialize<dynamic>(await chatResponse.Content.ReadAsStringAsync());
+            var responseContent = JsonSerializer.Deserialize<JsonObject>(await chatResponse.Content.ReadAsStringAsync());
             Console.WriteLine(responseContent);
-
-            model.Message = responseContent; //responseContent!.choices[0].message.content;
+ 
+            model.Message = responseContent?["choices"]?[0]?["message"]?["content"]?.ToString();
             model.Video = VIDEO_FILE_SAS_URL;
-
         }
         catch
         {

diff --git a/src/AIHub/Views/VideoAnalyzer/VideoAnalyzer.cshtml b/src/AIHub/Views/VideoAnalyzer/VideoAnalyzer.cshtml
@@ -102,7 +102,9 @@
                 <div class="row g-0 align-items-center">
                     <div class="col-md-4">
                         <div class="ratio ratio-16x9">
-                            <iframe id="bigpic" src="@Model.Video"></iframe>
+                            <video id="bigpic" width="320" height="240" controls>
+                                <!--- Here we have to insert the source video throught javascript -->
+                            </video>
                         </div>
                     </div>
                     <div class="col-md-8">

diff --git a/src/AIHub/appsettings.template.json b/src/AIHub/appsettings.template.json
@@ -37,9 +37,9 @@
   "VideoAnalyzer": {
     "OpenAIEndpoint": "<AOAI EndPoint>",
     "GPT4Vision": "openai/deployments/<deploymentName>/extensions/chat/completions?api-version=2023-07-01-preview",
-    "VisionAPI": "https://<deploymentName>.cognitiveservices.azure.com/",
-    "VisionKey": "<Vision Key>",
     "OpenAISubscriptionKey": "<AOAI Key>",
+    "VisionEndpoint": "https://<deploymentName>.cognitiveservices.azure.com/",
+    "VisionSubscriptionKey": "<Vision Key>",
     "ContainerName": "video-analyzer",
     "DeploymentName": "gpt4-vision"
   },

diff --git a/src/AIHub/wwwroot/js/ui/videoanalyzer.js b/src/AIHub/wwwroot/js/ui/videoanalyzer.js
@@ -1,30 +1,40 @@
-Dropzone.options.dropaiimage = { 
-    paramName: "videoFile", 
+Dropzone.options.dropaiimage = {
+    paramName: "videoFile",
     maxFilesize: 200, // MB
     createImageThumbnails:true,
     previewsContainer: "#file-previews",
     accept: function(file, done) {
         done();
     },
-    init: function() {         
+    init: function() {        
         var myDropzone = this;          
-        myDropzone.on("sending", function(file) {             
+        myDropzone.on("sending", function(file) {            
             $("#loader").removeClass("d-none");        
         });              
-        myDropzone.on("complete", 
+        myDropzone.on("complete",
         function(file) {                    
             $("#loader").addClass("d-none");                    
         });
     },
     success: function (file, response) {
         if (typeof response === "object") {        
             $("#show-message-result").text(response.message);
-            $("#bigpic").attr('src', response.video);
+
+            var isrc = document.createElement("source");
+            isrc.setAttribute("src", response.video);
+            isrc.setAttribute("type", "video/mp4");
+            document.getElementById("bigpic").appendChild(isrc);
+
         } else {
             try {
                 var parsedResponse = JSON.parse(response);
                 $("#show-message-result").val(parsedResponse.message);
-                $("#bigpic").attr('src',parsedResponse.video);
+
+                var isrc = document.createElement("source");
+                isrc.setAttribute("src", parsedResponse.video);
+                isrc.setAttribute("type", "video/mp4");
+                document.getElementById("bigpic").appendChild(isrc);
+
             } catch (e) {
                 console.error("Error parsing the response:", e);
             }