Merge branch 'master' into master

.azure-pipelines/callGithubWorkflow.yaml

@@ -47,6 +47,7 @@ jobs:
       eq(dependencies.YamlFileValidation.result, 'Succeeded')
     )
   steps:
+  - checkout: none
   - pwsh: |
       try
       {

.github/workflows/IssueComment.yml

@@ -1,6 +1,8 @@
 name: IssueComment
 
-on: [issues]
+on: 
+  issues:
+    types: opened
 
 jobs:
   commenting:

.github/workflows/addComment.yaml

@@ -15,16 +15,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/github-script@29423367f079522048aa7c63f671593b0556ffd5
+        env:
+          MESSAGE: ${{ inputs.message}}
+          PR_NUMBER: ${{ inputs.prNumber }}
         with:
           script: |
-            $message = "${{ inputs.message }}"
-            $prNumber = "${{ inputs.prNumber }}"
-            console.log('message ' + $message + ', prNumber ' + $prNumber)
+            $prNumber = process.env.PR_NUMBER
             $prNumber = $prNumber ? $prNumber : context.issue.number
 
             github.rest.issues.createComment({
               issue_number: $prNumber,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              body: $message
+              body: process.env.MESSAGE
             })

.github/workflows/addCommentToRemindUpdatingTemplateVersion.yml

@@ -4,18 +4,16 @@ on:
   pull_request:
     paths:
        - 'Detection**'
+       - 'Solutions/**/Analytic Rules/**'
 
 jobs:
   add-comment:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: mshick/add-pr-comment@v1
-        with:
-          message: |
-            **Hello how are you I am GitHub bot**
-            😀😀
-            I see that you changed templates under the detections folder. Did you remember to update the version of the templates you changed?
-            If not, and if you want customers to be aware that a new version of this template is available, please update the ``version`` property of the template you changed.
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          repo-token-user-login: 'github-actions[bot]' # The user.login for temporary GitHub tokens
-          allow-repeats: false # This is the default
+    uses: ./.github/workflows/addComment.yaml
+    if: ${{ !github.event.pull_request.head.repo.fork }}
+    with:
+      message: |
+        **Hello how are you I am GitHub bot**
+        😀😀
+        I see that you changed templates under the detections/analytic rules folder. Did you remember to update the version of the templates you changed?
+        If not, and if you want customers to be aware that a new version of this template is available, please update the ``version`` property of the template you changed.
+      prNumber: "${{ github.event.pull_request.number }}"

.github/workflows/checkAutomatedPR.yaml

@@ -30,7 +30,7 @@ jobs:
         id: ValidateAutomatedPR
 
         run: |
-          $prBodyContent = "${{ env.BODY }}"
+          $prBodyContent = '${{ env.BODY }}'
           $isAutomatedPR = $false
           if ($prBodyContent -like '*Automation have successfully*')
           {

.github/workflows/package-command.yaml

@@ -11,10 +11,11 @@ env:
   ADO_TEAMPROJECT: "${{ vars.ADO_TEAMPROJECT }}"
   ADO_PARENT_LINK: "${{ vars.ADO_PARENT_LINK }}"
   GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+  PAT_TOKEN: ${{ secrets.PAT }}
 
 on:
   repository_dispatch:
-    types: [package-command]
+    types: [package-command, Package-command, PACKAGE-command]
 jobs:
   solutionNameDetails:
     if: ${{ !github.event.pull_request.head.repo.fork }}
@@ -63,7 +64,8 @@ jobs:
       - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
         with:
           fetch-depth: 2
-          ref: ${{ env.BRANCH_NAME }} 
+          ref: ${{ env.BRANCH_NAME }}
+          token: ${{ env.PAT_TOKEN }}
       - shell: pwsh
         id: step1
         name: Check & Initiate Package Creation Process
@@ -134,137 +136,10 @@ jobs:
           name: "${{ env.DATA_INPUT_FILE_NAME }}"
           path: "${{ env.DATA_FOLDER_PATH }}"
 
-  getPullRequestInfo:
-    runs-on: ubuntu-latest
-    needs: createpackage
-    env:
-      PULL_REQUEST_NUMBER: "${{ github.event.client_payload.pull_request.number && github.event.client_payload.pull_request.number || github.event.client_payload.pullRequestNumber }}"
-    outputs:
-      existingPRTitle: ${{ steps.getPRDetails.outputs.existingPRTitle }}
-      existingPRBody: "${{ steps.getPRDetails.outputs.existingPRBody }}"
-    steps:
-    - shell: pwsh
-      id: getPRDetails
-      name: Get PR Details
-      run: |
-        try
-        {
-          $pullRequestNumber = "${{ env.PULL_REQUEST_NUMBER }}"
-          $token = "${{ secrets.GITHUB_TOKEN }}"
-  
-          if ($null -ne $pullRequestNumber -and $pullRequestNumber -ne '')
-          {
-            $headers = New-Object "System.Collections.Generic.Dictionary[[String],[String]]"
-            $headers.Add("Accept", "application/vnd.github+json")
-            $headers.Add("Authorization", "Bearer $token")
-            $headers.Add("X-GitHub-Api-Version", "2022-11-28")
-            Write-Host "PR Number $pullRequestNumber"
-
-            $link = 'https://api.github.com/repos/Azure/Azure-Sentinel/pulls/' + $pullRequestNumber
-            $response = Invoke-RestMethod -Uri $link -Method 'GET' -Headers $headers
-
-            if ($null -eq $response)
-            {
-              Write-Output "existingPRTitle=" >> $env:GITHUB_OUTPUT
-              Write-Output "existingPRBody=" >> $env:GITHUB_OUTPUT
-              exit 1
-            }
-            else
-            {
-              $title = "" + $response.title
-              $body = "" + $response.body
-              Write-Output "existingPRTitle=$title" >> $env:GITHUB_OUTPUT
-
-              Write-Output "existingPRBody<<EOF" >> $env:GITHUB_OUTPUT
-              Write-Output $body >> $env:GITHUB_OUTPUT
-              Write-Output "EOF" >> $env:GITHUB_OUTPUT
-            }
-          }
-          else
-          {
-            Write-Host "Either PullRequest title or Body is empty!"
-            Write-Output "existingPRTitle=" >> $env:GITHUB_OUTPUT
-            Write-Output "existingPRBody=" >> $env:GITHUB_OUTPUT
-            exit 1
-          }
-        }
-        catch
-        {
-          Write-Output "existingPRTitle=" >> $env:GITHUB_OUTPUT
-          Write-Output "existingPRBody=" >> $env:GITHUB_OUTPUT
-          Write-Host "Error occured while getting pull request title and body details. Error Details $_"
-          exit 1
-        }
-
-  updatedExistingPR:
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
-      contents: write
-    needs: [solutionNameDetails, neworexistingsolution, createpackage, getPullRequestInfo]
-    continue-on-error: true
-    env:
-      BLOBNAME: "${{ needs.createpackage.outputs.blobName }}"
-      IS_CREATE_PACKAGE: ${{ needs.createpackage.outputs.isCreatePackage }}
-      PACKAGE_CREATION_PATH: "${{ needs.createpackage.outputs.packageCreationPath }}"
-      DATA_FOLDER_PATH: "${{ needs.createpackage.outputs.dataFolderPath }}"
-      DATA_INPUT_FILE_NAME: "${{ needs.createpackage.outputs.dataInputFileName }}"
-      SOLUTION_NAME: "${{ needs.solutionNameDetails.outputs.solutionName }}"
-      PULL_REQUEST_NUMBER: "${{ github.event.client_payload.pull_request.number && github.event.client_payload.pull_request.number || github.event.client_payload.pullRequestNumber }}"
-      SOLUTION_SUPPORTED_BY: "${{ needs.neworexistingsolution.outputs.solutionSupportedBy }}"
-      ACTOR: "${{ github.actor }}"
-      ASSIGNEES: "${{ github.event.client_payload.github.actor }}"
-      CURRENT_BRANCH_NAME: "${{ github.event.client_payload.pull_request.head.ref && github.event.client_payload.pull_request.head.ref || github.event.client_payload.pullRequestBranchName }}"
-      RUNID: "${{ github.event.client_payload.github.runId }}"
-      APPINSIGHTS: "${{ vars.APPINSIGHTS }}"
-      EXISTING_PR_TITLE: "${{ needs.getPullRequestInfo.outputs.existingPRTitle }}"
-      EXISTING_PR_BODY: ${{ needs.getPullRequestInfo.outputs.existingPRBody }}
-    steps:
-      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
-      - uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
-        with:
-          name : "${{ env.BLOBNAME }}"
-          path: "${{ env.PACKAGE_CREATION_PATH }}"
-
-      - uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
-        with:
-          name: "${{ env.DATA_INPUT_FILE_NAME }}"
-          path: "${{ env.DATA_FOLDER_PATH }}"
-
-      - name: updateExistingPR
-        id: update-existing-pr
-        if: ${{ success() && (env.IS_CREATE_PACKAGE && env.PACKAGE_CREATION_PATH != '' && env.BLOBNAME != '' ) }}
-        uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5
-        with:
-          committer: GitHub <noreply@github.com>
-          commit-message: added/updated package to pr
-          author: GitHub <noreply@github.com>
-          signoff: false
-          branch: "${{ env.CURRENT_BRANCH_NAME }}" # EXISTING BRANCH NAME
-          title: "${{ env.EXISTING_PR_TITLE }}"
-          body: |
-            ${{ env.EXISTING_PR_BODY }}
-
-      - shell: pwsh
-        id: createPREvents
-        name: createPREvents
+      - name: Push changes to Existing PR
         run: |
-          . ./Tools/Create-Azure-Sentinel-Solution/common/LogAppInsights.ps1
-          $runId = "${{ env.RUNID }}"
-          $solutionName = "${{ env.SOLUTION_NAME }}"
-          $pullRequestNumber = "${{ env.PULL_REQUEST_NUMBER }}"
-          $instrumentationKey = "${{ env.APPINSIGHTS }}"
-          try
-          {
-            $customProperties = @{ 'RunId'="$runId"; 'SolutionName'="$solutionName"; 'PullRequestNumber'="$pullRequestNumber"; 'EventName'="UpdateExistingPR"; }
-        
-            Send-AppInsightsTraceTelemetry -InstrumentationKey $instrumentationKey -Message "Execution for updating existing pull request started for Solution Name : $solutionName, Job Run Id : $runId" -Severity Information -CustomProperties $customProperties
-                    
-            Send-AppInsightsEventTelemetry -InstrumentationKey $instrumentationKey -EventName "UpdateExistingPR" -CustomProperties $customProperties
-          }
-          catch
-          {
-            $customProperties = @{ 'RunId'="$runId"; 'SolutionName'="$solutionName"; 'PullRequestNumber'="$pullRequestNumber"; 'EventName'="UpdateExistingPR"; 'update-Existing-pr-output'="failed" }
-            Send-AppInsightsExceptionTelemetry -InstrumentationKey $instrumentationKey -Exception $_.Exception -CustomProperties $customProperties
-            exit 1
-          }
+          git config --global user.email "noreply@github.com"
+          git config --global user.name "Github Bot"
+          git add -A
+          git commit -m '[skip ci] Github Bot Added package to Pull Request!'
+          git push

.github/workflows/package-on-merge.yaml

@@ -196,7 +196,6 @@ jobs:
         uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5
         with:
           committer: GitHub <noreply@github.com>
-          author: "v-atulyadav@microsoft.com>"
           assignees: "${{ env.ASSIGNEES }}"
           signoff: false
           branch: "${{ env.CURRENT_BRANCH_NAME }}-automated-pr"
@@ -237,6 +236,15 @@ jobs:
       message: "@${{ github.event.client_payload.github.actor }} The requested package for the '${{ needs.solutionNameDetails.outputs.solutionName }}' solution has been generated based on Pull request #${{ github.event.client_payload.pull_request.number }}"
       prNumber: "${{ needs.createNewPR.outputs.newPRNumber }}"
 
+  addAutoPackageLabel:
+    name: Add Label of Auto-Package
+    needs: createNewPR
+    if: ${{ success() }} 
+    uses: ./.github/workflows/addLabelOnPr.yaml
+    with:
+      labelName: "auto-package"
+    secrets: inherit
+
   release-project:
     name: Publish-Artifacts
     runs-on: ubuntu-latest

.github/workflows/slash-command-dispatch.yaml

@@ -11,12 +11,12 @@ on:
 
 jobs:
   checkAutomatedPR:
-    if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork && github.event.issue.pull_request && contains(github.event.comment.body, '/package') }}
+    if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork && github.event.issue.pull_request && (contains(github.event.comment.body, '/package') || contains(github.event.comment.body, '/Package')) }}
     uses: ./.github/workflows/checkAutomatedPR.yaml
 
   pullRequestStatus:
     needs: checkAutomatedPR
-    if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork && github.event.issue.pull_request && contains(github.event.comment.body, '/package') }}
+    if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork && github.event.issue.pull_request && (contains(github.event.comment.body, '/package') || contains(github.event.comment.body, '/Package')) }}
     uses: ./.github/workflows/pullRequestStatus.yaml
 
   slashCommandDispatch:
@@ -28,7 +28,19 @@ jobs:
         uses: peter-evans/slash-command-dispatch@67dfeb76529b35541a7c536976cba367cd2d364b
         with:
           token: ${{ env.GITHUB_TOKEN }}
-          commands: package
+          commands: |
+            package
+            Package
+            PACKAGE
           repository: ${{env.REPO_OWNER}}/${{env.REPO_NAME}}
           issue-type: pull-request
           reactions: false
+
+  addAutoPackageLabel:
+    name: Add Label of Auto-Package
+    needs: slashCommandDispatch
+    if: ${{ success() }} 
+    uses: ./.github/workflows/addLabelOnPr.yaml
+    with:
+      labelName: "auto-package"
+    secrets: inherit

.script/package-automation/package-generator.ps1

@@ -102,7 +102,7 @@ try {
         $playbookFiles = $playbookFiles -match ([regex]::Escape(".json"))
 
         if ($playbookFiles.Count -gt 0) {
-            $playbookFiles = $playbookFiles | Where-Object { $_ -notlike '*swagger*' -and $_ -notlike '*gov*' } | Where-Object { $_ -notlike '*function.json*' }
+            $playbookFiles = $playbookFiles | Where-Object { $_ -notlike '*swagger*' -and $_ -notlike '*gov*' -and $_ -notlike '*function.json' -and $_ -notlike '*host.json' }
         }
 
         return $playbookFiles;
@@ -521,11 +521,22 @@ try {
                 $playbooksFolderHasFunctionAppsInSolutionsFolder = @()
                 $playbooksFolderHasFunctionAppsInSolutionsFolder += $filteredPlaybookFunctionApps
 
-                $playbooksFunctionAppFiles += GetPlaybooksJsonFileNames($playbooksFolderHasFunctionAppsInSolutionsFolder)
+                $playbooksFunctionAppFilesInSolutionsFolder = GetPlaybooksJsonFileNames($playbooksFolderHasFunctionAppsInSolutionsFolder)
 
-                if ($playbooksFunctionAppFiles -gt 0)
+                if ($playbooksFunctionAppFilesInSolutionsFolder.Count -gt 0)
                 {
-                    $playbooksFunctionAppFiles = $playbooksFunctionAppFiles | ForEach-Object { $_.replace("$solutionFolderPath", '', 'OrdinalIgnoreCase') }
+                    $filteredPlaybooksFunctionAppFiles = $playbooksFunctionAppFilesInSolutionsFolder | ForEach-Object { $_.replace("$solutionFolderPath", '', 'OrdinalIgnoreCase') }
+
+                    if ($filteredPlaybooksFunctionAppFiles.Count -gt 0)
+                    {
+                        foreach($item in $filteredPlaybooksFunctionAppFiles)
+                        {
+                            if ($playbooksFunctionAppFiles -notcontains $item)
+                            {
+                                $playbooksFunctionAppFiles += $item
+                            }
+                        }
+                    }
                 }
             }
         }
@@ -738,14 +749,20 @@ try {
             # ADD REMAINING PLAYBOOKS
             foreach ($fl in $formulatePlaybooksList)
             {
-                $playbooksFinalList += $fl.Replace("$solutionFolderPath", '')
+                if ($playbooksFinalList -notcontains $fl)
+                {
+                    $playbooksFinalList += $fl.Replace("$solutionFolderPath", '')
+                }
             }
         }
         else 
         {
             foreach ($fl in $formulatePlaybooksList)
             {
-                $playbooksFinalList += $fl.Replace("$solutionFolderPath", '')
+                if ($playbooksFinalList -notcontains $fl)
+                {
+                    $playbooksFinalList += $fl.Replace("$solutionFolderPath", '')
+                }
             }
         }