From d8b4ba042ff244285c6ca656adfc8e6e2fccf9d2 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 10 Sep 2024 15:16:45 -0400 Subject: [PATCH 01/19] bugfix: get-cippazdatatable with -property filters --- .../Orchestrator Functions/Start-WebhookOrchestrator.ps1 | 4 ++-- Modules/CippEntrypoints/CippEntrypoints.psm1 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-WebhookOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-WebhookOrchestrator.ps1 index 6f6a6499755e..1ba797d3e10d 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-WebhookOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-WebhookOrchestrator.ps1 @@ -12,9 +12,9 @@ function Start-WebhookOrchestrator { Write-Information 'No webhook subscriptions found. Exiting.' return } - + $WebhookIncomingTable = Get-CIPPTable -TableName WebhookIncoming - $WebhookIncoming = Get-CIPPAzDataTableEntity @WebhookIncomingTable -Property RowKey + $WebhookIncoming = Get-CIPPAzDataTableEntity @WebhookIncomingTable -Property PartitionKey, RowKey if (($WebhookIncoming | Measure-Object).Count -eq 0) { Write-Information 'No webhook incoming found. Exiting.' return diff --git a/Modules/CippEntrypoints/CippEntrypoints.psm1 b/Modules/CippEntrypoints/CippEntrypoints.psm1 index e5dd31931fb4..9dd8e857fb41 100644 --- a/Modules/CippEntrypoints/CippEntrypoints.psm1 +++ b/Modules/CippEntrypoints/CippEntrypoints.psm1 @@ -242,7 +242,7 @@ function Receive-CIPPTimerTrigger { if ($FunctionStatus.OrchestratorId) { $FunctionName = $env:WEBSITE_SITE_NAME $InstancesTable = Get-CippTable -TableName ('{0}Instances' -f ($FunctionName -replace '-', '')) - $Instance = Get-CIPPAzDataTableEntity @InstancesTable -Filter "PartitionKey eq '$($FunctionStatus.OrchestratorId)'" -Property RuntimeStatus + $Instance = Get-CIPPAzDataTableEntity @InstancesTable -Filter "PartitionKey eq '$($FunctionStatus.OrchestratorId)'" -Property PartitionKey, RowKey, RuntimeStatus if ($Instance.RuntimeStatus -eq 'Running') { Write-LogMessage -API 'TimerFunction' -message "$($Function.Command) - $($FunctionStatus.OrchestratorId) is still running" -sev Warn -LogData $FunctionStatus Write-Warning "CIPP Timer: $($Function.Command) - $($FunctionStatus.OrchestratorId) is still running, skipping execution" From f0779da75876b986586d306b0674f741b6abc219 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 10 Sep 2024 15:17:02 -0400 Subject: [PATCH 02/19] add caching gdap / access checks --- .../CIPP/Settings/Invoke-ExecAccessChecks.ps1 | 15 +++++++++++++-- .../Public/Test-CIPPAccessPermissions.ps1 | 18 +++++++++++++++++- .../Public/Test-CIPPGDAPRelationships.ps1 | 17 ++++++++++++++++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecAccessChecks.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecAccessChecks.ps1 index 557be474dd5a..08d9b59cc620 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecAccessChecks.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecAccessChecks.ps1 @@ -13,18 +13,29 @@ Function Invoke-ExecAccessChecks { $APIName = $TriggerMetadata.FunctionName Write-LogMessage -user $request.headers.'x-ms-client-principal' -API $APINAME -message 'Accessed this API' -Sev 'Debug' + $Table = Get-CIPPTable -tablename 'AccessChecks' # Write to the Azure Functions log stream. Write-Host 'PowerShell HTTP trigger function processed a request.' if ($Request.Query.Permissions -eq 'true') { - $Results = Test-CIPPAccessPermissions -tenantfilter $ENV:TenantID -APIName $APINAME -ExecutingUser $Request.Headers.'x-ms-client-principal' + if ($Request.Query.Cached -eq 'true') { + $Data = (Get-CIPPAzDataTableEntity @Table -Filter "RowKey eq 'AccessPermissions'").Data | ConvertFrom-Json + $Results = $Data + } else { + $Results = Test-CIPPAccessPermissions -tenantfilter $ENV:TenantID -APIName $APINAME -ExecutingUser $Request.Headers.'x-ms-client-principal' + } } if ($Request.Query.Tenants -eq 'true') { $Results = Test-CIPPAccessTenant -TenantCSV $Request.Body.tenantid -ExecutingUser $Request.Headers.'x-ms-client-principal' } if ($Request.Query.GDAP -eq 'true') { - $Results = Test-CIPPGDAPRelationships + if ($Request.Query.Cached -eq 'true') { + $Data = (Get-CIPPAzDataTableEntity @Table -Filter "RowKey eq 'GDAPRelationships'").Data | ConvertFrom-Json + $Results = $Data + } else { + $Results = Test-CIPPGDAPRelationships + } } $body = [pscustomobject]@{'Results' = $Results } diff --git a/Modules/CIPPCore/Public/Test-CIPPAccessPermissions.ps1 b/Modules/CIPPCore/Public/Test-CIPPAccessPermissions.ps1 index f46d290536c7..ff9878957e2b 100644 --- a/Modules/CIPPCore/Public/Test-CIPPAccessPermissions.ps1 +++ b/Modules/CIPPCore/Public/Test-CIPPAccessPermissions.ps1 @@ -123,7 +123,8 @@ function Test-CIPPAccessPermissions { if ($Success -eq $true) { $Messages.Add('No service account issues have been found. CIPP is ready for use.') | Out-Null } - return [PSCustomObject]@{ + + $AccessCheck = [PSCustomObject]@{ AccessTokenDetails = $AccessTokenDetails Messages = @($Messages) ErrorMessages = @($ErrorMessages) @@ -131,4 +132,19 @@ function Test-CIPPAccessPermissions { Links = @($Links) Success = $Success } + + $Table = Get-CIPPTable -TableName AccessChecks + $Data = Get-CIPPAzDataTableEntity @Table -Filter "PartitionKey eq 'AccessCheck' and RowKey eq 'AccessPermissions'" + if ($Data) { + $Data.Data = [string](ConvertTo-Json -InputObject $AccessCheck -Depth 10 -Compress) + } else { + $Data = @{ + PartitionKey = 'AccessCheck' + RowKey = 'AccessPermissions' + Data = [string](ConvertTo-Json -InputObject $AccessCheck -Depth 10 -Compress) + } + } + Add-CIPPAzDataTableEntity @Table -Entity $Data -Force + + return $AccessCheck } diff --git a/Modules/CIPPCore/Public/Test-CIPPGDAPRelationships.ps1 b/Modules/CIPPCore/Public/Test-CIPPGDAPRelationships.ps1 index 2d8b95ff2ea2..71b7cae9808a 100644 --- a/Modules/CIPPCore/Public/Test-CIPPGDAPRelationships.ps1 +++ b/Modules/CIPPCore/Public/Test-CIPPGDAPRelationships.ps1 @@ -93,10 +93,25 @@ function Test-CIPPGDAPRelationships { Write-LogMessage -user $ExecutingUser -API $APINAME -message "Failed to run GDAP check for $($TenantFilter): $($ErrorMessage.NormalizedError)" -Sev 'Error' -LogData $ErrorMessage } - return [PSCustomObject]@{ + $GDAPRelationships = [PSCustomObject]@{ GDAPIssues = @($GDAPissues) MissingGroups = @($MissingGroups) Memberships = @($SAMUserMemberships) CIPPGroupCount = $CIPPGroupCount } + + $Table = Get-CIPPTable -TableName AccessChecks + $Data = Get-CIPPAzDataTableEntity @Table -Filter "PartitionKey eq 'AccessCheck' and RowKey eq 'GDAPRelationships'" + if ($Data) { + $Data.Data = [string](ConvertTo-Json -InputObject $GDAPRelationships -Depth 10 -Compress) + } else { + $Data = @{ + PartitionKey = 'AccessCheck' + RowKey = 'GDAPRelationships' + Data = [string](ConvertTo-Json -InputObject $GDAPRelationships -Depth 10 -Compress) + } + } + Add-CIPPAzDataTableEntity @Table -Entity $Data -Force + + return $GDAPRelationships } From dececcd274094a57bbd21b3639e85f8bac19c4b9 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 10 Sep 2024 15:17:19 -0400 Subject: [PATCH 03/19] add try/catch to custom roles --- .../CIPP/Settings/Invoke-ExecCustomRole.ps1 | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecCustomRole.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecCustomRole.ps1 index c5725a2cafb8..44c87fd92425 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecCustomRole.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecCustomRole.ps1 @@ -39,14 +39,26 @@ function Invoke-ExecCustomRole { ) } else { $Body = foreach ($Role in $Body) { - $Role.Permissions = $Role.Permissions | ConvertFrom-Json + try { + $Role.Permissions = $Role.Permissions | ConvertFrom-Json + } catch { + $Role.Permissions = '' + } if ($Role.AllowedTenants) { - $Role.AllowedTenants = @($Role.AllowedTenants | ConvertFrom-Json) + try { + $Role.AllowedTenants = @($Role.AllowedTenants | ConvertFrom-Json) + } catch { + $Role.AllowedTenants = '' + } } else { $Role | Add-Member -NotePropertyName AllowedTenants -NotePropertyValue @() -Force } if ($Role.BlockedTenants) { - $Role.BlockedTenants = @($Role.BlockedTenants | ConvertFrom-Json) + try { + $Role.BlockedTenants = @($Role.BlockedTenants | ConvertFrom-Json) + } catch { + $Role.BlockedTenants = '' + } } else { $Role | Add-Member -NotePropertyName BlockedTenants -NotePropertyValue @() -Force } From a2124e0f03365ebaaf814b702fa49feaf67ed198 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Tue, 10 Sep 2024 15:18:20 -0400 Subject: [PATCH 04/19] add app secret support to graphtoken --- .../CIPPCore/Public/GraphHelper/Get-GraphToken.ps1 | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/GraphHelper/Get-GraphToken.ps1 b/Modules/CIPPCore/Public/GraphHelper/Get-GraphToken.ps1 index b0021973a701..6883f9b5cce0 100644 --- a/Modules/CIPPCore/Public/GraphHelper/Get-GraphToken.ps1 +++ b/Modules/CIPPCore/Public/GraphHelper/Get-GraphToken.ps1 @@ -1,4 +1,4 @@ -function Get-GraphToken($tenantid, $scope, $AsApp, $AppID, $refreshToken, $ReturnRefresh, $SkipCache) { +function Get-GraphToken($tenantid, $scope, $AsApp, $AppID, $AppSecret, $refreshToken, $ReturnRefresh, $SkipCache) { <# .FUNCTIONALITY Internal @@ -30,6 +30,15 @@ function Get-GraphToken($tenantid, $scope, $AsApp, $AppID, $refreshToken, $Retur } } + if ($null -ne $AppID -and $null -ne $AppSecret) { + $AuthBody = @{ + client_id = $AppID + client_secret = $AppSecret + scope = $Scope + grant_type = 'client_credentials' + } + } + if (!$tenantid) { $tenantid = $env:TenantID } $TokenKey = '{0}-{1}-{2}' -f $tenantid, $scope, $asApp From 31818f93485dd172d6b55ec2229492aefe69f7cb Mon Sep 17 00:00:00 2001 From: John Duprey Date: Sat, 14 Sep 2024 15:18:28 -0400 Subject: [PATCH 05/19] Dev token support --- .../Entrypoints/Timer Functions/Start-UpdateTokensTimer.ps1 | 2 +- Modules/CIPPCore/Public/Get-CIPPAuthentication.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-UpdateTokensTimer.ps1 b/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-UpdateTokensTimer.ps1 index 98e4f07891df..1f7cd63a8d3b 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-UpdateTokensTimer.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-UpdateTokensTimer.ps1 @@ -18,7 +18,7 @@ function Start-UpdateTokensTimer { $Secret = Get-CIPPAzDataTableEntity @Table -Filter "PartitionKey eq 'Secret' and RowKey eq 'Secret'" if ($Secret) { $Secret.RefreshToken = $Refreshtoken - Add-AzDataTableEntity @Table -Entity $Secret + Add-AzDataTableEntity @Table -Entity $Secret -Force } else { Write-LogMessage -message 'Could not update refresh token. Will try again in 7 days.' -sev 'CRITICAL' } diff --git a/Modules/CIPPCore/Public/Get-CIPPAuthentication.ps1 b/Modules/CIPPCore/Public/Get-CIPPAuthentication.ps1 index 33b9f119954e..7f12a2e481b3 100644 --- a/Modules/CIPPCore/Public/Get-CIPPAuthentication.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPAuthentication.ps1 @@ -9,7 +9,7 @@ function Get-CIPPAuthentication { try { if ($env:AzureWebJobsStorage -eq 'UseDevelopmentStorage=true') { $Table = Get-CIPPTable -tablename 'DevSecrets' - $Secret = Get-CIPPAzDataTableEntity @Table -Filter "PartitionKey eq 'Secret' and RowKey eq 'Secret'" + $Secret = Get-AzDataTableEntity @Table -Filter "PartitionKey eq 'Secret' and RowKey eq 'Secret'" if (!$Secret) { throw 'Development variables not set' } From 63447f38f2a4e277fb39191dcff6e56f7a6b0eb8 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 20:49:49 -0400 Subject: [PATCH 06/19] multi-node support --- CIPPTimers.json | 9 ++-- .../Public/Get-CIPPAuthentication.ps1 | 2 +- .../Public/Get-CIPPTimerFunctions.ps1 | 54 +++++++++++++------ Modules/CippEntrypoints/CippEntrypoints.psm1 | 1 + .../Public/Gradient/Get-GradientToken.ps1 | 2 +- .../Public/Hudu/Connect-HuduAPI.ps1 | 2 +- .../Public/NinjaOne/Get-NinjaOneToken.ps1 | 2 +- profile.ps1 | 4 +- 8 files changed, 50 insertions(+), 26 deletions(-) diff --git a/CIPPTimers.json b/CIPPTimers.json index 746d643494a8..dd15ba8095ba 100644 --- a/CIPPTimers.json +++ b/CIPPTimers.json @@ -4,14 +4,16 @@ "Description": "Orchestrator to process user scheduled tasks", "Cron": "0 */15 * * * *", "Priority": 1, - "RunOnProcessor": true + "RunOnProcessor": true, + "PreferredProcessor": "usertasks" }, { "Command": "Start-AuditLogOrchestrator", "Description": "Orchestrator to process audit logs", "Cron": "0 */15 * * * *", "Priority": 2, - "RunOnProcessor": true + "RunOnProcessor": true, + "PreferredProcessor": "auditlog" }, { "Command": "Start-WebhookOrchestrator", @@ -25,7 +27,8 @@ "Description": "Orchestrator to process standards", "Cron": "0 0 */4 * * *", "Priority": 4, - "RunOnProcessor": true + "RunOnProcessor": true, + "PreferredProcessor": "standards" }, { "Command": "Start-CIPPGraphSubscriptionCleanupTimer", diff --git a/Modules/CIPPCore/Public/Get-CIPPAuthentication.ps1 b/Modules/CIPPCore/Public/Get-CIPPAuthentication.ps1 index 7f12a2e481b3..4018dd43154b 100644 --- a/Modules/CIPPCore/Public/Get-CIPPAuthentication.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPAuthentication.ps1 @@ -20,7 +20,7 @@ function Get-CIPPAuthentication { } } else { Connect-AzAccount -Identity - $keyvaultname = $ENV:WEBSITE_DEPLOYMENT_ID -replace '-proc$', '' + $keyvaultname = ($ENV:WEBSITE_DEPLOYMENT_ID -split '-')[0] $Variables | ForEach-Object { Set-Item -Path ENV:$_ -Value (Get-AzKeyVaultSecret -VaultName $keyvaultname -Name $_ -AsPlainText -ErrorAction Stop) -Force } diff --git a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 index 07bbb7e36a15..bf52de396001 100644 --- a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 @@ -8,6 +8,18 @@ function Get-CIPPTimerFunctions { $ConfigTable = Get-CIPPTable -tablename Config $Config = Get-CIPPAzDataTableEntity @ConfigTable -Filter "PartitionKey eq 'OffloadFunctions' and RowKey eq 'OffloadFunctions'" + # Check running nodes + $VersionTable = Get-CIPPTable -tablename 'Version' + $Nodes = Get-CIPPAzDataTableEntity @VersionTable -Filter "PartitionKey eq 'Version' and RowKey ne 'Version'" | Where-Object { $_.RowKey -match '-' } + $AvailableNodes = $Nodes.RowKey | ForEach-Object { ($_ -split '-')[1] } + + # Get node name + if ($FunctionName -match '-') { + $Node = ($FunctionName -split '-')[1] + } else { + $Node = 'http' + } + $RunOnProcessor = $true if ($Config -and $Config.state -eq $true) { if ($env:CIPP_PROCESSOR -ne 'true' -and !$All.IsPresent) { @@ -46,6 +58,11 @@ function Get-CIPPTimerFunctions { continue } + if ($AvailableNodes -contains $Orchestrator.PreferredProcessor -and $Node -ne $Orchestrator.PreferredProcessor) { + # only run on preferred processor when available + continue + } + $Now = Get-Date if ($All.IsPresent) { $NextOccurrence = [datetime]$Cron.GetNextOccurrence($Now) @@ -62,15 +79,16 @@ function Get-CIPPTimerFunctions { if ($NextOccurrence) { if (!$Status) { $Status = [pscustomobject]@{ - PartitionKey = 'Timer' - RowKey = $Orchestrator.Command - Cron = $CronString - LastOccurrence = 'Never' - NextOccurrence = $NextOccurrence.ToUniversalTime() - Status = 'Not Scheduled' - OrchestratorId = '' - RunOnProcessor = $RunOnProcessor - IsSystem = $Orchestrator.IsSystem ?? $false + PartitionKey = 'Timer' + RowKey = $Orchestrator.Command + Cron = $CronString + LastOccurrence = 'Never' + NextOccurrence = $NextOccurrence.ToUniversalTime() + Status = 'Not Scheduled' + OrchestratorId = '' + RunOnProcessor = $RunOnProcessor + IsSystem = $Orchestrator.IsSystem ?? $false + PreferredProcessor = $Orchestrator.PreferredProcessor } Add-CIPPAzDataTableEntity @Table -Entity $Status } else { @@ -78,18 +96,20 @@ function Get-CIPPTimerFunctions { $Status.Cron = $CronString } $Status.NextOccurrence = $NextOccurrence.ToUniversalTime() + $Status.PreferredProcessor = $Orchestrator.PreferredProcessor Add-CIPPAzDataTableEntity @Table -Entity $Status -Force } [PSCustomObject]@{ - Command = $Orchestrator.Command - Cron = $CronString - NextOccurrence = $NextOccurrence.ToUniversalTime() - LastOccurrence = $Status.LastOccurrence.DateTime - Status = $Status.Status - OrchestratorId = $Status.OrchestratorId - RunOnProcessor = $Orchestrator.RunOnProcessor - IsSystem = $Orchestrator.IsSystem ?? $false + Command = $Orchestrator.Command + Cron = $CronString + NextOccurrence = $NextOccurrence.ToUniversalTime() + LastOccurrence = $Status.LastOccurrence.DateTime + Status = $Status.Status + OrchestratorId = $Status.OrchestratorId + RunOnProcessor = $Orchestrator.RunOnProcessor + IsSystem = $Orchestrator.IsSystem ?? $false + PreferredProcessor = $Orchestrator.PreferredProcessor } } } else { diff --git a/Modules/CippEntrypoints/CippEntrypoints.psm1 b/Modules/CippEntrypoints/CippEntrypoints.psm1 index 9dd8e857fb41..6286b64246f0 100644 --- a/Modules/CippEntrypoints/CippEntrypoints.psm1 +++ b/Modules/CippEntrypoints/CippEntrypoints.psm1 @@ -235,6 +235,7 @@ function Receive-CIPPTimerTrigger { $Functions = Get-CIPPTimerFunctions $Table = Get-CIPPTable -tablename CIPPTimers $Statuses = Get-CIPPAzDataTableEntity @Table + $FunctionName = $env:WEBSITE_SITE_NAME foreach ($Function in $Functions) { Write-Information "CIPPTimer: $($Function.Command) - $($Function.Cron)" diff --git a/Modules/CippExtensions/Public/Gradient/Get-GradientToken.ps1 b/Modules/CippExtensions/Public/Gradient/Get-GradientToken.ps1 index 641020e3e70a..a944d16b805d 100644 --- a/Modules/CippExtensions/Public/Gradient/Get-GradientToken.ps1 +++ b/Modules/CippExtensions/Public/Gradient/Get-GradientToken.ps1 @@ -4,7 +4,7 @@ function Get-GradientToken { ) if ($Configuration.vendorKey) { $null = Connect-AzAccount -Identity - $keyvaultname = $ENV:WEBSITE_DEPLOYMENT_ID -replace '-proc$', '' + $keyvaultname = ($ENV:WEBSITE_DEPLOYMENT_ID -split '-')[0] $partnerApiKey = (Get-AzKeyVaultSecret -VaultName $keyvaultname -Name 'Gradient' -AsPlainText) $authorizationToken = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes("$($configuration.vendorKey):$($partnerApiKey)")) diff --git a/Modules/CippExtensions/Public/Hudu/Connect-HuduAPI.ps1 b/Modules/CippExtensions/Public/Hudu/Connect-HuduAPI.ps1 index 58f4b7080232..05c2f9b1e78f 100644 --- a/Modules/CippExtensions/Public/Hudu/Connect-HuduAPI.ps1 +++ b/Modules/CippExtensions/Public/Hudu/Connect-HuduAPI.ps1 @@ -8,7 +8,7 @@ function Connect-HuduAPI { $DevSecretsTable = Get-CIPPTable -tablename 'DevSecrets' $APIKey = (Get-CIPPAzDataTableEntity @DevSecretsTable -Filter "PartitionKey eq 'Hudu' and RowKey eq 'Hudu'").APIKey } else { - $keyvaultname = $ENV:WEBSITE_DEPLOYMENT_ID -replace '-proc$', '' + $keyvaultname = ($ENV:WEBSITE_DEPLOYMENT_ID -split '-')[0] $null = Connect-AzAccount -Identity $APIKey = (Get-AzKeyVaultSecret -VaultName $keyvaultname -Name 'Hudu' -AsPlainText) } diff --git a/Modules/CippExtensions/Public/NinjaOne/Get-NinjaOneToken.ps1 b/Modules/CippExtensions/Public/NinjaOne/Get-NinjaOneToken.ps1 index 6bf5490c7fec..d7102d7d0e27 100644 --- a/Modules/CippExtensions/Public/NinjaOne/Get-NinjaOneToken.ps1 +++ b/Modules/CippExtensions/Public/NinjaOne/Get-NinjaOneToken.ps1 @@ -11,7 +11,7 @@ function Get-NinjaOneToken { $ClientSecret = (Get-CIPPAzDataTableEntity @DevSecretsTable -Filter "PartitionKey eq 'NinjaOne' and RowKey eq 'NinjaOne'").APIKey } else { $null = Connect-AzAccount -Identity - $keyvaultname = $ENV:WEBSITE_DEPLOYMENT_ID -replace '-proc$', '' + $keyvaultname = ($ENV:WEBSITE_DEPLOYMENT_ID -split '-')[0] $ClientSecret = (Get-AzKeyVaultSecret -VaultName $keyvaultname -Name 'NinjaOne' -AsPlainText) } } else { diff --git a/profile.ps1 b/profile.ps1 index c4f49830181a..85ada2f4dc2f 100644 --- a/profile.ps1 +++ b/profile.ps1 @@ -49,7 +49,7 @@ try { Set-Location -Path $PSScriptRoot $CurrentVersion = (Get-Content .\version_latest.txt).trim() $Table = Get-CippTable -tablename 'Version' -$LastStartup = Get-CIPPAzDataTableEntity @Table -Filter "PartitionKey eq 'Version' and RowKey eq 'Version'" +$LastStartup = Get-CIPPAzDataTableEntity @Table -Filter "PartitionKey eq 'Version' and RowKey eq '$($env:WEBSITE_SITE_NAME)'" if ($CurrentVersion -ne $LastStartup.Version) { Write-Host "Version has changed from $($LastStartup.Version) to $CurrentVersion" Clear-CippDurables @@ -58,7 +58,7 @@ if ($CurrentVersion -ne $LastStartup.Version) { } else { $LastStartup = [PSCustomObject]@{ PartitionKey = 'Version' - RowKey = 'Version' + RowKey = $env:WEBSITE_SITE_NAME Version = $CurrentVersion } } From d1f4fb97cbe7f3fcb4b778815b62be938803878b Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 21:01:52 -0400 Subject: [PATCH 07/19] Update logging and clear function --- Modules/CIPPCore/Public/Clear-CippDurables.ps1 | 2 +- profile.ps1 | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Modules/CIPPCore/Public/Clear-CippDurables.ps1 b/Modules/CIPPCore/Public/Clear-CippDurables.ps1 index 089ca10282a7..7b230eaf6b96 100644 --- a/Modules/CIPPCore/Public/Clear-CippDurables.ps1 +++ b/Modules/CIPPCore/Public/Clear-CippDurables.ps1 @@ -3,7 +3,7 @@ function Clear-CippDurables { Param() # Collect info $StorageContext = New-AzStorageContext -ConnectionString $env:AzureWebJobsStorage - $FunctionName = $env:WEBSITE_SITE_NAME + $FunctionName = $env:WEBSITE_SITE_NAME -replace '-', '' # Get orchestrators $InstancesTable = Get-CippTable -TableName ('{0}Instances' -f $FunctionName) diff --git a/profile.ps1 b/profile.ps1 index 85ada2f4dc2f..173bc71ffb8c 100644 --- a/profile.ps1 +++ b/profile.ps1 @@ -26,7 +26,7 @@ if ($env:ExternalDurablePowerShellSDK -eq $true) { try { Import-Module AzureFunctions.PowerShell.Durable.SDK -ErrorAction Stop - Write-Host 'External Durable SDK enabled' + Write-Information 'External Durable SDK enabled' } catch { Write-LogMessage -message 'Failed to import module - AzureFunctions.PowerShell.Durable.SDK' -LogData (Get-CippException -Exception $_) -Sev 'debug' $_.Exception.Message @@ -39,7 +39,7 @@ try { try { if (!$ENV:SetFromProfile) { - Write-Host "We're reloading from KV" + Write-Information "We're reloading from KV" $Auth = Get-CIPPAuthentication } } catch { @@ -49,9 +49,10 @@ try { Set-Location -Path $PSScriptRoot $CurrentVersion = (Get-Content .\version_latest.txt).trim() $Table = Get-CippTable -tablename 'Version' +Write-Information "Function: $($env:WEBSITE_SITE_NAME) Version: $CurrentVersion" $LastStartup = Get-CIPPAzDataTableEntity @Table -Filter "PartitionKey eq 'Version' and RowKey eq '$($env:WEBSITE_SITE_NAME)'" -if ($CurrentVersion -ne $LastStartup.Version) { - Write-Host "Version has changed from $($LastStartup.Version) to $CurrentVersion" +if (!$LastStartup -or $CurrentVersion -ne $LastStartup.Version) { + Write-Information "Version has changed from $($LastStartup.Version ?? 'None') to $CurrentVersion" Clear-CippDurables if ($LastStartup) { $LastStartup.Version = $CurrentVersion From fddc60ea96cb24571ad2f3fc467c3c0de53c4c41 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 21:09:21 -0400 Subject: [PATCH 08/19] Fix cleanup function --- Modules/CIPPCore/Public/Clear-CippDurables.ps1 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Modules/CIPPCore/Public/Clear-CippDurables.ps1 b/Modules/CIPPCore/Public/Clear-CippDurables.ps1 index 7b230eaf6b96..b63d399647d6 100644 --- a/Modules/CIPPCore/Public/Clear-CippDurables.ps1 +++ b/Modules/CIPPCore/Public/Clear-CippDurables.ps1 @@ -8,9 +8,6 @@ function Clear-CippDurables { # Get orchestrators $InstancesTable = Get-CippTable -TableName ('{0}Instances' -f $FunctionName) $HistoryTable = Get-CippTable -TableName ('{0}History' -f $FunctionName) - $Yesterday = (Get-Date).AddDays(-1).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') - $Filter = "CreatedTime ge datetime'$Yesterday' or RuntimeStatus eq 'Pending' or RuntimeStatus eq 'Running'" - $Instances = Get-CippAzDataTableEntity @InstancesTable -Filter $Filter $Queues = Get-AzStorageQueue -Context $StorageContext -Name ('{0}*' -f $FunctionName) | Select-Object -Property Name, ApproximateMessageCount, QueueClient @@ -50,7 +47,7 @@ function Clear-CippDurables { Remove-AzDataTable @InstancesTable Remove-AzDataTable @HistoryTable - $BlobContainer = '{0}-largemessages' -f $Function.Name + $BlobContainer = '{0}-largemessages' -f $FunctionName if (Get-AzStorageContainer -Name $BlobContainer -Context $StorageContext -ErrorAction SilentlyContinue) { Write-Information "- Removing blob container: $BlobContainer" if ($PSCmdlet.ShouldProcess($BlobContainer, 'Remove Blob Container')) { From ee83996d7de90317f91c83449dfeb673b86a2be8 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 21:16:34 -0400 Subject: [PATCH 09/19] Update Get-CIPPTimerFunctions.ps1 --- Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 index bf52de396001..e6cdbeefa673 100644 --- a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 @@ -96,7 +96,11 @@ function Get-CIPPTimerFunctions { $Status.Cron = $CronString } $Status.NextOccurrence = $NextOccurrence.ToUniversalTime() - $Status.PreferredProcessor = $Orchestrator.PreferredProcessor + if (!$Status.PSObject.Properites.Name -Contains 'PreferredProcessor') { + $Status | Add-Member -MemberType NoteProperty -Name 'PreferredProcessor' -Value $Orchestrator.PreferredProcessor + } else { + $Status.PreferredProcessor = $Orchestrator.PreferredProcessor + } Add-CIPPAzDataTableEntity @Table -Entity $Status -Force } From 6e0dc2f39280fd8f4740dc94fc3fa248a364b8f9 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 21:38:51 -0400 Subject: [PATCH 10/19] Update Get-CIPPTimerFunctions.ps1 --- Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 index e6cdbeefa673..7ee5582ad275 100644 --- a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 @@ -58,7 +58,7 @@ function Get-CIPPTimerFunctions { continue } - if ($AvailableNodes -contains $Orchestrator.PreferredProcessor -and $Node -ne $Orchestrator.PreferredProcessor) { + if (($AvailableNodes -contains $Orchestrator.PreferredProcessor -and $Node -ne $Orchestrator.PreferredProcessor) -and ($Node -notin ('http', 'proc'))) { # only run on preferred processor when available continue } From a536d1de92bfb75c738e22ac0d203c3d9084f9c2 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 21:44:24 -0400 Subject: [PATCH 11/19] Update Get-CIPPTimerFunctions.ps1 --- Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 index 7ee5582ad275..83b037a9a924 100644 --- a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 @@ -96,8 +96,8 @@ function Get-CIPPTimerFunctions { $Status.Cron = $CronString } $Status.NextOccurrence = $NextOccurrence.ToUniversalTime() - if (!$Status.PSObject.Properites.Name -Contains 'PreferredProcessor') { - $Status | Add-Member -MemberType NoteProperty -Name 'PreferredProcessor' -Value $Orchestrator.PreferredProcessor + if ($Status.PSObject.Properites.Name -notcontains 'PreferredProcessor') { + $Status | Add-Member -MemberType NoteProperty -Name 'PreferredProcessor' -Value $Orchestrator.PreferredProcessor -Force } else { $Status.PreferredProcessor = $Orchestrator.PreferredProcessor } From f3c8d25b6203075b892a9da5d0e53705950a5af1 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 21:50:58 -0400 Subject: [PATCH 12/19] Update Get-CIPPTimerFunctions.ps1 --- Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 index 83b037a9a924..30ec0b251531 100644 --- a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 @@ -58,9 +58,12 @@ function Get-CIPPTimerFunctions { continue } - if (($AvailableNodes -contains $Orchestrator.PreferredProcessor -and $Node -ne $Orchestrator.PreferredProcessor) -and ($Node -notin ('http', 'proc'))) { + if ($AvailableNodes -contains $Orchestrator.PreferredProcessor -and $Node -ne $Orchestrator.PreferredProcessor) { # only run on preferred processor when available continue + } elseif ($AvailableNodes -notcontains $Orchestrator.PreferredProcessor -and $Node -notin ('http', 'proc')) { + # Catchall function nodes + continue } $Now = Get-Date From 839a63265700963c7a737248e4575a386af825df Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 21:57:25 -0400 Subject: [PATCH 13/19] prevent duplicate queue executions --- .../Queue Function/Push-CIPPFunctionProcessor.ps1 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 b/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 index f40930a00f7e..b3e0529072db 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 @@ -11,7 +11,14 @@ function Push-CIPPFunctionProcessor { $ConfigTable = Get-CIPPTable -tablename Config $Config = Get-CIPPAzDataTableEntity @ConfigTable -Filter "PartitionKey eq 'OffloadFunctions' and RowKey eq 'OffloadFunctions'" - if ($Config -and $Config.state -eq $true) { + $FunctionName = $env:WEBSITE_SITE_NAME + if ($FunctionName -match '-') { + $Node = ($FunctionName -split '-')[1] + } else { + $Node = 'http' + } + + if ($Config -and $Config.state -eq $true -and $Node -eq 'proc') { if ($env:CIPP_PROCESSOR -ne 'true') { return } From 2ceb54129179310cde13da8567e7eb4c68a5e087 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 22:10:08 -0400 Subject: [PATCH 14/19] Update Get-CIPPTimerFunctions.ps1 --- Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 index 30ec0b251531..2d5425c8d992 100644 --- a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 @@ -58,10 +58,10 @@ function Get-CIPPTimerFunctions { continue } - if ($AvailableNodes -contains $Orchestrator.PreferredProcessor -and $Node -ne $Orchestrator.PreferredProcessor) { + if ($Orchestrator.PreferredProcessor -and $AvailableNodes -contains $Orchestrator.PreferredProcessor -and $Node -ne $Orchestrator.PreferredProcessor) { # only run on preferred processor when available continue - } elseif ($AvailableNodes -notcontains $Orchestrator.PreferredProcessor -and $Node -notin ('http', 'proc')) { + } elseif ((!$Orchestrator.PreferredProcessor -or $AvailableNodes -notcontains $Orchestrator.PreferredProcessor) -and $Node -notin ('http', 'proc')) { # Catchall function nodes continue } From dfdf96cc71820534055b26cd89bcc633f9aa1c4b Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 22:14:27 -0400 Subject: [PATCH 15/19] Update Get-CIPPTimerFunctions.ps1 --- Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 index 2d5425c8d992..b814f7a11b8f 100644 --- a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 @@ -12,6 +12,7 @@ function Get-CIPPTimerFunctions { $VersionTable = Get-CIPPTable -tablename 'Version' $Nodes = Get-CIPPAzDataTableEntity @VersionTable -Filter "PartitionKey eq 'Version' and RowKey ne 'Version'" | Where-Object { $_.RowKey -match '-' } $AvailableNodes = $Nodes.RowKey | ForEach-Object { ($_ -split '-')[1] } + $FunctionName = $env:WEBSITE_SITE_NAME # Get node name if ($FunctionName -match '-') { From ae84a0e34d9033779ac8427b5880d1111f80e953 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 22:23:32 -0400 Subject: [PATCH 16/19] Update Get-CIPPTimerFunctions.ps1 --- Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 index b814f7a11b8f..8454a21e7b48 100644 --- a/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 +++ b/Modules/CIPPCore/Public/Get-CIPPTimerFunctions.ps1 @@ -92,7 +92,7 @@ function Get-CIPPTimerFunctions { OrchestratorId = '' RunOnProcessor = $RunOnProcessor IsSystem = $Orchestrator.IsSystem ?? $false - PreferredProcessor = $Orchestrator.PreferredProcessor + PreferredProcessor = $Orchestrator.PreferredProcessor ?? '' } Add-CIPPAzDataTableEntity @Table -Entity $Status } else { @@ -100,10 +100,11 @@ function Get-CIPPTimerFunctions { $Status.Cron = $CronString } $Status.NextOccurrence = $NextOccurrence.ToUniversalTime() + $PreferredProcessor = $Orchestrator.PreferredProcessor ?? '' if ($Status.PSObject.Properites.Name -notcontains 'PreferredProcessor') { - $Status | Add-Member -MemberType NoteProperty -Name 'PreferredProcessor' -Value $Orchestrator.PreferredProcessor -Force + $Status | Add-Member -MemberType NoteProperty -Name 'PreferredProcessor' -Value $PreferredProcessor -Force } else { - $Status.PreferredProcessor = $Orchestrator.PreferredProcessor + $Status.PreferredProcessor = $PreferredProcessor } Add-CIPPAzDataTableEntity @Table -Entity $Status -Force } @@ -117,7 +118,7 @@ function Get-CIPPTimerFunctions { OrchestratorId = $Status.OrchestratorId RunOnProcessor = $Orchestrator.RunOnProcessor IsSystem = $Orchestrator.IsSystem ?? $false - PreferredProcessor = $Orchestrator.PreferredProcessor + PreferredProcessor = $Orchestrator.PreferredProcessor ?? '' } } } else { From 03d17e77457c123ce58154a66ec77c93182e2d21 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 22:37:25 -0400 Subject: [PATCH 17/19] Update Push-CIPPFunctionProcessor.ps1 --- .../Queue Function/Push-CIPPFunctionProcessor.ps1 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 b/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 index b3e0529072db..26111a5dc609 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 @@ -18,10 +18,11 @@ function Push-CIPPFunctionProcessor { $Node = 'http' } + if ($env:CIPP_PROCESSOR -ne 'true') { + return + } if ($Config -and $Config.state -eq $true -and $Node -eq 'proc') { - if ($env:CIPP_PROCESSOR -ne 'true') { - return - } + return } $Parameters = $QueueItem.Parameters | ConvertTo-Json -Depth 10 | ConvertFrom-Json -AsHashtable From 9da04b29665e8bf2d97fae2b9a09fc729a8304f7 Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 23:12:56 -0400 Subject: [PATCH 18/19] Switch from queue to timer function and table --- CIPPTimers.json | 7 ++++ .../Applications/Invoke-ExecAppUpload.ps1 | 12 ++++--- .../Tenant/Standards/Invoke-ExecBPA.ps1 | 18 ++++++---- .../Standards/Invoke-ExecDomainAnalyser.ps1 | 17 +++++++--- .../Standards/Invoke-ExecStandardsRun.ps1 | 14 ++++---- .../Push-CIPPFunctionProcessor.ps1 | 34 ------------------- .../Start-CIPPProcessorQueue.ps1 | 23 +++++++++++++ 7 files changed, 70 insertions(+), 55 deletions(-) delete mode 100644 Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 create mode 100644 Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-CIPPProcessorQueue.ps1 diff --git a/CIPPTimers.json b/CIPPTimers.json index dd15ba8095ba..aed1c3313710 100644 --- a/CIPPTimers.json +++ b/CIPPTimers.json @@ -7,6 +7,13 @@ "RunOnProcessor": true, "PreferredProcessor": "usertasks" }, + { + "Command": "Start-CIPPProcessorQueue", + "Description": "Timer to handle user initiated tasks", + "Cron": "0 */15 * * * *", + "Priority": 1, + "RunOnProcessor": true + }, { "Command": "Start-AuditLogOrchestrator", "Description": "Orchestrator to process audit logs", diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Endpoint/Applications/Invoke-ExecAppUpload.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Endpoint/Applications/Invoke-ExecAppUpload.ps1 index 528f2c66c40d..2202e7f58ccd 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Endpoint/Applications/Invoke-ExecAppUpload.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Endpoint/Applications/Invoke-ExecAppUpload.ps1 @@ -14,23 +14,27 @@ function Invoke-ExecAppUpload { if ($Config -and $Config.state -eq $true) { if ($env:CIPP_PROCESSOR -ne 'true') { $ProcessorFunction = [PSCustomObject]@{ - FunctionName = 'CIPPFunctionProcessor' + PartitionKey = 'Function' + RowKey = 'Start-ApplicationOrchestrator' ProcessorFunction = 'Start-ApplicationOrchestrator' } - Push-OutputBinding -Name QueueItem -Value $ProcessorFunction + $ProcessorQueue = Get-CIPPTable -TableName 'ProcessorQueue' + Add-AzDataTableEntity @ProcessorQueue -Entity $ProcessorFunction -Force + $Results = [pscustomobject]@{'Results' = 'Queueing application upload' } } } else { try { Start-ApplicationOrchestrator + $Results = [pscustomobject]@{'Results' = 'Started application upload' } } catch { - Write-Host "orchestrator error: $($_.Exception.Message)" + $Results = [pscustomobject]@{'Results' = "Failed to start application upload. Error: $($_.Exception.Message)" } } } $Results = [pscustomobject]@{'Results' = 'Started application queue' } Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ StatusCode = [HttpStatusCode]::OK - Body = $results + Body = $Results }) } diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecBPA.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecBPA.ps1 index faa81a43174d..3a1a16241b68 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecBPA.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecBPA.ps1 @@ -13,19 +13,23 @@ function Invoke-ExecBPA { if ($Config -and $Config.state -eq $true) { if ($env:CIPP_PROCESSOR -ne 'true') { + $ProcessorQueue = Get-CIPPTable -TableName 'ProcessorQueue' $ProcessorFunction = [PSCustomObject]@{ - FunctionName = 'CIPPFunctionProcessor' - ProcessorFunction = 'Start-BPAOrchestrator' - Parameters = [PSCustomObject]@{ - TenantFilter = $Request.Query.TenantFilter - } + PartitionKey = 'Function' + RowKey = "Start-BPAOrchestrator-$($Request.Query.TenantFilter)" + FunctionName = 'Start-BPAOrchestrator' + Parameters = [string](ConvertTo-Json -Compress -InputObject @{ + TenantFilter = $Request.Query.TenantFilter + }) } - Push-OutputBinding -Name QueueItem -Value $ProcessorFunction + Add-AzDataTableEntity @ProcessorQueue -Entity $ProcessorFunction -Force + $Results = [pscustomobject]@{'Results' = 'BPA queued for execution' } } } else { Start-BPAOrchestrator -TenantFilter $Request.Query.TenantFilter + $Results = [pscustomobject]@{'Results' = 'BPA started' } } - $Results = [pscustomobject]@{'Results' = 'BPA started' } + Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ StatusCode = [HttpStatusCode]::OK Body = $Results diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecDomainAnalyser.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecDomainAnalyser.ps1 index 3cee5d844438..2b31c6bb1d44 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecDomainAnalyser.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecDomainAnalyser.ps1 @@ -14,17 +14,26 @@ function Invoke-ExecDomainAnalyser { if ($Config -and $Config.state -eq $true) { if ($env:CIPP_PROCESSOR -ne 'true') { $ProcessorFunction = [PSCustomObject]@{ - FunctionName = 'CIPPFunctionProcessor' + PartitionKey = 'Function' + RowKey = 'Start-DomainOrchestrator' ProcessorFunction = 'Start-DomainOrchestrator' } - Push-OutputBinding -Name QueueItem -Value $ProcessorFunction + $ProcessorQueue = Get-CIPPTable -TableName 'ProcessorQueue' + Add-AzDataTableEntity @ProcessorQueue -Entity $ProcessorFunction -Force + $Results = [pscustomobject]@{'Results' = 'Queueing Domain Analyser' } } } else { - Start-DomainOrchestrator + $OrchStatus = Start-DomainOrchestrator + if ($OrchStatus) { + $Message = 'Domain Analyser started' + } else { + $Message = 'Domain Analyser error: check logs' + } + $Results = [pscustomobject]@{'Results' = $Message } } Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ StatusCode = [HttpStatusCode]::OK - Body = $results + Body = $Results }) } diff --git a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecStandardsRun.ps1 b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecStandardsRun.ps1 index 3452d9b33d00..2b3200bf553f 100644 --- a/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecStandardsRun.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Tenant/Standards/Invoke-ExecStandardsRun.ps1 @@ -19,14 +19,16 @@ Function Invoke-ExecStandardsRun { if ($Config -and $Config.state -eq $true) { if ($env:CIPP_PROCESSOR -ne 'true') { $ProcessorFunction = [PSCustomObject]@{ - FunctionName = 'CIPPFunctionProcessor' + PartitionKey = 'Function' + RowKey = "Invoke-CIPPStandardsRun-$tenantfilter" ProcessorFunction = 'Invoke-CIPPStandardsRun' - Parameters = [PSCustomObject]@{ - TenantFilter = $tenantfilter - Force = $true - } + Parameters = [string](ConvertTo-Json -Compress -InputObject @{ + TenantFilter = $tenantfilter + Force = $true + }) } - Push-OutputBinding -Name QueueItem -Value $ProcessorFunction + $ProcessorQueue = Get-CIPPTable -TableName 'ProcessorQueue' + Add-AzDataTableEntity @ProcessorQueue -Entity $ProcessorFunction -Force $Results = "Successfully Queued Standards Run for Tenant $tenantfilter" } } else { diff --git a/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 b/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 deleted file mode 100644 index 26111a5dc609..000000000000 --- a/Modules/CIPPCore/Public/Entrypoints/Queue Function/Push-CIPPFunctionProcessor.ps1 +++ /dev/null @@ -1,34 +0,0 @@ -function Push-CIPPFunctionProcessor { - <# - .SYNOPSIS - Starts a specified function on the processor node - #> - [CmdletBinding()] - param($QueueItem, $TriggerMetadata) - - Write-Information 'Processor - Received message from queue' - - $ConfigTable = Get-CIPPTable -tablename Config - $Config = Get-CIPPAzDataTableEntity @ConfigTable -Filter "PartitionKey eq 'OffloadFunctions' and RowKey eq 'OffloadFunctions'" - - $FunctionName = $env:WEBSITE_SITE_NAME - if ($FunctionName -match '-') { - $Node = ($FunctionName -split '-')[1] - } else { - $Node = 'http' - } - - if ($env:CIPP_PROCESSOR -ne 'true') { - return - } - if ($Config -and $Config.state -eq $true -and $Node -eq 'proc') { - return - } - - $Parameters = $QueueItem.Parameters | ConvertTo-Json -Depth 10 | ConvertFrom-Json -AsHashtable - if (Get-Command -Name $QueueItem.ProcessorFunction -Module CIPPCore -ErrorAction SilentlyContinue) { - & $QueueItem.ProcessorFunction @Parameters - } else { - Write-Warning "Function $($QueueItem.ProcessorFunction) not found" - } -} diff --git a/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-CIPPProcessorQueue.ps1 b/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-CIPPProcessorQueue.ps1 new file mode 100644 index 000000000000..aba2c3e3a84f --- /dev/null +++ b/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-CIPPProcessorQueue.ps1 @@ -0,0 +1,23 @@ +function Start-CIPPProcessorQueue { + <# + .SYNOPSIS + Starts a specified function on the processor node + #> + [CmdletBinding(SupportsShouldProcess = $true)] + param() + + $QueueTable = Get-CIPPTable -tablename 'ProcessorQueue' + $QueueItems = Get-CIPPAzDataTableEntity @QueueTable -Filter "PartitionKey eq 'Function'" + + foreach ($QueueItem in $QueueItems) { + if ($PSCmdlet.ShouldProcess("Processing function $($QueueItem.ProcessorFunction)")) { + Remove-AzDataTableEntity @QueueTable -Entity $QueueItem + $Parameters = $Queue.Parameters | ConvertFrom-Json -AsHashtable + if (Get-Command -Name $QueueItem.ProcessorFunction -Module CIPPCore -ErrorAction SilentlyContinue) { + & $QueueItem.ProcessorFunction @Parameters + } else { + Write-Warning "Function $($QueueItem.ProcessorFunction) not found" + } + } + } +} From 1f446323e8e2b9f4744c0370888a7848eb392d3a Mon Sep 17 00:00:00 2001 From: John Duprey Date: Mon, 16 Sep 2024 23:16:49 -0400 Subject: [PATCH 19/19] Update Start-CIPPProcessorQueue.ps1 --- .../Timer Functions/Start-CIPPProcessorQueue.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-CIPPProcessorQueue.ps1 b/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-CIPPProcessorQueue.ps1 index aba2c3e3a84f..ed6e09bebf62 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-CIPPProcessorQueue.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-CIPPProcessorQueue.ps1 @@ -12,11 +12,11 @@ function Start-CIPPProcessorQueue { foreach ($QueueItem in $QueueItems) { if ($PSCmdlet.ShouldProcess("Processing function $($QueueItem.ProcessorFunction)")) { Remove-AzDataTableEntity @QueueTable -Entity $QueueItem - $Parameters = $Queue.Parameters | ConvertFrom-Json -AsHashtable - if (Get-Command -Name $QueueItem.ProcessorFunction -Module CIPPCore -ErrorAction SilentlyContinue) { - & $QueueItem.ProcessorFunction @Parameters + $Parameters = $QueueItem.Parameters | ConvertFrom-Json -AsHashtable + if (Get-Command -Name $QueueItem.FunctionName -Module CIPPCore -ErrorAction SilentlyContinue) { + & $QueueItem.FunctionName @Parameters } else { - Write-Warning "Function $($QueueItem.ProcessorFunction) not found" + Write-Warning "Function $($QueueItem.FunctionName) not found" } } }