From 53191ca2ad72e3ca9defae2f6a41870573eb0d60 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Fri, 24 Jan 2025 22:07:18 +0000 Subject: [PATCH 1/8] Error during the import when required Az.Resource module is not loaded #336 --- azure.datafactory.tools.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index d64320c..9bb7052 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -51,7 +51,7 @@ # ProcessorArchitecture = '' # Modules that must be imported into the global environment prior to importing this module - # RequiredModules = @('Az.DataFactory') + RequiredModules = @('Az.Resources') # Assemblies that must be loaded prior to importing this module # RequiredAssemblies = @() From d859396f640222ffa55befb2d3e95ba1a0891c7d Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Fri, 24 Jan 2025 22:16:01 +0000 Subject: [PATCH 2/8] Az.DataFactory module is required to import adftools --- azure.datafactory.tools.psd1 | 2 +- azure.datafactory.tools.psm1 | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index 9bb7052..1e2c1bc 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -51,7 +51,7 @@ # ProcessorArchitecture = '' # Modules that must be imported into the global environment prior to importing this module - RequiredModules = @('Az.Resources') + RequiredModules = @('Az.Resources', 'Az.DataFactory') # Assemblies that must be loaded prior to importing this module # RequiredAssemblies = @() diff --git a/azure.datafactory.tools.psm1 b/azure.datafactory.tools.psm1 index fe4346b..e8e70f1 100644 --- a/azure.datafactory.tools.psm1 +++ b/azure.datafactory.tools.psm1 @@ -39,7 +39,8 @@ if ($null -ne $module -and $module.Version -lt [System.Version]$minVer) } elseif ($null -eq $module) { - Write-Host "Importing module $module (> $minVer)..." + Write-Host "Importing module $moduleName (> $minVer)..." Import-Module -Name $moduleName -MinimumVersion "$minVer" -Scope Global - Write-Host "Module imported." + $module = Get-Module $moduleName + Write-Host "Module $ModuleName (v.$($module.Version)) imported." } From a626c5fc81bf21e0e87f79f28965cfbc65a70cf9 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Fri, 24 Jan 2025 22:18:06 +0000 Subject: [PATCH 3/8] updated changes log --- changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.md b/changelog.md index 228c0ce..717764f 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.12.0] - NEXT +### Added +* Error during the import when required Az.Resource module is not loaded #336 +* Az.DataFactory module is required to import adftools + ## [1.11.2] - 2024-12-04 ### Fixed * Unknown object type: SparkJobDefinition #428 by adding the type to the ignored list From 047f937497aa02041db32eb1c6550cbf87669593 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Fri, 24 Jan 2025 22:18:19 +0000 Subject: [PATCH 4/8] updated psd --- azure.datafactory.tools.psd1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index 1e2c1bc..4a4a76c 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.11.2' + ModuleVersion = '1.12.0' # Supported PSEditions # CompatiblePSEditions = @() @@ -27,7 +27,7 @@ CompanyName = 'SQLPlayer' # Copyright statement for this module - Copyright = '(c) 2020-2024 Kamil Nowinski. All rights reserved.' + Copyright = '(c) 2020-2025 Kamil Nowinski. All rights reserved.' # Description of the functionality provided by this module Description = 'PowerShell module to help with CI&CD for Azure Data Factory, mainly to publish to ADF service in multiple environments. Check https://github.com/Azure-Player/azure.datafactory.tools/ & https://azureplayer.net/adf/' From 741ea7d5c2386b210444266fd95fde2ea44c0f08 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Fri, 24 Jan 2025 23:00:50 +0000 Subject: [PATCH 5/8] Fixed: Error when deleting credentials #403 --- adhoc/issue-156/publish-credentials.ps1 | 11 ++++-- changelog.md | 2 + private/Remove-AdfObjectRestAPI.ps1 | 2 +- test/Remove-AdfObjectRestAPI.Tests.ps1 | 49 +++++++++++++++++++++++++ test/adf2/credential/credential1.json | 2 +- 5 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 test/Remove-AdfObjectRestAPI.Tests.ps1 diff --git a/adhoc/issue-156/publish-credentials.ps1 b/adhoc/issue-156/publish-credentials.ps1 index 4b7e804..c94ab44 100644 --- a/adhoc/issue-156/publish-credentials.ps1 +++ b/adhoc/issue-156/publish-credentials.ps1 @@ -1,4 +1,6 @@ -Select-AzSubscription -SubscriptionName 'Microsoft Azure Sponsorship' +$SubscriptionName = 'Microsoft Azure Sponsorship' +if ($null -eq (Get-AzContext)) { Connect-AzAccount } +Select-AzSubscription -SubscriptionName $SubscriptionName Get-AzContext . .\adhoc\~~Load-all-cmdlets-locally.ps1 # Load to this session @@ -14,7 +16,7 @@ $json = $body | ConvertFrom-Json #$resType = Get-AzureResourceType $obj.Type -$DataFactoryName = "$testAdf-17274af2" +$DataFactoryName = "$testAdf-7d6cdb5f" $ResourceGroupName = 'rg-devops-factory' $resType = 'Microsoft.DataFactory/factories/credentials' $resName = "$DataFactoryName/credential1" @@ -28,7 +30,7 @@ New-AzResource ` -IsFullObject -Force # ------------------------------------------------------------ -Select-AzSubscription -SubscriptionName 'MVP' +Select-AzSubscription -SubscriptionName $SubscriptionName # Delete credential $adfi = Get-AzDataFactoryV2 -ResourceGroupName "$ResourceGroupName" -Name "$DataFactoryName" @@ -48,3 +50,6 @@ $adf = Import-AdfFromFolder -FactoryName "$DataFactoryName" -RootFolder "$testPa Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $adfIns.Credentials[0] -adfInstance $adfIns $adfIns.Credentials[0].Name + + +Import-Module ".\azure.datafactory.tools.psd1" -Force \ No newline at end of file diff --git a/changelog.md b/changelog.md index 717764f..0c30f96 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added * Error during the import when required Az.Resource module is not loaded #336 * Az.DataFactory module is required to import adftools +### Fixed +* Error when deleting credentials #403 ## [1.11.2] - 2024-12-04 ### Fixed diff --git a/private/Remove-AdfObjectRestAPI.ps1 b/private/Remove-AdfObjectRestAPI.ps1 index 2d54259..290f928 100644 --- a/private/Remove-AdfObjectRestAPI.ps1 +++ b/private/Remove-AdfObjectRestAPI.ps1 @@ -13,7 +13,7 @@ function Remove-AdfObjectRestAPI { 'Content-Type' = 'application/json' 'Authorization' = 'Bearer ' + $token.Token } - $url = "https://management.azure.com$($adfi.DataFactoryId)/$type_plural/$($name)?api-version=2018-06-01" + $url = "https://management.azure.com$($adfInstance.Id)/$type_plural/$($name)?api-version=2018-06-01" # Delete given object via Rest API $r = Invoke-RestMethod -Method 'DELETE' -Uri $url -Headers $authHeader -ContentType "application/json" diff --git a/test/Remove-AdfObjectRestAPI.Tests.ps1 b/test/Remove-AdfObjectRestAPI.Tests.ps1 new file mode 100644 index 0000000..09bbbcc --- /dev/null +++ b/test/Remove-AdfObjectRestAPI.Tests.ps1 @@ -0,0 +1,49 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.datafactory.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + + Import-Module -Name $moduleManifestPath -Force -Verbose:$false +} + +InModuleScope azure.datafactory.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + # Variables for use in tests + $t = Get-TargetEnv 'adf2' + $script:DataFactoryOrigName = $t.DataFactoryOrigName + $script:DataFactoryName = $t.DataFactoryName + $script:Location = $t.Location + $script:ResourceGroupName = $t.ResourceGroupName + $script:Stage = 'UAT' + $script:RootFolder = Join-Path $PSScriptRoot $t.DataFactoryOrigName + $script:adf = Import-AdfFromFolder -FactoryName $script:DataFactoryName -RootFolder $script:RootFolder + $adf.ResourceGroupName = $t.ResourceGroupName + $adf.Region = $t.Location + $o = New-AdfPublishOption + $o.StopStartTriggers = $false + $o.Includes.Add('cred*.*','') + $adf.PublishOptions = $o + Publish-AdfV2FromJson -RootFolder $RootFolder -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName -Option $o -Location $Location + $script:adfIns = Get-AdfFromService -FactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName + + + Describe 'Remove-AdfObjectRestAPI' { + It 'Should exist' { + { Get-Command -Name 'Remove-AdfObjectRestAPI' -ErrorAction Stop } | Should -Not -Throw + } + + Context 'When called Remove-AdfObjectRestAPI' { + It 'Should delete the existing credential object' { + $objCountBefore = $adfIns.AllObjects().Count + Remove-AdfObjectRestAPI -type_plural 'credentials' -name 'credential1' -adfInstance $script:adfIns + $adfIns = Get-AdfFromService -FactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName + $objCountAfter = $adfIns.AllObjects().Count + $objCountAfter | Should -Be ($objCountBefore-1) + } + } + + + } +} diff --git a/test/adf2/credential/credential1.json b/test/adf2/credential/credential1.json index 0869cdb..86a65f0 100644 --- a/test/adf2/credential/credential1.json +++ b/test/adf2/credential/credential1.json @@ -3,7 +3,7 @@ "properties": { "type": "ManagedIdentity", "typeProperties": { - "resourceId": "/subscriptions/7d6cdb5f-be97-4907-a34c-66635a3f607d/resourceGroups/ADF.procfwk/providers/Microsoft.ManagedIdentity/userAssignedIdentities/iau-test1" + "resourceId": "/subscriptions/7d6cdb5f-be97-4907-a34c-66635a3f607d/resourceGroups/rg-devops-factory/providers/Microsoft.ManagedIdentity/userAssignedIdentities/adftoolstest1" } } } \ No newline at end of file From b97a69c7931fccc27564802c99bfbc6dba2a2faa Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 26 Feb 2025 22:18:43 +0000 Subject: [PATCH 6/8] Prep for unit test #435 --- public/Export-AdfToArmTemplate.ps1 | 2 +- test/Export-AdfToArmTemplate.Tests.ps1 | 2 +- test/adf2/trigger/TR_without_pipeline.json | 34 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 test/adf2/trigger/TR_without_pipeline.json diff --git a/public/Export-AdfToArmTemplate.ps1 b/public/Export-AdfToArmTemplate.ps1 index e8c0d57..824d356 100644 --- a/public/Export-AdfToArmTemplate.ps1 +++ b/public/Export-AdfToArmTemplate.ps1 @@ -9,7 +9,7 @@ function Export-AdfToArmTemplate { [parameter(Mandatory = $false)] [String] $ResourceGroup = 'abcxyz', [parameter(Mandatory = $false)] - [String] $AdfUtilitiesVersion = '0.1.6', + [String] $AdfUtilitiesVersion = '1.0.2', [parameter(Mandatory = $false)] [String] $OutputFolder = 'ArmTemplate' ) diff --git a/test/Export-AdfToArmTemplate.Tests.ps1 b/test/Export-AdfToArmTemplate.Tests.ps1 index 43a7926..debee48 100644 --- a/test/Export-AdfToArmTemplate.Tests.ps1 +++ b/test/Export-AdfToArmTemplate.Tests.ps1 @@ -36,7 +36,7 @@ InModuleScope azure.datafactory.tools { Describe 'Export-AdfToArmTemplate' -Tag 'Unit' { It 'Should exist' { - { Get-Command -Name 'Publish-AdfV2UsingArm' -ErrorAction Stop } | Should -Not -Throw + { Get-Command -Name 'Export-AdfToArmTemplate' -ErrorAction Stop } | Should -Not -Throw } It 'Should completed successfully' { { Export-AdfToArmTemplate -RootFolder $script:RootFolder } | Should -Not -Throw diff --git a/test/adf2/trigger/TR_without_pipeline.json b/test/adf2/trigger/TR_without_pipeline.json new file mode 100644 index 0000000..a3df8e5 --- /dev/null +++ b/test/adf2/trigger/TR_without_pipeline.json @@ -0,0 +1,34 @@ +{ + "name": "tr_without_pipeline", + "properties": { + "description": "Backend will fail with this", + "annotations": [], + "runtimeState": "Started", + "pipelines": [ + ], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Week", + "interval": 1, + "startTime": "2023-02-15T19:32:00", + "timeZone": "FLE Standard Time", + "schedule": { + "minutes": [ + 11 + ], + "hours": [ + 5 + ], + "weekDays": [ + "Thursday", + "Friday", + "Monday", + "Tuesday", + "Wednesday" + ] + } + } + } + } +} \ No newline at end of file From f9dfc6c395e710c24968af3091255351aec78f54 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 19 Mar 2025 23:44:07 +0000 Subject: [PATCH 7/8] Added function `Set-AdfToolsAuthToken` to enable changing URL to API when target environment is different than default Global Azure #356 --- azure.datafactory.tools.psd1 | 1 + azure.datafactory.tools.psm1 | 1 + changelog.md | 1 + private/GlobalParam.ps1 | 7 +++-- private/Test-LinkedServiceConnection.ps1 | 6 ++-- public/Set-AdfToolsAuthToken.ps1 | 39 ++++++++++++++++++++++++ 6 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 public/Set-AdfToolsAuthToken.ps1 diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index 4a4a76c..44f9162 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -82,6 +82,7 @@ , 'Get-AdfDocDiagram' , 'Export-AdfToArmTemplate' , 'Test-AdfArmTemplate' + , 'Set-AdfToolsAuthToken' ) # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. diff --git a/azure.datafactory.tools.psm1 b/azure.datafactory.tools.psm1 index e8e70f1..f2dfb78 100644 --- a/azure.datafactory.tools.psm1 +++ b/azure.datafactory.tools.psm1 @@ -30,6 +30,7 @@ if (Test-Path -Path "$PSScriptRoot\public" -ErrorAction Ignore) } } +$script:BaseApiUrl = "https://management.azure.com" $moduleName = 'Az.DataFactory' $module = Get-Module $moduleName $minVer = "1.10.0" diff --git a/changelog.md b/changelog.md index 0c30f96..11cec67 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added * Error during the import when required Az.Resource module is not loaded #336 * Az.DataFactory module is required to import adftools +* Added function `Set-AdfToolsAuthToken` to enable changing URL to API when target environment is different than default Global Azure #356 ### Fixed * Error when deleting credentials #403 diff --git a/private/GlobalParam.ps1 b/private/GlobalParam.ps1 index dde71fd..f4b364a 100644 --- a/private/GlobalParam.ps1 +++ b/private/GlobalParam.ps1 @@ -12,7 +12,9 @@ function Get-GlobalParam([string]$ResourceGroupName, [string]$DataFactoryName) 'Authorization'='Bearer ' + $token.AccessToken } - $restUri = "https://management.azure.com/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/globalParameters/default?api-version=2018-06-01" + $restUri = "$BaseApiUrl/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/globalParameters/default?api-version=2018-06-01" + Write-Debug "Get-GlobalParam:Request preparing to URL: $restUri" + $params = @{ Headers = $authHeader Method = 'GET' @@ -51,7 +53,8 @@ function Set-GlobalParam([Adf] $adf) ""properties"": $gp }" - $restUri = "https://management.azure.com/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/globalParameters/default?api-version=2018-06-01" + $restUri = "$BaseApiUrl/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/globalParameters/default?api-version=2018-06-01" + Write-Debug "Set-GlobalParam:Request preparing to URL: $restUri" $params = @{ Headers = $authHeader Body = $body diff --git a/private/Test-LinkedServiceConnection.ps1 b/private/Test-LinkedServiceConnection.ps1 index 8b8f7ae..6146e22 100644 --- a/private/Test-LinkedServiceConnection.ps1 +++ b/private/Test-LinkedServiceConnection.ps1 @@ -41,7 +41,8 @@ function Get-LinkedServiceBody( ) { Write-Debug "BEGIN: Get-LinkedServiceBody()" - $ADFEndpoint = "https://management.azure.com/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/linkedservices/$($LinkedServiceName)?api-version=2018-06-01" + $ADFEndpoint = "$BaseApiUrl/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/linkedservices/$($LinkedServiceName)?api-version=2018-06-01" + Write-Debug "Get-LinkedServiceBody:Request preparing to URL: $ADFEndpoint" $request = @{ ContentType = 'application/json' @@ -93,7 +94,8 @@ function Test-LinkedServiceConnection( $body = Get-LinkedServiceBody -LinkedServiceName $LinkedServiceName -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName -BearerToken $bearerToken -SubscriptionID $SubscriptionID - $AzureEndpoint = "https://management.azure.com/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/testConnectivity?api-version=2018-06-01" + $AzureEndpoint = "$BaseApiUrl/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/testConnectivity?api-version=2018-06-01" + Write-Debug "Test-LinkedServiceConnection:Request preparing to URL: $AzureEndpoint" if ($Params) { Write-Verbose "Found input parameters for this Linked Service, adding to the API request..." diff --git a/public/Set-AdfToolsAuthToken.ps1 b/public/Set-AdfToolsAuthToken.ps1 new file mode 100644 index 0000000..bb7c56a --- /dev/null +++ b/public/Set-AdfToolsAuthToken.ps1 @@ -0,0 +1,39 @@ +<# +.SYNOPSIS + Sets the tool's URL of base API. Default is "https://management.azure.com" + +.DESCRIPTION + Sets the tool's URL of base API. Default is "https://management.azure.com". + Currently function accepts only one parameter, apiUrl, which is the URL of the Azure cloud. + In the future, the function will accept additional parameters to aquire the authentication token. + +.PARAMETER apiUrl + The Azure context. If not provided, the function connects to the Azure account and gets the context. + +.EXAMPLE + Set-AdfToolsAuthToken -apiUrl "https://management.usgovcloudapi.net" + + This command sets BaseApiUrl to "https://management.usgovcloudapi.net", which is the URL of the Azure Government cloud. + +.INPUTS + None. This function does not accept pipeline input. + +.OUTPUTS + None. This function does not return any output. + +.NOTES + +#> + +function Set-AdfToolsAuthToken { + param + ( + [string] $apiUrl + ) + + if ($apiUrl) { + $script:BaseApiUrl = $apiUrl + Write-Host "BaseApiUrl set to $apiUrl" + } + +} \ No newline at end of file From a0506765cdb70489c1b94abbcae8f7fc7f603143 Mon Sep 17 00:00:00 2001 From: Kamil Nowinski Date: Wed, 19 Mar 2025 23:52:10 +0000 Subject: [PATCH 8/8] ver.1.12.0 --- changelog.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 11cec67..fc06184 100644 --- a/changelog.md +++ b/changelog.md @@ -4,11 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [1.12.0] - NEXT +## [1.12.0] - 2025-03-20 ### Added * Error during the import when required Az.Resource module is not loaded #336 * Az.DataFactory module is required to import adftools -* Added function `Set-AdfToolsAuthToken` to enable changing URL to API when target environment is different than default Global Azure #356 +* Added function `Set-AdfToolsAuthToken` to enable changing URL to API when target environment is different than default Global Azure #356 #441 ### Fixed * Error when deleting credentials #403