diff --git a/adhoc/issue-156/publish-credentials.ps1 b/adhoc/issue-156/publish-credentials.ps1 index 4b7e804..c94ab44 100644 --- a/adhoc/issue-156/publish-credentials.ps1 +++ b/adhoc/issue-156/publish-credentials.ps1 @@ -1,4 +1,6 @@ -Select-AzSubscription -SubscriptionName 'Microsoft Azure Sponsorship' +$SubscriptionName = 'Microsoft Azure Sponsorship' +if ($null -eq (Get-AzContext)) { Connect-AzAccount } +Select-AzSubscription -SubscriptionName $SubscriptionName Get-AzContext . .\adhoc\~~Load-all-cmdlets-locally.ps1 # Load to this session @@ -14,7 +16,7 @@ $json = $body | ConvertFrom-Json #$resType = Get-AzureResourceType $obj.Type -$DataFactoryName = "$testAdf-17274af2" +$DataFactoryName = "$testAdf-7d6cdb5f" $ResourceGroupName = 'rg-devops-factory' $resType = 'Microsoft.DataFactory/factories/credentials' $resName = "$DataFactoryName/credential1" @@ -28,7 +30,7 @@ New-AzResource ` -IsFullObject -Force # ------------------------------------------------------------ -Select-AzSubscription -SubscriptionName 'MVP' +Select-AzSubscription -SubscriptionName $SubscriptionName # Delete credential $adfi = Get-AzDataFactoryV2 -ResourceGroupName "$ResourceGroupName" -Name "$DataFactoryName" @@ -48,3 +50,6 @@ $adf = Import-AdfFromFolder -FactoryName "$DataFactoryName" -RootFolder "$testPa Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $adfIns.Credentials[0] -adfInstance $adfIns $adfIns.Credentials[0].Name + + +Import-Module ".\azure.datafactory.tools.psd1" -Force \ No newline at end of file diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1 index d64320c..44f9162 100644 --- a/azure.datafactory.tools.psd1 +++ b/azure.datafactory.tools.psd1 @@ -12,7 +12,7 @@ RootModule = 'azure.datafactory.tools.psm1' # Version number of this module. - ModuleVersion = '1.11.2' + ModuleVersion = '1.12.0' # Supported PSEditions # CompatiblePSEditions = @() @@ -27,7 +27,7 @@ CompanyName = 'SQLPlayer' # Copyright statement for this module - Copyright = '(c) 2020-2024 Kamil Nowinski. All rights reserved.' + Copyright = '(c) 2020-2025 Kamil Nowinski. All rights reserved.' # Description of the functionality provided by this module Description = 'PowerShell module to help with CI&CD for Azure Data Factory, mainly to publish to ADF service in multiple environments. Check https://github.com/Azure-Player/azure.datafactory.tools/ & https://azureplayer.net/adf/' @@ -51,7 +51,7 @@ # ProcessorArchitecture = '' # Modules that must be imported into the global environment prior to importing this module - # RequiredModules = @('Az.DataFactory') + RequiredModules = @('Az.Resources', 'Az.DataFactory') # Assemblies that must be loaded prior to importing this module # RequiredAssemblies = @() @@ -82,6 +82,7 @@ , 'Get-AdfDocDiagram' , 'Export-AdfToArmTemplate' , 'Test-AdfArmTemplate' + , 'Set-AdfToolsAuthToken' ) # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. diff --git a/azure.datafactory.tools.psm1 b/azure.datafactory.tools.psm1 index fe4346b..f2dfb78 100644 --- a/azure.datafactory.tools.psm1 +++ b/azure.datafactory.tools.psm1 @@ -30,6 +30,7 @@ if (Test-Path -Path "$PSScriptRoot\public" -ErrorAction Ignore) } } +$script:BaseApiUrl = "https://management.azure.com" $moduleName = 'Az.DataFactory' $module = Get-Module $moduleName $minVer = "1.10.0" @@ -39,7 +40,8 @@ if ($null -ne $module -and $module.Version -lt [System.Version]$minVer) } elseif ($null -eq $module) { - Write-Host "Importing module $module (> $minVer)..." + Write-Host "Importing module $moduleName (> $minVer)..." Import-Module -Name $moduleName -MinimumVersion "$minVer" -Scope Global - Write-Host "Module imported." + $module = Get-Module $moduleName + Write-Host "Module $ModuleName (v.$($module.Version)) imported." } diff --git a/changelog.md b/changelog.md index 228c0ce..fc06184 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.12.0] - 2025-03-20 +### Added +* Error during the import when required Az.Resource module is not loaded #336 +* Az.DataFactory module is required to import adftools +* Added function `Set-AdfToolsAuthToken` to enable changing URL to API when target environment is different than default Global Azure #356 #441 +### Fixed +* Error when deleting credentials #403 + ## [1.11.2] - 2024-12-04 ### Fixed * Unknown object type: SparkJobDefinition #428 by adding the type to the ignored list diff --git a/private/GlobalParam.ps1 b/private/GlobalParam.ps1 index dde71fd..f4b364a 100644 --- a/private/GlobalParam.ps1 +++ b/private/GlobalParam.ps1 @@ -12,7 +12,9 @@ function Get-GlobalParam([string]$ResourceGroupName, [string]$DataFactoryName) 'Authorization'='Bearer ' + $token.AccessToken } - $restUri = "https://management.azure.com/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/globalParameters/default?api-version=2018-06-01" + $restUri = "$BaseApiUrl/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/globalParameters/default?api-version=2018-06-01" + Write-Debug "Get-GlobalParam:Request preparing to URL: $restUri" + $params = @{ Headers = $authHeader Method = 'GET' @@ -51,7 +53,8 @@ function Set-GlobalParam([Adf] $adf) ""properties"": $gp }" - $restUri = "https://management.azure.com/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/globalParameters/default?api-version=2018-06-01" + $restUri = "$BaseApiUrl/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/globalParameters/default?api-version=2018-06-01" + Write-Debug "Set-GlobalParam:Request preparing to URL: $restUri" $params = @{ Headers = $authHeader Body = $body diff --git a/private/Remove-AdfObjectRestAPI.ps1 b/private/Remove-AdfObjectRestAPI.ps1 index 2d54259..290f928 100644 --- a/private/Remove-AdfObjectRestAPI.ps1 +++ b/private/Remove-AdfObjectRestAPI.ps1 @@ -13,7 +13,7 @@ function Remove-AdfObjectRestAPI { 'Content-Type' = 'application/json' 'Authorization' = 'Bearer ' + $token.Token } - $url = "https://management.azure.com$($adfi.DataFactoryId)/$type_plural/$($name)?api-version=2018-06-01" + $url = "https://management.azure.com$($adfInstance.Id)/$type_plural/$($name)?api-version=2018-06-01" # Delete given object via Rest API $r = Invoke-RestMethod -Method 'DELETE' -Uri $url -Headers $authHeader -ContentType "application/json" diff --git a/private/Test-LinkedServiceConnection.ps1 b/private/Test-LinkedServiceConnection.ps1 index 8b8f7ae..6146e22 100644 --- a/private/Test-LinkedServiceConnection.ps1 +++ b/private/Test-LinkedServiceConnection.ps1 @@ -41,7 +41,8 @@ function Get-LinkedServiceBody( ) { Write-Debug "BEGIN: Get-LinkedServiceBody()" - $ADFEndpoint = "https://management.azure.com/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/linkedservices/$($LinkedServiceName)?api-version=2018-06-01" + $ADFEndpoint = "$BaseApiUrl/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/linkedservices/$($LinkedServiceName)?api-version=2018-06-01" + Write-Debug "Get-LinkedServiceBody:Request preparing to URL: $ADFEndpoint" $request = @{ ContentType = 'application/json' @@ -93,7 +94,8 @@ function Test-LinkedServiceConnection( $body = Get-LinkedServiceBody -LinkedServiceName $LinkedServiceName -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName -BearerToken $bearerToken -SubscriptionID $SubscriptionID - $AzureEndpoint = "https://management.azure.com/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/testConnectivity?api-version=2018-06-01" + $AzureEndpoint = "$BaseApiUrl/subscriptions/$SubscriptionID/resourcegroups/$ResourceGroupName/providers/Microsoft.DataFactory/factories/$DataFactoryName/testConnectivity?api-version=2018-06-01" + Write-Debug "Test-LinkedServiceConnection:Request preparing to URL: $AzureEndpoint" if ($Params) { Write-Verbose "Found input parameters for this Linked Service, adding to the API request..." diff --git a/public/Export-AdfToArmTemplate.ps1 b/public/Export-AdfToArmTemplate.ps1 index e8c0d57..824d356 100644 --- a/public/Export-AdfToArmTemplate.ps1 +++ b/public/Export-AdfToArmTemplate.ps1 @@ -9,7 +9,7 @@ function Export-AdfToArmTemplate { [parameter(Mandatory = $false)] [String] $ResourceGroup = 'abcxyz', [parameter(Mandatory = $false)] - [String] $AdfUtilitiesVersion = '0.1.6', + [String] $AdfUtilitiesVersion = '1.0.2', [parameter(Mandatory = $false)] [String] $OutputFolder = 'ArmTemplate' ) diff --git a/public/Set-AdfToolsAuthToken.ps1 b/public/Set-AdfToolsAuthToken.ps1 new file mode 100644 index 0000000..bb7c56a --- /dev/null +++ b/public/Set-AdfToolsAuthToken.ps1 @@ -0,0 +1,39 @@ +<# +.SYNOPSIS + Sets the tool's URL of base API. Default is "https://management.azure.com" + +.DESCRIPTION + Sets the tool's URL of base API. Default is "https://management.azure.com". + Currently function accepts only one parameter, apiUrl, which is the URL of the Azure cloud. + In the future, the function will accept additional parameters to aquire the authentication token. + +.PARAMETER apiUrl + The Azure context. If not provided, the function connects to the Azure account and gets the context. + +.EXAMPLE + Set-AdfToolsAuthToken -apiUrl "https://management.usgovcloudapi.net" + + This command sets BaseApiUrl to "https://management.usgovcloudapi.net", which is the URL of the Azure Government cloud. + +.INPUTS + None. This function does not accept pipeline input. + +.OUTPUTS + None. This function does not return any output. + +.NOTES + +#> + +function Set-AdfToolsAuthToken { + param + ( + [string] $apiUrl + ) + + if ($apiUrl) { + $script:BaseApiUrl = $apiUrl + Write-Host "BaseApiUrl set to $apiUrl" + } + +} \ No newline at end of file diff --git a/test/Export-AdfToArmTemplate.Tests.ps1 b/test/Export-AdfToArmTemplate.Tests.ps1 index 43a7926..debee48 100644 --- a/test/Export-AdfToArmTemplate.Tests.ps1 +++ b/test/Export-AdfToArmTemplate.Tests.ps1 @@ -36,7 +36,7 @@ InModuleScope azure.datafactory.tools { Describe 'Export-AdfToArmTemplate' -Tag 'Unit' { It 'Should exist' { - { Get-Command -Name 'Publish-AdfV2UsingArm' -ErrorAction Stop } | Should -Not -Throw + { Get-Command -Name 'Export-AdfToArmTemplate' -ErrorAction Stop } | Should -Not -Throw } It 'Should completed successfully' { { Export-AdfToArmTemplate -RootFolder $script:RootFolder } | Should -Not -Throw diff --git a/test/Remove-AdfObjectRestAPI.Tests.ps1 b/test/Remove-AdfObjectRestAPI.Tests.ps1 new file mode 100644 index 0000000..09bbbcc --- /dev/null +++ b/test/Remove-AdfObjectRestAPI.Tests.ps1 @@ -0,0 +1,49 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.datafactory.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + + Import-Module -Name $moduleManifestPath -Force -Verbose:$false +} + +InModuleScope azure.datafactory.tools { + $testHelperPath = $PSScriptRoot | Join-Path -ChildPath 'TestHelper' + Import-Module -Name $testHelperPath -Force + + # Variables for use in tests + $t = Get-TargetEnv 'adf2' + $script:DataFactoryOrigName = $t.DataFactoryOrigName + $script:DataFactoryName = $t.DataFactoryName + $script:Location = $t.Location + $script:ResourceGroupName = $t.ResourceGroupName + $script:Stage = 'UAT' + $script:RootFolder = Join-Path $PSScriptRoot $t.DataFactoryOrigName + $script:adf = Import-AdfFromFolder -FactoryName $script:DataFactoryName -RootFolder $script:RootFolder + $adf.ResourceGroupName = $t.ResourceGroupName + $adf.Region = $t.Location + $o = New-AdfPublishOption + $o.StopStartTriggers = $false + $o.Includes.Add('cred*.*','') + $adf.PublishOptions = $o + Publish-AdfV2FromJson -RootFolder $RootFolder -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName -Option $o -Location $Location + $script:adfIns = Get-AdfFromService -FactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName + + + Describe 'Remove-AdfObjectRestAPI' { + It 'Should exist' { + { Get-Command -Name 'Remove-AdfObjectRestAPI' -ErrorAction Stop } | Should -Not -Throw + } + + Context 'When called Remove-AdfObjectRestAPI' { + It 'Should delete the existing credential object' { + $objCountBefore = $adfIns.AllObjects().Count + Remove-AdfObjectRestAPI -type_plural 'credentials' -name 'credential1' -adfInstance $script:adfIns + $adfIns = Get-AdfFromService -FactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName + $objCountAfter = $adfIns.AllObjects().Count + $objCountAfter | Should -Be ($objCountBefore-1) + } + } + + + } +} diff --git a/test/adf2/credential/credential1.json b/test/adf2/credential/credential1.json index 0869cdb..86a65f0 100644 --- a/test/adf2/credential/credential1.json +++ b/test/adf2/credential/credential1.json @@ -3,7 +3,7 @@ "properties": { "type": "ManagedIdentity", "typeProperties": { - "resourceId": "/subscriptions/7d6cdb5f-be97-4907-a34c-66635a3f607d/resourceGroups/ADF.procfwk/providers/Microsoft.ManagedIdentity/userAssignedIdentities/iau-test1" + "resourceId": "/subscriptions/7d6cdb5f-be97-4907-a34c-66635a3f607d/resourceGroups/rg-devops-factory/providers/Microsoft.ManagedIdentity/userAssignedIdentities/adftoolstest1" } } } \ No newline at end of file diff --git a/test/adf2/trigger/TR_without_pipeline.json b/test/adf2/trigger/TR_without_pipeline.json new file mode 100644 index 0000000..a3df8e5 --- /dev/null +++ b/test/adf2/trigger/TR_without_pipeline.json @@ -0,0 +1,34 @@ +{ + "name": "tr_without_pipeline", + "properties": { + "description": "Backend will fail with this", + "annotations": [], + "runtimeState": "Started", + "pipelines": [ + ], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Week", + "interval": 1, + "startTime": "2023-02-15T19:32:00", + "timeZone": "FLE Standard Time", + "schedule": { + "minutes": [ + 11 + ], + "hours": [ + 5 + ], + "weekDays": [ + "Thursday", + "Friday", + "Monday", + "Tuesday", + "Wednesday" + ] + } + } + } + } +} \ No newline at end of file