azurebatch Configuration
To be used with the azurebatchdev profile by specifying the -profile azurebatchdev when running nf-core pipelines.
Custom queue and storage need to be supplied with params.az_location, params.batch_name, params.storage_name, params.principal_id, params.principal_secret, params.tenant_id
Required Parameters
--storage_name
Name of Azure blob storage account.
--az_location
The Azure Batch region where the computation is executed in VMs. Default (westus2).
--principal_id
The service principal client ID.
--principal_secret
The service principal client secret.
--tenant_id
The Azure tenant ID.
-w
The Azure Blob container to be used as Nextflow work directory (-w az://work).
--autopoolmode
Whether to use Nextflow autopool mode which creates an autoscaling pool for running Nextflow jobs. Defaults to false.
--allowpoolcreation
Allow Nextflow to create a pool for running Nextflow jobs. Defaults to false.
--deletejobs
Allow Nextflow to delete pools after completion. Defaults to true.
--acr_registry
URL to Azure container registry for private docker images.
--acr_username
Username to access private Azure container registry.
--acr_password
Password to access private Azure container registry.
Azure Batch Setup
Please refer to the Nextflow documentation which describe how to setup the Azure Batch environment.
Config file
// Nextflow config file for running on Azure batch
params {
config_profile_description = 'Azure BATCH Dev Cloud Profile'
config_profile_contact = "Venkat Malladi (@vsmalladi)'; Abhinav Sharma (@abhi18av)"
config_profile_url = 'https://azure.microsoft.com/services/batch/'
// Active Directory
principal_id = null
principal_secret = null
tenant_id = null
// Storage
storage_name = null
// Batch
az_location = "westus2"
batch_name = null
vm_type = "Standard_D8s_v3"
autopoolmode = false
allowpoolcreation = true
deletejobs = true
deletepools = false
// ACR
acr_registry = null
acr_username = null
acr_password = null
}
process {
executor = "azurebatch"
}
azure {
process {
queue = 'Standard_D2d_v4'
withLabel: process_low {
queue = 'Standard_D4d_v4'
}
withLabel: process_medium {
queue = 'Standard_D16d_v4'
}
withLabel: process_high {
queue = 'Standard_D32d_v4'
}
withLabel: process_high_memory {
queue = 'Standard_D48d_v4'
}
}
activeDirectory {
servicePrincipalId = params.principal_id
servicePrincipalSecret = params.principal_secret
tenantId = params.tenant_id
}
storage {
accountName = params.storage_name
}
batch {
location = params.az_location
accountName = params.batch_name
tokenDuration = "24h"
autoPoolMode = params.autopoolmode
allowPoolCreation = params.allowpoolcreation
deleteJobsOnCompletion = params.deletejobs
deletePoolsOnCompletion = params.deletepools
pools {
Standard_D2d_v4 {
autoScale = true
vmType = 'Standard_D2d_v4'
vmCount = 2
maxVmCount = 20
scaleFormula = '''
$TargetLowPriorityNodes = 1;
$TargetDedicatedNodes = 0;
$NodeDeallocationOption = taskcompletion;
'''
}
Standard_D4d_v4 {
autoScale = true
vmType = 'Standard_D4d_v4'
vmCount = 2
maxVmCount = 20
scaleFormula = '''
$TargetLowPriorityNodes = 1;
$TargetDedicatedNodes = 0;
$NodeDeallocationOption = taskcompletion;
'''
}
Standard_D16d_v4 {
autoScale = true
vmType = 'Standard_D16d_v4'
vmCount = 2
maxVmCount = 20
scaleFormula = '''
$TargetLowPriorityNodes = 1;
$TargetDedicatedNodes = 0;
$NodeDeallocationOption = taskcompletion;
'''
}
Standard_D32d_v4 {
autoScale = true
vmType = 'Standard_D32d_v4'
vmCount = 2
maxVmCount = 20
scaleFormula = '''
$TargetLowPriorityNodes = 1;
$TargetDedicatedNodes = 0;
$NodeDeallocationOption = taskcompletion;
'''
}
Standard_D48d_v4 {
autoScale = true
vmType = 'Standard_D48d_v4'
vmCount = 2
maxVmCount = 10
scaleFormula = '''
$TargetLowPriorityNodes = 1;
$TargetDedicatedNodes = 0;
$NodeDeallocationOption = taskcompletion;
'''
}
}
}
registry {
server = params.acr_registry
userName = params.acr_username
password = params.acr_password
}
}
executor
contact
