Perlmutter config for Tianle (HEP-CCE)

perlmutter_config.yaml

# Perlmutter config for HEP-CCE
display_name: Permutter@NERSC
engine:
    type: GlobusComputeEngine
    worker_debug: False
    
    # Limit to a single worker per node
    # works with SimpleLauncher to get exactly 1 worker per batch job
    max_workers_per_node: 1
    
    # Disable fetching additional tasks on the worker to prevent
    # the worker from fetching >1 tasks
    drain_period: 60
    
    # Disable task prefetch on the workers. Tasks should queue on the
    # endpoint until capacity exists
    prefetch_capacity: 0
    
    # Set scaling strategy
    strategy:
        type: SimpleStrategy
        max_idletime: 60
    
    address:
        type: address_by_interface
        ifname: hsn0

    provider:
        type: SlurmProvider
        partition: debug
        
        # Parallelism of 1 forces GC to provision as many workers
        # as there are tasks.
        parallelism: 1

        # Use SimpleLauncher to force a single worker to start on 
        # the lead node of the batch job
        launcher:
            type: SimpleLauncher
            
        # string to prepend to #SBATCH blocks in the submit
        # script to the scheduler
        # For GPUs in the debug qos eg: "#SBATCH --constraint=gpu\n#SBATCH --gpus-per-node=4"
        scheduler_options: {{ OPTIONS }}

        # Your NERSC account, eg: "m0000"
        account: {{ NERSC_ACCOUNT }}

        # Command to be run before starting a worker
        # e.g., "module load Anaconda; source activate parsl_env"
        worker_init: {{ COMMAND }}

        # increase the command timeouts
        cmd_timeout: 120

        # The following block allows you to scale from 0 to MAX_NODES
        # each block(batch job) will have 2 nodes
        nodes_per_block: 2
        init_blocks: 0
        min_blocks: 0
        max_blocks: {{ MAX_NODES }}

        # Hold blocks for 10 minutes
        walltime: 00:10:00